From d650aa03229350e3c7edfc55defd305585fc9bc8 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 4 Dec 2024 01:28:21 -0500 Subject: [PATCH 001/408] GH-14: Add JNI CI test (#441) Fixes #14. --- .env | 2 +- .github/workflows/test.yml | 11 +++-- ci/docker/conda-jni.dockerfile | 30 +++++++++++++ ci/scripts/java_jni_build.sh | 80 ++++++++++++++++++++++++++++++++++ docker-compose.yml | 42 +++++++++++++++--- 5 files changed, 155 insertions(+), 10 deletions(-) create mode 100644 ci/docker/conda-jni.dockerfile create mode 100755 ci/scripts/java_jni_build.sh diff --git a/.env b/.env index bb9f63ab30..b50a16eb01 100644 --- a/.env +++ b/.env @@ -47,4 +47,4 @@ ULIMIT_CORE=-1 # Default versions for various dependencies JDK=11 -MAVEN=3.9.6 +MAVEN=3.9.9 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 473ce84ca7..69c8db0f15 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,7 +38,7 @@ env: jobs: ubuntu: - name: AMD64 Ubuntu 22.04 JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }} + name: AMD64 ${{ matrix.name }} JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }} runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} timeout-minutes: 30 @@ -46,8 +46,13 @@ jobs: fail-fast: false matrix: jdk: [11, 17, 21, 22] - maven: [3.9.6] - image: [java] + maven: [3.9.9] + image: [ubuntu, conda-jni-cdata] + include: + - image: ubuntu + name: "Ubuntu" + - image: conda-jni-cdata + name: "Conda JNI" env: JDK: ${{ matrix.jdk }} MAVEN: ${{ matrix.maven }} diff --git a/ci/docker/conda-jni.dockerfile b/ci/docker/conda-jni.dockerfile new file mode 100644 index 0000000000..e14db73688 --- /dev/null +++ b/ci/docker/conda-jni.dockerfile @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM ghcr.io/mamba-org/micromamba:ubuntu24.04 + +ARG jdk=11 +ARG maven=3.9.9 + +RUN micromamba install -y \ + -c conda-forge \ + cmake \ + compilers \ + maven=${maven} \ + ninja \ + openjdk=${jdk} && \ + micromamba clean --all diff --git a/ci/scripts/java_jni_build.sh b/ci/scripts/java_jni_build.sh new file mode 100755 index 0000000000..7b63dbeeaf --- /dev/null +++ b/ci/scripts/java_jni_build.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eo pipefail + +arrow_dir=${1} +arrow_install_dir=${2} +build_dir=${3}/java_jni +# The directory where the final binaries will be stored when scripts finish +dist_dir=${4} +prefix_dir="${build_dir}/java-jni" + +echo "=== Clear output directories and leftovers ===" +# Clear output directories and leftovers +rm -rf ${build_dir} + +echo "=== Building Arrow Java C Data Interface native library ===" +mkdir -p "${build_dir}" +pushd "${build_dir}" + +case "$(uname)" in + Linux) + n_jobs=$(nproc) + ;; + Darwin) + n_jobs=$(sysctl -n hw.logicalcpu) + ;; + *) + n_jobs=${NPROC:-1} + ;; +esac + +: ${ARROW_JAVA_BUILD_TESTS:=${ARROW_BUILD_TESTS:-OFF}} +: ${CMAKE_BUILD_TYPE:=release} +cmake \ + -DARROW_JAVA_JNI_ENABLE_DATASET=${ARROW_DATASET:-OFF} \ + -DARROW_JAVA_JNI_ENABLE_GANDIVA=${ARROW_GANDIVA:-OFF} \ + -DARROW_JAVA_JNI_ENABLE_ORC=${ARROW_ORC:-OFF} \ + -DBUILD_TESTING=${ARROW_JAVA_BUILD_TESTS} \ + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ + -DCMAKE_PREFIX_PATH=${arrow_install_dir} \ + -DCMAKE_INSTALL_PREFIX=${prefix_dir} \ + -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \ + -DProtobuf_USE_STATIC_LIBS=ON \ + -GNinja \ + ${JAVA_JNI_CMAKE_ARGS:-} \ + ${arrow_dir} +export CMAKE_BUILD_PARALLEL_LEVEL=${n_jobs} +cmake --build . --config ${CMAKE_BUILD_TYPE} +if [ "${ARROW_JAVA_BUILD_TESTS}" = "ON" ]; then + ctest \ + --output-on-failure \ + --parallel ${n_jobs} \ + --timeout 300 +fi +cmake --build . --config ${CMAKE_BUILD_TYPE} --target install +popd + +mkdir -p ${dist_dir} +# For Windows. *.dll are installed into bin/ on Windows. +if [ -d "${prefix_dir}/bin" ]; then + mv ${prefix_dir}/bin/* ${dist_dir}/ +else + mv ${prefix_dir}/lib/* ${dist_dir}/ +fi diff --git a/docker-compose.yml b/docker-compose.yml index 103f2f3ad0..ae378865b3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,18 +30,48 @@ volumes: name: maven-cache services: - java: + ubuntu: # Usage: - # docker compose build java - # docker compose run java + # docker compose build ubuntu + # docker compose run ubuntu # Parameters: - # MAVEN: 3.9.6 + # MAVEN: 3.9.9 # JDK: 11, 17, 21 image: ${ARCH}/maven:${MAVEN}-eclipse-temurin-${JDK} - volumes: &java-volumes + volumes: - .:/arrow-java:delegated - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated - command: &java-command > + command: /bin/bash -c " /arrow-java/ci/scripts/java_build.sh /arrow-java /build && /arrow-java/ci/scripts/java_test.sh /arrow-java /build" + + conda-jni-cdata: + # Usage: + # docker compose build conda-jni-cdata + # docker compose run conda-jni-cdata + # Parameters: + # MAVEN: 3.9.9 + # JDK: 11, 17, 21 + image: ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration + build: + context: . + dockerfile: ci/docker/conda-jni.dockerfile + cache_from: + - ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration + args: + jdk: ${JDK} + maven: ${MAVEN} + # required to use micromamba with rootless docker + # https://github.com/mamba-org/micromamba-docker/issues/407#issuecomment-2088523507 + user: root + volumes: + - .:/arrow-java:delegated + - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated + environment: + ARROW_JAVA_CDATA: "ON" + command: + /bin/bash -c " + /arrow-java/ci/scripts/java_jni_build.sh /arrow-java /build/jni /build /jni && + /arrow-java/ci/scripts/java_build.sh /arrow-java /build /jni && + /arrow-java/ci/scripts/java_test.sh /arrow-java /build /jni" From be89beccc40fcb945eee3468c6ce15787a9c9559 Mon Sep 17 00:00:00 2001 From: Gang Wu Date: Fri, 6 Dec 2024 00:35:21 +0800 Subject: [PATCH 002/408] GH-22: Add CODEOWNERS (#444) Fixes #22. Co-authored-by: Sutou Kouhei --- .github/CODEOWNERS | 56 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000..9e8acb8d85 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Any committer can add themselves to any of the path patterns +# and will subsequently get requested as a reviewer for any PRs +# that change matching files. +# +# This file uses .gitignore syntax with a few exceptions see the +# documentation about the syntax: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners + +# modules +/adapter/ @lidavidm @laurentgo @wgtmac +/algorithm/ @lidavidm @laurentgo @wgtmac +/arrow-format/ @lidavidm @laurentgo @wgtmac +/c/ @lidavidm @laurentgo @wgtmac +/compression/ @lidavidm @laurentgo @wgtmac +/dataset/ @lidavidm @laurentgo @wgtmac +/flight/ @lidavidm @laurentgo @wgtmac +/format/ @lidavidm @laurentgo @wgtmac +/gandiva/ @lidavidm @laurentgo @wgtmac +/memory/ @lidavidm @laurentgo @wgtmac +/performance/ @lidavidm @laurentgo @wgtmac +/tools/ @lidavidm @laurentgo @wgtmac +/vector/ @lidavidm @laurentgo @wgtmac +CMakeLists.txt @lidavidm @laurentgo @wgtmac + +# release scripts +/ci/ @lidavidm @laurentgo @wgtmac @kou +/dev/ @lidavidm @laurentgo @wgtmac @kou + +# PR CI and repository files +/.github/ @lidavidm @laurentgo @wgtmac @kou +/.asf.yaml @lidavidm @laurentgo @wgtmac @kou +/.env @lidavidm @laurentgo @wgtmac +/.pre-commit-config.yaml @lidavidm @laurentgo @wgtmac +/Brewfile @lidavidm @laurentgo @wgtmac +/docker-compose.yaml @lidavidm @laurentgo @wgtmac + +# Java specific +/.mvn/ @lidavidm @laurentgo @wgtmac +/bom/ @lidavidm @laurentgo @wgtmac +pom.xml @lidavidm @laurentgo @wgtmac From e82aba879e768501a0a8576eccc5ad7b2be7d6ba Mon Sep 17 00:00:00 2001 From: Laurent Goujon Date: Mon, 9 Dec 2024 16:36:28 -0800 Subject: [PATCH 003/408] MINOR: Update latest java version to 23 (#452) Update latest Java version used by Github workflows to Java 23. --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 69c8db0f15..6f8c954881 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,7 +45,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17, 21, 22] + jdk: [11, 17, 21, 23] maven: [3.9.9] image: [ubuntu, conda-jni-cdata] include: From 3ef5450919dd3ebc0b566d1556c33c1207a10514 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 11 Dec 2024 19:38:18 -0500 Subject: [PATCH 004/408] GH-436: Remove Maven RAT check in favor of pre-commit check (#442) Fixes #436. --- pom.xml | 71 --------------------------------------------------------- 1 file changed, 71 deletions(-) diff --git a/pom.xml b/pom.xml index f2c8d8f1f6..82ac714a9a 100644 --- a/pom.xml +++ b/pom.xml @@ -397,19 +397,6 @@ under the License. - - - org.apache.rat - apache-rat-plugin - [0.10,) - - check - - - - - - org.apache.maven.plugins @@ -540,64 +527,6 @@ under the License. - - org.apache.rat - apache-rat-plugin - - false - - **/dependency-reduced-pom.xml - **/*.log - **/*.css - **/*.js - **/*.md - **/*.eps - **/*.json - **/*.seq - **/*.parquet - **/*.sql - **/arrow-git.properties - **/*.csv - **/*.csvh - **/*.csvh-test - **/*.tsv - **/*.txt - **/*.ssv - **/arrow-*.conf - **/.buildpath - **/*.proto - **/*.fmpp - **/target/** - **/*.tdd - **/*.project - **/TAGS - **/*.checkstyle - **/.classpath - **/.factorypath - **/.settings/** - .*/** - **/*.patch - **/*.pb.cc - **/*.pb.h - **/*.linux - **/client/build/** - **/*.tbl - **/*.iml - **/flight.properties - **/*.idea/** - - - - - rat-checks - - check - - validate - - - - org.apache.maven.plugins maven-jar-plugin From 480e1be6b7c5fa7bb0aab67c6f700059e4fcd3ea Mon Sep 17 00:00:00 2001 From: Kazuaki Ishizaki Date: Thu, 26 Dec 2024 11:51:35 +0900 Subject: [PATCH 005/408] GH-10: Add basic checks into pre-commit-config (#435) Fixes #10 Fixes #11 `Removed gh 12 now` --- .editorconfig | 20 +++++++++++++ .pre-commit-config.yaml | 13 ++++++++- ci/scripts/java_build.sh | 15 +++++----- ci/scripts/java_jni_build.sh | 56 ++++++++++++++++++------------------ ci/scripts/java_test.sh | 14 +++++---- 5 files changed, 75 insertions(+), 43 deletions(-) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000..1b04cdad3c --- /dev/null +++ b/.editorconfig @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[*.sh] +indent_style = space +indent_size = 2 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 63b7ab9f91..64f9e1cda9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,7 +25,6 @@ repos: - id: check-added-large-files - id: file-contents-sorter files: .gitignore - - repo: local hooks: - id: rat @@ -39,3 +38,15 @@ repos: dev/release/run_rat.sh apache-arrow-go.tar.gz" always_run: true pass_filenames: false + - repo: https://github.com/koalaman/shellcheck-precommit + rev: v0.10.0 + hooks: + - id: shellcheck + - repo: https://github.com/scop/pre-commit-shfmt + rev: v3.9.0-1 + hooks: + - id: shfmt + args: + # The default args is "--write --simplify" but we don't use + # "--simplify". Because it's conflicted will ShellCheck. + - "--write" diff --git a/ci/scripts/java_build.sh b/ci/scripts/java_build.sh index b5a12d9171..f3f2023759 100755 --- a/ci/scripts/java_build.sh +++ b/ci/scripts/java_build.sh @@ -22,16 +22,15 @@ if [[ "${ARROW_JAVA_BUILD:-ON}" != "ON" ]]; then exit fi -arrow_dir=${1} source_dir=${1} build_dir=${2} java_jni_dist_dir=${3} -: ${BUILD_DOCS_JAVA:=OFF} +: "${BUILD_DOCS_JAVA:=OFF}" mvn="mvn -B -DskipTests -Drat.skip=true -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" -if [ $ARROW_JAVA_SKIP_GIT_PLUGIN ]; then +if [ "$ARROW_JAVA_SKIP_GIT_PLUGIN" ]; then mvn="${mvn} -Dmaven.gitcommitid.skip=true" fi @@ -50,13 +49,13 @@ cp -r "${source_dir}/dev" "${build_dir}" poms=$(find "${source_dir}" -not \( -path "${source_dir}"/build -prune \) -type f -name pom.xml) if [[ "$OSTYPE" == "darwin"* ]]; then - poms=$(echo "$poms" | xargs -n1 python -c "import sys; import os.path; print(os.path.relpath(sys.argv[1], '${source_dir}'))") + poms=$(echo "$poms" | xargs -n1 python -c "import sys; import os.path; print(os.path.relpath(sys.argv[1], '${source_dir}'))") else - poms=$(echo "$poms" | xargs -n1 realpath -s --relative-to="${source_dir}") + poms=$(echo "$poms" | xargs -n1 realpath -s --relative-to="${source_dir}") fi for source_root in $(echo "${poms}" | awk -F/ '{print $1}' | sort -u); do - cp -r "${source_dir}/${source_root}" "${build_dir}" + cp -r "${source_dir}/${source_root}" "${build_dir}" done pushd "${build_dir}" @@ -79,9 +78,9 @@ ${mvn} -T 2C clean install if [ "${BUILD_DOCS_JAVA}" == "ON" ]; then # HTTP pooling is turned of to avoid download issues https://issues.apache.org/jira/browse/ARROW-11633 # GH-43378: Maven site plugins not compatible with multithreading - mkdir -p ${build_dir}/docs/java/reference + mkdir -p "${build_dir}"/docs/java/reference ${mvn} -Dcheckstyle.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false clean install site - rsync -a target/site/apidocs/ ${build_dir}/docs/java/reference + rsync -a target/site/apidocs/ "${build_dir}"/docs/java/reference fi popd diff --git a/ci/scripts/java_jni_build.sh b/ci/scripts/java_jni_build.sh index 7b63dbeeaf..44388e33fe 100755 --- a/ci/scripts/java_jni_build.sh +++ b/ci/scripts/java_jni_build.sh @@ -27,54 +27,54 @@ prefix_dir="${build_dir}/java-jni" echo "=== Clear output directories and leftovers ===" # Clear output directories and leftovers -rm -rf ${build_dir} +rm -rf "${build_dir}" echo "=== Building Arrow Java C Data Interface native library ===" mkdir -p "${build_dir}" pushd "${build_dir}" case "$(uname)" in - Linux) - n_jobs=$(nproc) - ;; - Darwin) - n_jobs=$(sysctl -n hw.logicalcpu) - ;; - *) - n_jobs=${NPROC:-1} - ;; +Linux) + n_jobs=$(nproc) + ;; +Darwin) + n_jobs=$(sysctl -n hw.logicalcpu) + ;; +*) + n_jobs=${NPROC:-1} + ;; esac -: ${ARROW_JAVA_BUILD_TESTS:=${ARROW_BUILD_TESTS:-OFF}} -: ${CMAKE_BUILD_TYPE:=release} +: "${ARROW_JAVA_BUILD_TESTS:=${ARROW_BUILD_TESTS:-OFF}}" +: "${CMAKE_BUILD_TYPE:=release}" cmake \ - -DARROW_JAVA_JNI_ENABLE_DATASET=${ARROW_DATASET:-OFF} \ - -DARROW_JAVA_JNI_ENABLE_GANDIVA=${ARROW_GANDIVA:-OFF} \ - -DARROW_JAVA_JNI_ENABLE_ORC=${ARROW_ORC:-OFF} \ - -DBUILD_TESTING=${ARROW_JAVA_BUILD_TESTS} \ - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DCMAKE_PREFIX_PATH=${arrow_install_dir} \ - -DCMAKE_INSTALL_PREFIX=${prefix_dir} \ - -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \ + -DARROW_JAVA_JNI_ENABLE_DATASET="${ARROW_DATASET:-OFF}" \ + -DARROW_JAVA_JNI_ENABLE_GANDIVA="${ARROW_GANDIVA:-OFF}" \ + -DARROW_JAVA_JNI_ENABLE_ORC="${ARROW_ORC:-OFF}" \ + -DBUILD_TESTING="${ARROW_JAVA_BUILD_TESTS}" \ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" \ + -DCMAKE_PREFIX_PATH="${arrow_install_dir}" \ + -DCMAKE_INSTALL_PREFIX="${prefix_dir}" \ + -DCMAKE_UNITY_BUILD="${CMAKE_UNITY_BUILD:-OFF}" \ -DProtobuf_USE_STATIC_LIBS=ON \ -GNinja \ - ${JAVA_JNI_CMAKE_ARGS:-} \ - ${arrow_dir} + "${JAVA_JNI_CMAKE_ARGS:-}" \ + "${arrow_dir}" export CMAKE_BUILD_PARALLEL_LEVEL=${n_jobs} -cmake --build . --config ${CMAKE_BUILD_TYPE} +cmake --build . --config "${CMAKE_BUILD_TYPE}" if [ "${ARROW_JAVA_BUILD_TESTS}" = "ON" ]; then ctest \ --output-on-failure \ - --parallel ${n_jobs} \ + --parallel "${n_jobs}" \ --timeout 300 fi -cmake --build . --config ${CMAKE_BUILD_TYPE} --target install +cmake --build . --config "${CMAKE_BUILD_TYPE}" --target install popd -mkdir -p ${dist_dir} +mkdir -p "${dist_dir}" # For Windows. *.dll are installed into bin/ on Windows. if [ -d "${prefix_dir}/bin" ]; then - mv ${prefix_dir}/bin/* ${dist_dir}/ + mv "${prefix_dir}"/bin/* "${dist_dir}"/ else - mv ${prefix_dir}/lib/* ${dist_dir}/ + mv "${prefix_dir}"/lib/* "${dist_dir}"/ fi diff --git a/ci/scripts/java_test.sh b/ci/scripts/java_test.sh index 9d4bc018b3..6449bbc2c5 100755 --- a/ci/scripts/java_test.sh +++ b/ci/scripts/java_test.sh @@ -22,7 +22,6 @@ if [[ "${ARROW_JAVA_TEST:-ON}" != "ON" ]]; then exit fi -arrow_dir=${1} source_dir=${1} build_dir=${2} java_jni_dist_dir=${3} @@ -31,7 +30,7 @@ mvn="mvn -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMave # Use `2 * ncores` threads mvn="${mvn} -T 2C" -pushd ${build_dir} +pushd "${build_dir}" ${mvn} -Darrow.test.dataRoot="${source_dir}/testing/data" clean test @@ -43,13 +42,16 @@ if [ "${ARROW_JAVA_JNI}" = "ON" ]; then fi if [ "${#projects[@]}" -gt 0 ]; then ${mvn} clean test \ - -Parrow-jni \ - -pl $(IFS=,; echo "${projects[*]}") \ - -Darrow.cpp.build.dir=${java_jni_dist_dir} + -Parrow-jni \ + -pl "$( + IFS=, + echo \""${projects[*]}"\" + )" \ + -Darrow.cpp.build.dir="${java_jni_dist_dir}" fi if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then - ${mvn} clean test -Parrow-c-data -pl c -Darrow.c.jni.dist.dir=${java_jni_dist_dir} + ${mvn} clean test -Parrow-c-data -pl c -Darrow.c.jni.dist.dir="${java_jni_dist_dir}" fi popd From 5a418f90f628d29e10a1cac65cedb4352d531c93 Mon Sep 17 00:00:00 2001 From: David Li Date: Sun, 29 Dec 2024 21:29:03 -0500 Subject: [PATCH 006/408] GH-466: Export namespace from Flight package (#467) Fixes #466. --- flight/flight-core/src/main/java/module-info.java | 1 + 1 file changed, 1 insertion(+) diff --git a/flight/flight-core/src/main/java/module-info.java b/flight/flight-core/src/main/java/module-info.java index 28dbb732c4..669797ac93 100644 --- a/flight/flight-core/src/main/java/module-info.java +++ b/flight/flight-core/src/main/java/module-info.java @@ -20,6 +20,7 @@ exports org.apache.arrow.flight.auth; exports org.apache.arrow.flight.auth2; exports org.apache.arrow.flight.client; + exports org.apache.arrow.flight.grpc; exports org.apache.arrow.flight.impl; exports org.apache.arrow.flight.sql.impl; From ad9bad9373bec03fdac01911977d5fc8c3703f2a Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 1 Jan 2025 23:31:32 -0500 Subject: [PATCH 007/408] GH-13: Set up JNI build (dataset, etc.) (#449) Fixes #13. --------- Co-authored-by: Sutou Kouhei --- .env | 7 + .github/workflows/test_jni.yml | 267 ++++++++++++++++++ .../arrow/adapter/orc/OrcReaderTest.java | 2 + ci/docker/vcpkg-jni.dockerfile | 46 +++ .../{java_jni_build.sh => jni_build.sh} | 5 +- ci/scripts/jni_full_build.sh | 74 +++++ ci/scripts/jni_macos_build.sh | 146 ++++++++++ ci/scripts/jni_manylinux_build.sh | 178 ++++++++++++ .../apache/arrow/dataset/TestAllTypes.java | 3 +- .../org/apache/arrow/dataset/TestDataset.java | 2 - docker-compose.yml | 34 ++- 11 files changed, 757 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/test_jni.yml create mode 100644 ci/docker/vcpkg-jni.dockerfile rename ci/scripts/{java_jni_build.sh => jni_build.sh} (95%) create mode 100755 ci/scripts/jni_full_build.sh create mode 100755 ci/scripts/jni_macos_build.sh create mode 100755 ci/scripts/jni_manylinux_build.sh diff --git a/.env b/.env index b50a16eb01..5398249b24 100644 --- a/.env +++ b/.env @@ -40,6 +40,7 @@ ARCH_SHORT=amd64 # Default repository to pull and push images from REPO=ghcr.io/apache/arrow-java-dev +ARROW_REPO=apache/arrow-dev # The setup attempts to generate coredumps by default, in order to disable the # coredump generation set it to 0 @@ -48,3 +49,9 @@ ULIMIT_CORE=-1 # Default versions for various dependencies JDK=11 MAVEN=3.9.9 + +# Versions for various dependencies used to build artifacts +# Keep in sync with apache/arrow +ARROW_REPO_ROOT=./arrow +PYTHON=3.9 +VCPKG="943c5ef1c8f6b5e6ced092b242c8299caae2ff01" # 2024.04.26 Release diff --git a/.github/workflows/test_jni.yml b/.github/workflows/test_jni.yml new file mode 100644 index 0000000000..31eb9b8743 --- /dev/null +++ b/.github/workflows/test_jni.yml @@ -0,0 +1,267 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Test (JNI) + +on: + push: + branches: + - '**' + - '!dependabot/**' + tags: + - '**' + pull_request: + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +env: + DOCKER_VOLUME_PREFIX: ".docker/" + +jobs: + cpp-ubuntu: + name: Build C++ libraries ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }} + runs-on: ${{ matrix.platform.runs_on }} + strategy: + fail-fast: false + matrix: + platform: + - runs_on: ubuntu-latest + arch: "x86_64" + archery_arch: "amd64" + env: + # architecture name used for archery build + ARCH: ${{ matrix.platform.archery_arch }} + permissions: + contents: read + packages: write + steps: + - name: Checkout apache/arrow-java + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + fetch-depth: 0 + submodules: recursive + - name: Checkout apache/arrow + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + repository: apache/arrow + fetch-depth: 0 + path: arrow + submodules: recursive + - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build C++ libraries + run: | + docker compose run vcpkg-jni + - name: Push Docker image + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow-java' && github.ref_name == 'main' + run: | + docker compose push vcpkg-jni + - name: Compress into single artifact to keep directory structure + run: tar -cvzf arrow-shared-libs-linux-${{ matrix.platform.arch }}.tar.gz dist/ + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: ubuntu-shared-lib-${{ matrix.platform.arch }} + path: arrow-shared-libs-linux-${{ matrix.platform.arch }}.tar.gz + + cpp-macos: + name: Build C++ libraries macOS ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }} + runs-on: ${{ matrix.platform.runs_on }} + strategy: + fail-fast: false + matrix: + platform: + - { runs_on: macos-13, arch: "x86_64"} + - { runs_on: macos-14, arch: "aarch_64" } + env: + MACOSX_DEPLOYMENT_TARGET: "14.0" + steps: + - name: Checkout apache/arrow-java + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + fetch-depth: 0 + submodules: recursive + - name: Checkout apache/arrow + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + repository: apache/arrow + fetch-depth: 0 + path: arrow + submodules: recursive + - name: Set up Python + uses: actions/setup-python@v4 + with: + cache: 'pip' + python-version: 3.12 + - name: Install Archery + run: pip install -e arrow/dev/archery[all] + - name: Install dependencies + run: | + # We want to use llvm@14 to avoid shared z3 + # dependency. llvm@14 doesn't depend on z3 and llvm depends + # on z3. And Homebrew's z3 provides only shared library. It + # doesn't provides static z3 because z3's CMake doesn't accept + # building both shared and static libraries at once. + # See also: Z3_BUILD_LIBZ3_SHARED in + # https://github.com/Z3Prover/z3/blob/master/README-CMake.md + # + # If llvm is installed, Apache Arrow C++ uses llvm rather than + # llvm@14 because llvm is newer than llvm@14. + brew uninstall llvm || : + + # Ensure updating python@XXX with the "--overwrite" option. + # If python@XXX is updated without "--overwrite", it causes + # a conflict error. Because Python 3 installed not by + # Homebrew exists in /usr/local on GitHub Actions. If + # Homebrew's python@XXX is updated without "--overwrite", it + # tries to replace /usr/local/bin/2to3 and so on and causes + # a conflict error. + brew update + for python_package in $(brew list | grep python@); do + brew install --overwrite ${python_package} + done + brew install --overwrite python + + if [ "$(uname -m)" = "arm64" ]; then + # pkg-config formula is deprecated but it's still installed + # in GitHub Actions runner now. We can remove this once + # pkg-config formula is removed from GitHub Actions runner. + brew uninstall pkg-config || : + brew uninstall pkg-config@0.29.2 || : + fi + + brew bundle --file=arrow/cpp/Brewfile + # We want to link aws-sdk-cpp statically but Homebrew's + # aws-sdk-cpp provides only shared library. If we have + # Homebrew's aws-sdk-cpp, our build mix Homebrew's + # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's + # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp. + brew uninstall aws-sdk-cpp + # We want to use bundled RE2 for static linking. If + # Homebrew's RE2 is installed, its header file may be used. + # We uninstall Homebrew's RE2 to ensure using bundled RE2. + brew uninstall grpc || : # gRPC depends on RE2 + brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too + brew uninstall re2 + # We want to use bundled Protobuf for static linking. If + # Homebrew's Protobuf is installed, its library file may be + # used on test We uninstall Homebrew's Protobuf to ensure using + # bundled Protobuf. + brew uninstall protobuf + + brew bundle --file=Brewfile + - name: Build C++ libraries + run: | + set -e + # make brew Java available to CMake + export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home + ./ci/scripts/jni_macos_build.sh \ + $GITHUB_WORKSPACE \ + $GITHUB_WORKSPACE/arrow \ + $GITHUB_WORKSPACE/arrow-java/cpp-build \ + $GITHUB_WORKSPACE/dist + - name: Compress into single artifact to keep directory structure + run: tar -cvzf arrow-shared-libs-macos-${{ matrix.platform.arch }}.tar.gz dist/ + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: macos-shared-lib-${{ matrix.platform.arch }} + path: arrow-shared-libs-macos-${{ matrix.platform.arch }}.tar.gz + + java-jars: + name: Build JAR files + runs-on: ubuntu-latest + needs: + - cpp-ubuntu + - cpp-macos + steps: + - name: Checkout apache/arrow-java + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + fetch-depth: 0 + submodules: recursive + - name: Checkout apache/arrow + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + repository: apache/arrow + fetch-depth: 0 + path: arrow + submodules: recursive + - name: Download Libraries + uses: actions/download-artifact@v4 + with: + path: artifacts + - name: Decompress artifacts + run: | + mv artifacts/*/*.tar.gz . + tar -xvzf arrow-shared-libs-linux-x86_64.tar.gz + # tar -xvzf arrow-shared-libs-linux-aarch_64.tar.gz + tar -xvzf arrow-shared-libs-macos-x86_64.tar.gz + tar -xvzf arrow-shared-libs-macos-aarch_64.tar.gz + # tar -xvzf arrow-shared-libs-windows.tar.gz + - name: Test that shared libraries exist + run: | + set -x + + test -f dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.so + test -f dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.so + test -f dist/arrow_orc_jni/x86_64/libarrow_orc_jni.so + test -f dist/gandiva_jni/x86_64/libgandiva_jni.so + + # test -f dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.so + # test -f dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.so + # test -f dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.so + # test -f dist/gandiva_jni/aarch_64/libgandiva_jni.so + + test -f dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.dylib + test -f dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.dylib + test -f dist/arrow_orc_jni/x86_64/libarrow_orc_jni.dylib + test -f dist/gandiva_jni/x86_64/libgandiva_jni.dylib + + test -f dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.dylib + test -f dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.dylib + test -f dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib + test -f dist/gandiva_jni/aarch_64/libgandiva_jni.dylib + + # test -f dist/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll + # test -f dist/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll + # test -f dist/arrow_orc_jni/x86_64/arrow_orc_jni.dll + - name: Build bundled jar + env: + MAVEN_ARGS: >- + --no-transfer-progress + run: | + set -e + # mvn versions:set -DnewVersion={{ arrow.no_rc_snapshot_version }} + # mvn versions:set -DnewVersion={{ arrow.no_rc_snapshot_version }} -f bom + ./ci/scripts/jni_full_build.sh \ + $GITHUB_WORKSPACE \ + $GITHUB_WORKSPACE/arrow \ + $GITHUB_WORKSPACE/dist + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: java-jars + path: ${{ github.workspace }}/arrow-java/java-dist diff --git a/adapter/orc/src/test/java/org/apache/arrow/adapter/orc/OrcReaderTest.java b/adapter/orc/src/test/java/org/apache/arrow/adapter/orc/OrcReaderTest.java index f8eb91a1cc..f48e6bb95e 100644 --- a/adapter/orc/src/test/java/org/apache/arrow/adapter/orc/OrcReaderTest.java +++ b/adapter/orc/src/test/java/org/apache/arrow/adapter/orc/OrcReaderTest.java @@ -38,6 +38,7 @@ import org.apache.orc.TypeDescription; import org.apache.orc.Writer; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -53,6 +54,7 @@ public static void beforeClass() { allocator = new RootAllocator(MAX_ALLOCATION); } + @Disabled("ORC is flaky: https://github.com/apache/arrow-java/pull/449") @Test public void testOrcJniReader() throws Exception { TypeDescription schema = TypeDescription.fromString("struct"); diff --git a/ci/docker/vcpkg-jni.dockerfile b/ci/docker/vcpkg-jni.dockerfile new file mode 100644 index 0000000000..55fa35e0d1 --- /dev/null +++ b/ci/docker/vcpkg-jni.dockerfile @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG base +FROM ${base} + +# Install the libraries required by Gandiva to run +# Use enable llvm[enable-rtti] in the vcpkg.json to avoid link problems in Gandiva +RUN vcpkg install \ + --clean-after-build \ + --x-install-root=${VCPKG_ROOT}/installed \ + --x-manifest-root=/arrow/ci/vcpkg \ + --x-feature=dev \ + --x-feature=flight \ + --x-feature=gcs \ + --x-feature=json \ + --x-feature=parquet \ + --x-feature=gandiva \ + --x-feature=s3 + +# Install Java +# We need Java for JNI headers, but we don't invoke Maven in this build. +ARG java=11 +RUN yum install -y java-$java-openjdk-devel && yum clean all + +# For ci/scripts/{cpp,java}_*.sh +ENV ARROW_HOME=/tmp/local \ + ARROW_JAVA_CDATA=ON \ + ARROW_JAVA_JNI=ON \ + ARROW_USE_CCACHE=ON + +LABEL org.opencontainers.image.source https://github.com/apache/arrow-java diff --git a/ci/scripts/java_jni_build.sh b/ci/scripts/jni_build.sh similarity index 95% rename from ci/scripts/java_jni_build.sh rename to ci/scripts/jni_build.sh index 44388e33fe..4462646347 100755 --- a/ci/scripts/java_jni_build.sh +++ b/ci/scripts/jni_build.sh @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -set -eo pipefail +set -exo pipefail arrow_dir=${1} arrow_install_dir=${2} @@ -47,6 +47,7 @@ esac : "${ARROW_JAVA_BUILD_TESTS:=${ARROW_BUILD_TESTS:-OFF}}" : "${CMAKE_BUILD_TYPE:=release}" +read -ra EXTRA_CMAKE_OPTIONS <<<"${JAVA_JNI_CMAKE_ARGS:-}" cmake \ -DARROW_JAVA_JNI_ENABLE_DATASET="${ARROW_DATASET:-OFF}" \ -DARROW_JAVA_JNI_ENABLE_GANDIVA="${ARROW_GANDIVA:-OFF}" \ @@ -58,7 +59,7 @@ cmake \ -DCMAKE_UNITY_BUILD="${CMAKE_UNITY_BUILD:-OFF}" \ -DProtobuf_USE_STATIC_LIBS=ON \ -GNinja \ - "${JAVA_JNI_CMAKE_ARGS:-}" \ + "${EXTRA_CMAKE_OPTIONS[@]}" \ "${arrow_dir}" export CMAKE_BUILD_PARALLEL_LEVEL=${n_jobs} cmake --build . --config "${CMAKE_BUILD_TYPE}" diff --git a/ci/scripts/jni_full_build.sh b/ci/scripts/jni_full_build.sh new file mode 100755 index 0000000000..1a39aeb510 --- /dev/null +++ b/ci/scripts/jni_full_build.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +arrow_java_dir="${1}" +arrow_dir="${2}" +dist_dir="${3}" + +export ARROW_TEST_DATA="${arrow_dir}/testing/data" + +pushd "${arrow_java_dir}" + +# Ensure that there is no old jar +# inside the maven repository +maven_repo=~/.m2/repository/org/apache/arrow +if [ -d "$maven_repo" ]; then + find "$maven_repo" \ + "(" -name "*.jar" -o -name "*.zip" -o -name "*.pom" ")" \ + -exec echo {} ";" \ + -exec rm -rf {} ";" +fi + +# generate dummy GPG key for -Papache-release. +# -Papache-release generates signs (*.asc) of artifacts. +# We don't use these signs in our release process. +( + echo "Key-Type: RSA" + echo "Key-Length: 4096" + echo "Name-Real: Build" + echo "Name-Email: build@example.com" + echo "%no-protection" +) | + gpg --full-generate-key --batch + +# build the entire project +mvn clean \ + install \ + -Papache-release \ + -Parrow-c-data \ + -Parrow-jni \ + -Darrow.cpp.build.dir="$dist_dir" \ + -Darrow.c.jni.dist.dir="$dist_dir" \ + --no-transfer-progress + +# copy all jar, zip and pom files to the distribution folder +find ~/.m2/repository/org/apache/arrow \ + "(" \ + -name "*.jar" -o \ + -name "*.json" -o \ + -name "*.pom" -o \ + -name "*.xml" -o \ + -name "*.zip" \ + ")" \ + -exec echo "{}" ";" \ + -exec cp "{}" "$dist_dir" ";" + +popd diff --git a/ci/scripts/jni_macos_build.sh b/ci/scripts/jni_macos_build.sh new file mode 100755 index 0000000000..eeabfd1334 --- /dev/null +++ b/ci/scripts/jni_macos_build.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script is like java_jni_build.sh, but is meant for release artifacts +# and hardcodes assumptions about the environment it is being run in. + +set -ex + +arrow_java_dir="${1}" +arrow_dir="${2}" +build_dir="${3}" +normalized_arch="$(arch)" +case "${normalized_arch}" in +arm64) + normalized_arch=aarch_64 + ;; +i386) + normalized_arch=x86_64 + ;; +esac +# The directory where the final binaries will be stored when scripts finish +dist_dir="${4}" + +echo "=== Clear output directories and leftovers ===" +# Clear output directories and leftovers +rm -rf "${build_dir}" + +echo "=== Building Arrow C++ libraries ===" +install_dir="${build_dir}/cpp-install" +: "${ARROW_ACERO:=ON}" +export ARROW_ACERO +: "${ARROW_BUILD_TESTS:=ON}" +: "${ARROW_DATASET:=ON}" +export ARROW_DATASET +: "${ARROW_GANDIVA:=ON}" +export ARROW_GANDIVA +: "${ARROW_ORC:=ON}" +export ARROW_ORC +: "${ARROW_PARQUET:=ON}" +: "${ARROW_S3:=ON}" +: "${ARROW_USE_CCACHE:=OFF}" +: "${CMAKE_BUILD_TYPE:=Release}" +: "${CMAKE_UNITY_BUILD:=ON}" + +if [ "${ARROW_USE_CCACHE}" == "ON" ]; then + echo "=== ccache statistics before build ===" + ccache -sv 2>/dev/null || ccache -s +fi + +export ARROW_TEST_DATA="${arrow_dir}/testing/data" +export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data" +export AWS_EC2_METADATA_DISABLED=TRUE + +mkdir -p "${build_dir}/cpp" +pushd "${build_dir}/cpp" + +cmake \ + -DARROW_ACERO="${ARROW_ACERO}" \ + -DARROW_BUILD_SHARED=OFF \ + -DARROW_BUILD_TESTS="${ARROW_BUILD_TESTS}" \ + -DARROW_CSV="${ARROW_DATASET}" \ + -DARROW_DATASET="${ARROW_DATASET}" \ + -DARROW_SUBSTRAIT="${ARROW_DATASET}" \ + -DARROW_DEPENDENCY_USE_SHARED=OFF \ + -DARROW_GANDIVA="${ARROW_GANDIVA}" \ + -DARROW_GANDIVA_STATIC_LIBSTDCPP=ON \ + -DARROW_JSON="${ARROW_DATASET}" \ + -DARROW_ORC="${ARROW_ORC}" \ + -DARROW_PARQUET="${ARROW_PARQUET}" \ + -DARROW_S3="${ARROW_S3}" \ + -DARROW_USE_CCACHE="${ARROW_USE_CCACHE}" \ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" \ + -DCMAKE_INSTALL_PREFIX="${install_dir}" \ + -DCMAKE_UNITY_BUILD="${CMAKE_UNITY_BUILD}" \ + -DGTest_SOURCE=BUNDLED \ + -DPARQUET_BUILD_EXAMPLES=OFF \ + -DPARQUET_BUILD_EXECUTABLES=OFF \ + -DPARQUET_REQUIRE_ENCRYPTION=OFF \ + -Dre2_SOURCE=BUNDLED \ + -GNinja \ + "${arrow_dir}/cpp" +cmake --build . --target install + +if [ "${ARROW_BUILD_TESTS}" == "ON" ]; then + # MinIO is required + exclude_tests="arrow-s3fs-test" + # unstable + exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test" + exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test" + ctest \ + --exclude-regex "${exclude_tests}" \ + --label-regex unittest \ + --output-on-failure \ + --parallel "$(sysctl -n hw.ncpu)" \ + --timeout 300 +fi + +popd + +export JAVA_JNI_CMAKE_ARGS="-DProtobuf_ROOT=${build_dir}/cpp/protobuf_ep-install" +"${arrow_java_dir}/ci/scripts/jni_build.sh" \ + "${arrow_java_dir}" \ + "${install_dir}" \ + "${build_dir}" \ + "${dist_dir}" + +if [ "${ARROW_USE_CCACHE}" == "ON" ]; then + echo "=== ccache statistics after build ===" + ccache -sv 2>/dev/null || ccache -s +fi + +echo "=== Checking shared dependencies for libraries ===" +pushd "${dist_dir}" +archery linking check-dependencies \ + --allow CoreFoundation \ + --allow Security \ + --allow libSystem \ + --allow libarrow_cdata_jni \ + --allow libarrow_dataset_jni \ + --allow libarrow_orc_jni \ + --allow libc++ \ + --allow libcurl \ + --allow libgandiva_jni \ + --allow libncurses \ + --allow libobjc \ + --allow libz \ + "arrow_cdata_jni/${normalized_arch}/libarrow_cdata_jni.dylib" \ + "arrow_dataset_jni/${normalized_arch}/libarrow_dataset_jni.dylib" \ + "arrow_orc_jni/${normalized_arch}/libarrow_orc_jni.dylib" \ + "gandiva_jni/${normalized_arch}/libgandiva_jni.dylib" +popd diff --git a/ci/scripts/jni_manylinux_build.sh b/ci/scripts/jni_manylinux_build.sh new file mode 100755 index 0000000000..2551d67239 --- /dev/null +++ b/ci/scripts/jni_manylinux_build.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script is like java_jni_build.sh, but is meant for release artifacts +# and hardcodes assumptions about the environment it is being run in. + +set -exo pipefail + +arrow_java_dir="${1}" +arrow_dir="${2}" +build_dir="${3}" +normalized_arch="$(arch)" +case "${normalized_arch}" in +aarch64) + normalized_arch=aarch_64 + ;; +esac +# The directory where the final binaries will be stored when scripts finish +dist_dir="${4}" + +echo "=== Install Archery ===" +pip install -e "${arrow_dir}/dev/archery[all]" + +echo "=== Clear output directories and leftovers ===" +# Clear output directories and leftovers +rm -rf "${build_dir}" +rm -rf "${dist_dir}" + +echo "=== Building Arrow C++ libraries ===" +devtoolset_version="$(rpm -qa "devtoolset-*-gcc" --queryformat '%{VERSION}' | grep -o "^[0-9]*")" +devtoolset_include_cpp="/opt/rh/devtoolset-${devtoolset_version}/root/usr/include/c++/${devtoolset_version}" +: "${ARROW_ACERO:=ON}" +export ARROW_ACERO +: "${ARROW_BUILD_TESTS:=OFF}" +: "${ARROW_DATASET:=ON}" +export ARROW_DATASET +: "${ARROW_GANDIVA:=ON}" +export ARROW_GANDIVA +: "${ARROW_GCS:=ON}" +: "${ARROW_JEMALLOC:=OFF}" +: "${ARROW_MIMALLOC:=ON}" +: "${ARROW_RPATH_ORIGIN:=ON}" +: "${ARROW_ORC:=ON}" +export ARROW_ORC +: "${ARROW_PARQUET:=ON}" +: "${ARROW_S3:=ON}" +: "${ARROW_USE_CCACHE:=OFF}" +: "${CMAKE_BUILD_TYPE:=release}" +: "${CMAKE_UNITY_BUILD:=ON}" +: "${VCPKG_ROOT:=/opt/vcpkg}" +: "${VCPKG_FEATURE_FLAGS:=-manifests}" +: "${VCPKG_TARGET_TRIPLET:=${VCPKG_DEFAULT_TRIPLET:-x64-linux-static-${CMAKE_BUILD_TYPE}}}" +: "${GANDIVA_CXX_FLAGS:=-isystem;${devtoolset_include_cpp};-isystem;${devtoolset_include_cpp}/x86_64-redhat-linux;-lpthread}" + +if [ "${ARROW_USE_CCACHE}" == "ON" ]; then + echo "=== ccache statistics before build ===" + ccache -sv 2>/dev/null || ccache -s +fi + +export ARROW_TEST_DATA="${arrow_dir}/testing/data" +export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data" +export AWS_EC2_METADATA_DISABLED=TRUE + +mkdir -p "${build_dir}/cpp" +pushd "${build_dir}/cpp" + +cmake \ + -DARROW_ACERO="${ARROW_ACERO}" \ + -DARROW_BUILD_SHARED=OFF \ + -DARROW_BUILD_TESTS="${ARROW_BUILD_TESTS}" \ + -DARROW_CSV="${ARROW_DATASET}" \ + -DARROW_DATASET="${ARROW_DATASET}" \ + -DARROW_SUBSTRAIT="${ARROW_DATASET}" \ + -DARROW_DEPENDENCY_SOURCE="VCPKG" \ + -DARROW_DEPENDENCY_USE_SHARED=OFF \ + -DARROW_GANDIVA_PC_CXX_FLAGS="${GANDIVA_CXX_FLAGS}" \ + -DARROW_GANDIVA="${ARROW_GANDIVA}" \ + -DARROW_GCS="${ARROW_GCS}" \ + -DARROW_JEMALLOC="${ARROW_JEMALLOC}" \ + -DARROW_JSON="${ARROW_DATASET}" \ + -DARROW_MIMALLOC="${ARROW_MIMALLOC}" \ + -DARROW_ORC="${ARROW_ORC}" \ + -DARROW_PARQUET="${ARROW_PARQUET}" \ + -DARROW_RPATH_ORIGIN="${ARROW_RPATH_ORIGIN}" \ + -DARROW_S3="${ARROW_S3}" \ + -DARROW_USE_CCACHE="${ARROW_USE_CCACHE}" \ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" \ + -DCMAKE_INSTALL_PREFIX="${ARROW_HOME}" \ + -DCMAKE_UNITY_BUILD="${CMAKE_UNITY_BUILD}" \ + -DGTest_SOURCE=BUNDLED \ + -DORC_SOURCE=BUNDLED \ + -DORC_PROTOBUF_EXECUTABLE="${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/tools/protobuf/protoc" \ + -DPARQUET_BUILD_EXAMPLES=OFF \ + -DPARQUET_BUILD_EXECUTABLES=OFF \ + -DPARQUET_REQUIRE_ENCRYPTION=OFF \ + -DVCPKG_MANIFEST_MODE=OFF \ + -DVCPKG_TARGET_TRIPLET="${VCPKG_TARGET_TRIPLET}" \ + -GNinja \ + "${arrow_dir}/cpp" +ninja install + +if [ "${ARROW_BUILD_TESTS}" = "ON" ]; then + # MinIO is required + exclude_tests="arrow-s3fs-test" + case $(arch) in + aarch64) + # GCS testbench is crashed on aarch64: + # ImportError: ../grpc/_cython/cygrpc.cpython-38-aarch64-linux-gnu.so: + # undefined symbol: vtable for std::__cxx11::basic_ostringstream< + # char, std::char_traits, std::allocator > + exclude_tests="${exclude_tests}|arrow-gcsfs-test" + ;; + esac + # unstable + exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test" + exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test" + # external dependency + exclude_tests="${exclude_tests}|arrow-gcsfs-test" + # strptime + exclude_tests="${exclude_tests}|arrow-utility-test" + ctest \ + --exclude-regex "${exclude_tests}" \ + --label-regex unittest \ + --output-on-failure \ + --parallel "$(nproc)" \ + --timeout 300 +fi + +popd + +JAVA_JNI_CMAKE_ARGS="-DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" +JAVA_JNI_CMAKE_ARGS="${JAVA_JNI_CMAKE_ARGS} -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET}" +export JAVA_JNI_CMAKE_ARGS +"${arrow_java_dir}/ci/scripts/jni_build.sh" \ + "${arrow_java_dir}" \ + "${ARROW_HOME}" \ + "${build_dir}" \ + "${dist_dir}" + +if [ "${ARROW_USE_CCACHE}" == "ON" ]; then + echo "=== ccache statistics after build ===" + ccache -sv 2>/dev/null || ccache -s +fi + +echo "=== Checking shared dependencies for libraries ===" +pushd "${dist_dir}" +archery linking check-dependencies \ + --allow ld-linux-aarch64 \ + --allow ld-linux-x86-64 \ + --allow libc \ + --allow libdl \ + --allow libgcc_s \ + --allow libm \ + --allow libpthread \ + --allow librt \ + --allow libstdc++ \ + --allow libz \ + --allow linux-vdso \ + arrow_cdata_jni/"${normalized_arch}"/libarrow_cdata_jni.so \ + arrow_dataset_jni/"${normalized_arch}"/libarrow_dataset_jni.so \ + arrow_orc_jni/"${normalized_arch}"/libarrow_orc_jni.so \ + gandiva_jni/"${normalized_arch}"/libgandiva_jni.so +popd diff --git a/dataset/src/test/java/org/apache/arrow/dataset/TestAllTypes.java b/dataset/src/test/java/org/apache/arrow/dataset/TestAllTypes.java index eb73663191..0edc428254 100644 --- a/dataset/src/test/java/org/apache/arrow/dataset/TestAllTypes.java +++ b/dataset/src/test/java/org/apache/arrow/dataset/TestAllTypes.java @@ -95,8 +95,7 @@ private VectorSchemaRoot generateAllTypesVector(BufferAllocator allocator) { // DenseUnion List childFields = new ArrayList<>(); childFields.add( - new Field( - "int-child", new FieldType(false, new ArrowType.Int(32, true), null, null), null)); + new Field("int-child", new FieldType(true, new ArrowType.Int(32, true), null, null), null)); Field structField = new Field( "struct", new FieldType(true, ArrowType.Struct.INSTANCE, null, null), childFields); diff --git a/dataset/src/test/java/org/apache/arrow/dataset/TestDataset.java b/dataset/src/test/java/org/apache/arrow/dataset/TestDataset.java index f3ca04d77b..4b155137ed 100644 --- a/dataset/src/test/java/org/apache/arrow/dataset/TestDataset.java +++ b/dataset/src/test/java/org/apache/arrow/dataset/TestDataset.java @@ -123,8 +123,6 @@ protected void assertParquetFileEquals(String expectedURI, String actualURI) thr VectorSchemaRoot actualVsr = VectorSchemaRoot.create(actualFactory.inspect(), rootAllocator())) { - // fast-fail by comparing metadata - assertEquals(expectedBatches.toString(), actualBatches.toString()); // compare ArrowRecordBatches assertEquals(expectedBatches.size(), actualBatches.size()); VectorLoader expectLoader = new VectorLoader(expectVsr); diff --git a/docker-compose.yml b/docker-compose.yml index ae378865b3..44d58c96a0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,6 +31,8 @@ volumes: services: ubuntu: + # Build and test arrow-java on Ubuntu. + # # Usage: # docker compose build ubuntu # docker compose run ubuntu @@ -47,6 +49,10 @@ services: /arrow-java/ci/scripts/java_test.sh /arrow-java /build" conda-jni-cdata: + # Builds and tests just the C Data Interface JNI library and JARs. + # (No dependencies on arrow-cpp.) + # This build isn't meant for distribution. It's for testing only. + # # Usage: # docker compose build conda-jni-cdata # docker compose run conda-jni-cdata @@ -72,6 +78,32 @@ services: ARROW_JAVA_CDATA: "ON" command: /bin/bash -c " - /arrow-java/ci/scripts/java_jni_build.sh /arrow-java /build/jni /build /jni && + /arrow-java/ci/scripts/jni_build.sh /arrow-java /build/jni /build /jni && /arrow-java/ci/scripts/java_build.sh /arrow-java /build /jni && /arrow-java/ci/scripts/java_test.sh /arrow-java /build /jni" + + vcpkg-jni: + # Builds all the JNI libraries, but not the JARs. + # (Requires arrow-cpp.) + # The artifacts from this build are meant to be used for packaging. + # + # Usage: + # docker compose build vcpkg-jni + # docker compose run vcpkg-jni + image: ${REPO}:${ARCH}-vcpkg-jni + build: + context: . + dockerfile: ci/docker/vcpkg-jni.dockerfile + cache_from: + - ${REPO}:${ARCH}-vcpkg-jni + args: + base: ${ARROW_REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2014-vcpkg-${VCPKG} + volumes: + - .:/arrow-java:delegated + - ${ARROW_REPO_ROOT}:/arrow:delegated + - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated + environment: + ARROW_JAVA_CDATA: "ON" + command: + ["git config --global --add safe.directory /arrow-java && \ + /arrow-java/ci/scripts/jni_manylinux_build.sh /arrow-java /arrow /build /arrow-java/dist"] From 2d3f9c30ab4ced178036da4ae6e749d919421158 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Jan 2025 01:56:37 -0500 Subject: [PATCH 008/408] MINOR: [Java] Bump commons-io:commons-io from 2.17.0 to 2.18.0 (#30) Bumps commons-io:commons-io from 2.17.0 to 2.18.0. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dataset/pom.xml | 2 +- flight/flight-sql-jdbc-core/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dataset/pom.xml b/dataset/pom.xml index 21f67f1a69..bf4daf9a8b 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -156,7 +156,7 @@ under the License. commons-io commons-io - 2.17.0 + 2.18.0 test diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 8ad7e76801..204e17efe8 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -90,7 +90,7 @@ under the License. commons-io commons-io - 2.17.0 + 2.18.0 test From 4f9433d800b89864a6aefb8964e4afe29aba0e23 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Jan 2025 03:04:27 -0500 Subject: [PATCH 009/408] MINOR: Bump com.fasterxml.jackson:jackson-bom from 2.18.1 to 2.18.2 (#440) Bumps [com.fasterxml.jackson:jackson-bom](https://github.com/FasterXML/jackson-bom) from 2.18.1 to 2.18.2. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 82ac714a9a..0cab5c77f6 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,7 @@ under the License. 4.1.115.Final 1.65.0 3.25.5 - 2.18.1 + 2.18.2 3.4.1 24.3.25 1.12.0 From d3321b84e0c3daa1bd05d07c6cf06efb3f3a27a7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Jan 2025 05:10:14 -0500 Subject: [PATCH 010/408] MINOR: Bump com.github.luben:zstd-jni from 1.5.6-7 to 1.5.6-9 (#474) Bumps [com.github.luben:zstd-jni](https://github.com/luben/zstd-jni) from 1.5.6-7 to 1.5.6-9. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- compression/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compression/pom.xml b/compression/pom.xml index 8cc4909034..3c339d557a 100644 --- a/compression/pom.xml +++ b/compression/pom.xml @@ -55,7 +55,7 @@ under the License. com.github.luben zstd-jni - 1.5.6-7 + 1.5.6-9 From bd2604317b996a0e36b9e704bbddcd712ea75f64 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Jan 2025 05:24:53 -0500 Subject: [PATCH 011/408] MINOR: [Java] Bump org.apache.orc:orc-core from 1.9.5 to 2.0.3 (#33) Bumps org.apache.orc:orc-core from 1.9.5 to 2.0.3. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- adapter/orc/pom.xml | 2 +- dataset/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/adapter/orc/pom.xml b/adapter/orc/pom.xml index 6061feb4ad..6bde7119fa 100644 --- a/adapter/orc/pom.xml +++ b/adapter/orc/pom.xml @@ -61,7 +61,7 @@ under the License. org.apache.orc orc-core - 1.9.5 + 2.0.3 test diff --git a/dataset/pom.xml b/dataset/pom.xml index bf4daf9a8b..1618b2b3b9 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -130,7 +130,7 @@ under the License. org.apache.orc orc-core - 1.9.5 + 2.0.3 test From 8a08ecf97af3b5c112ab8a6f2357fd10a8f62101 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Jan 2025 20:16:58 -0500 Subject: [PATCH 012/408] MINOR: Bump org.cyclonedx:cyclonedx-maven-plugin from 2.9.0 to 2.9.1 (#438) Bumps [org.cyclonedx:cyclonedx-maven-plugin](https://github.com/CycloneDX/cyclonedx-maven-plugin) from 2.9.0 to 2.9.1. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0cab5c77f6..9bd1e6f1f1 100644 --- a/pom.xml +++ b/pom.xml @@ -516,7 +516,7 @@ under the License. org.cyclonedx cyclonedx-maven-plugin - 2.9.0 + 2.9.1 org.apache.drill.tools From 5b6ee220175bebcd4585314cc7af88017b74451c Mon Sep 17 00:00:00 2001 From: David Li Date: Fri, 3 Jan 2025 20:26:06 -0500 Subject: [PATCH 013/408] GH-447: Port fix from apache/arrow that was missed (#475) https://github.com/apache/arrow/commit/ded148c2ef5dd441a3b6ab9496d0ed0aeb940f71 was merged after the migration. Fixes #447. --- .../java/org/apache/arrow/dataset/TestAllTypes.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/dataset/src/test/java/org/apache/arrow/dataset/TestAllTypes.java b/dataset/src/test/java/org/apache/arrow/dataset/TestAllTypes.java index 0edc428254..79800f3570 100644 --- a/dataset/src/test/java/org/apache/arrow/dataset/TestAllTypes.java +++ b/dataset/src/test/java/org/apache/arrow/dataset/TestAllTypes.java @@ -95,10 +95,9 @@ private VectorSchemaRoot generateAllTypesVector(BufferAllocator allocator) { // DenseUnion List childFields = new ArrayList<>(); childFields.add( - new Field("int-child", new FieldType(true, new ArrowType.Int(32, true), null, null), null)); + new Field("int-child", FieldType.notNullable(new ArrowType.Int(32, true)), null)); Field structField = - new Field( - "struct", new FieldType(true, ArrowType.Struct.INSTANCE, null, null), childFields); + new Field("struct", FieldType.nullable(ArrowType.Struct.INSTANCE), childFields); Field[] fields = new Field[] { Field.nullablePrimitive("null", ArrowType.Null.INSTANCE), @@ -238,7 +237,11 @@ private VectorSchemaRoot generateAllTypesVector(BufferAllocator allocator) { largeListWriter.integer().writeInt(1); largeListWriter.endList(); - ((StructVector) root.getVector("struct")).getChild("int-child", IntVector.class).set(1, 1); + IntVector intChildVector = + ((StructVector) root.getVector("struct")).getChild("int-child", IntVector.class); + // Non-nullable vector, make sure to fill all slots + intChildVector.set(0, 0); + intChildVector.set(1, 1); return root; } From be627bf9748db3954fb8fc0be5a7077ac32b0bf1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jan 2025 16:19:24 +0900 Subject: [PATCH 014/408] MINOR: [CI] Bump actions/setup-python from 4 to 5 (#482) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5.
Release notes

Sourced from actions/setup-python's releases.

v5.0.0

What's Changed

In scope of this release, we update node version runtime from node16 to node20 (actions/setup-python#772). Besides, we update dependencies to the latest versions.

Full Changelog: https://github.com/actions/setup-python/compare/v4.8.0...v5.0.0

v4.8.0

What's Changed

In scope of this release we added support for GraalPy (actions/setup-python#694). You can use this snippet to set up GraalPy:

steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
  with:
    python-version: 'graalpy-22.3'
- run: python my_script.py

Besides, the release contains such changes as:

New Contributors

Full Changelog: https://github.com/actions/setup-python/compare/v4...v4.8.0

v4.7.1

What's Changed

Full Changelog: https://github.com/actions/setup-python/compare/v4...v4.7.1

v4.7.0

In scope of this release, the support for reading python version from pyproject.toml was added (actions/setup-python#669).

      - name: Setup Python
        uses: actions/setup-python@v4
</tr></table>

... (truncated)

Commits
  • 0b93645 Enhance workflows: Add macOS 13 support, upgrade publish-action, and update d...
  • 9c76e71 Bump pillow from 7.2 to 10.2.0 in /tests/data (#956)
  • f4c5a11 Revise isGhes logic (#963)
  • 19dfb7b Bump default versions to latest (#905)
  • e9675cc Merge pull request #943 from actions/Jcambass-patch-1
  • 3226af6 Upgrade IA publish
  • 70dcb22 Merge pull request #941 from actions/Jcambass-patch-1
  • 65b48c7 Create publish-immutable-actions.yml
  • 29a37be initial commit (#938)
  • f677139 Bump pyinstaller from 3.6 to 5.13.1 in /tests/data (#923)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/setup-python&package-manager=github_actions&previous-version=4&new-version=5)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/test_jni.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_jni.yml b/.github/workflows/test_jni.yml index 31eb9b8743..15b535e17f 100644 --- a/.github/workflows/test_jni.yml +++ b/.github/workflows/test_jni.yml @@ -111,7 +111,7 @@ jobs: path: arrow submodules: recursive - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: cache: 'pip' python-version: 3.12 From 316757362eda78e3f73c204313a90504b8b14442 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Jan 2025 19:58:48 -0500 Subject: [PATCH 015/408] MINOR: Bump org.mockito:mockito-core from 5.14.2 to 5.15.2 (#480) Bumps [org.mockito:mockito-core](https://github.com/mockito/mockito) from 5.14.2 to 5.15.2. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9bd1e6f1f1..6afdb4c399 100644 --- a/pom.xml +++ b/pom.xml @@ -108,7 +108,7 @@ under the License. 10.20.1 true 2.31.0 - 5.14.2 + 5.15.2 3.48.2 1.5.12 none From e2ea8fab71e15de3e4ce15a55dfac99e0d76d5f1 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 8 Jan 2025 11:14:31 +0900 Subject: [PATCH 016/408] GH-16: Add integration test CI (#489) Fixes GH-16. --- .github/workflows/test.yml | 57 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6f8c954881..d985f0895a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -145,3 +145,60 @@ jobs: env: DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} run: ci/scripts/java_test.sh $(pwd) $(pwd)/build + + integration: + name: AMD64 integration + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout Arrow + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + repository: apache/arrow + submodules: recursive + - name: Checkout Arrow Rust + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: apache/arrow-rs + path: rust + - name: Checkout Arrow nanoarrow + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: apache/arrow-nanoarrow + path: nanoarrow + - name: Checkout Arrow Go + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: apache/arrow-go + path: go + - name: Checkout Arrow Java + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: java + - name: Free up disk space + run: | + ci/scripts/util_free_space.sh + - name: Cache Docker Volumes + uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + with: + path: .docker + key: integration-conda-${{ hashFiles('cpp/**') }} + restore-keys: conda- + - name: Setup Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: 3.12 + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + run: | + source ci/scripts/util_enable_core_dumps.sh + archery docker run \ + -e ARCHERY_DEFAULT_BRANCH=main \ + -e ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS=java \ + -e ARCHERY_INTEGRATION_WITH_GO=1 \ + -e ARCHERY_INTEGRATION_WITH_JAVA=1 \ + -e ARCHERY_INTEGRATION_WITH_NANOARROW=1 \ + -e ARCHERY_INTEGRATION_WITH_RUST=1 \ + conda-integration From 33717d4dec2acc3dac9d708e9f00d80edc5f4a0f Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 8 Jan 2025 11:14:45 +0900 Subject: [PATCH 017/408] GH-490: Use apache/arrow-java not apache/arrow in pom.xml (#491) Fixes #490. --- bom/pom.xml | 8 ++++---- pom.xml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index ccb70d5fb3..61441048ff 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -66,15 +66,15 @@ under the License. - scm:git:https://github.com/apache/arrow.git - scm:git:https://github.com/apache/arrow.git + scm:git:https://github.com/apache/arrow-java.git + scm:git:https://github.com/apache/arrow-java.git main - https://github.com/apache/arrow/tree/${project.scm.tag} + https://github.com/apache/arrow-java/tree/${project.scm.tag} GitHub - https://github.com/apache/arrow/issues + https://github.com/apache/arrow-java/issues diff --git a/pom.xml b/pom.xml index 6afdb4c399..961b1e5a69 100644 --- a/pom.xml +++ b/pom.xml @@ -79,15 +79,15 @@ under the License. - scm:git:https://github.com/apache/arrow.git - scm:git:https://github.com/apache/arrow.git + scm:git:https://github.com/apache/arrow-java.git + scm:git:https://github.com/apache/arrow-java.git main - https://github.com/apache/arrow/tree/${project.scm.tag} + https://github.com/apache/arrow-java/tree/${project.scm.tag} GitHub - https://github.com/apache/arrow/issues + https://github.com/apache/arrow-java/issues From f38412692980eb6d8aea98e2075cec49c6d42e6d Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 8 Jan 2025 13:08:11 +0900 Subject: [PATCH 018/408] GH-492: [CI] Remove `java_` prefix from `ci/scripts/*.sh` (#493) Fixes GH-492. --- .github/workflows/test.yml | 8 ++++---- ci/scripts/{java_build.sh => build.sh} | 0 ci/scripts/{java_test.sh => test.sh} | 0 docker-compose.yml | 8 ++++---- 4 files changed, 8 insertions(+), 8 deletions(-) rename ci/scripts/{java_build.sh => build.sh} (100%) rename ci/scripts/{java_test.sh => test.sh} (100%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d985f0895a..a61578572e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -108,12 +108,12 @@ jobs: shell: bash env: DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} - run: ci/scripts/java_build.sh $(pwd) $(pwd)/build + run: ci/scripts/build.sh $(pwd) $(pwd)/build - name: Test shell: bash env: DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} - run: ci/scripts/java_test.sh $(pwd) $(pwd)/build + run: ci/scripts/test.sh $(pwd) $(pwd)/build windows: name: AMD64 Windows Server 2022 Java JDK ${{ matrix.jdk }} @@ -139,12 +139,12 @@ jobs: shell: bash env: DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} - run: ci/scripts/java_build.sh $(pwd) $(pwd)/build + run: ci/scripts/build.sh $(pwd) $(pwd)/build - name: Test shell: bash env: DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} - run: ci/scripts/java_test.sh $(pwd) $(pwd)/build + run: ci/scripts/test.sh $(pwd) $(pwd)/build integration: name: AMD64 integration diff --git a/ci/scripts/java_build.sh b/ci/scripts/build.sh similarity index 100% rename from ci/scripts/java_build.sh rename to ci/scripts/build.sh diff --git a/ci/scripts/java_test.sh b/ci/scripts/test.sh similarity index 100% rename from ci/scripts/java_test.sh rename to ci/scripts/test.sh diff --git a/docker-compose.yml b/docker-compose.yml index 44d58c96a0..c1d270287d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -45,8 +45,8 @@ services: - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated command: /bin/bash -c " - /arrow-java/ci/scripts/java_build.sh /arrow-java /build && - /arrow-java/ci/scripts/java_test.sh /arrow-java /build" + /arrow-java/ci/scripts/build.sh /arrow-java /build && + /arrow-java/ci/scripts/test.sh /arrow-java /build" conda-jni-cdata: # Builds and tests just the C Data Interface JNI library and JARs. @@ -79,8 +79,8 @@ services: command: /bin/bash -c " /arrow-java/ci/scripts/jni_build.sh /arrow-java /build/jni /build /jni && - /arrow-java/ci/scripts/java_build.sh /arrow-java /build /jni && - /arrow-java/ci/scripts/java_test.sh /arrow-java /build /jni" + /arrow-java/ci/scripts/build.sh /arrow-java /build /jni && + /arrow-java/ci/scripts/test.sh /arrow-java /build /jni" vcpkg-jni: # Builds all the JNI libraries, but not the JARs. From 2f6e2455ceb0cb02fabfc6b77d2c46b2f0a7732a Mon Sep 17 00:00:00 2001 From: Amit Mittal Date: Wed, 8 Jan 2025 14:01:30 +0530 Subject: [PATCH 019/408] GH-483: Use ArrowReader instead of ArrowStreamReader in FlightSqlClient.executeIngest method (#485) Please see #483 --- .../java/org/apache/arrow/flight/sql/FlightSqlClient.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlClient.java b/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlClient.java index 9a6ffdfdca..623f9311e8 100644 --- a/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlClient.java +++ b/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlClient.java @@ -92,7 +92,7 @@ import org.apache.arrow.util.AutoCloseables; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.ipc.ArrowStreamReader; +import org.apache.arrow.vector.ipc.ArrowReader; import org.apache.arrow.vector.ipc.ReadChannel; import org.apache.arrow.vector.ipc.message.MessageSerializer; import org.apache.arrow.vector.types.pojo.Schema; @@ -236,7 +236,7 @@ public long executeIngest( * @return the number of rows affected. */ public long executeIngest( - final ArrowStreamReader dataReader, + final ArrowReader dataReader, final ExecuteIngestOptions ingestOptions, final CallOption... options) { return executeIngest(dataReader, ingestOptions, /*transaction*/ null, options); @@ -270,7 +270,7 @@ public long executeIngest( * @return the number of rows affected. */ public long executeIngest( - final ArrowStreamReader dataReader, + final ArrowReader dataReader, final ExecuteIngestOptions ingestOptions, Transaction transaction, final CallOption... options) { From 3440633b2f9a8293289c2b8f40929fe299ddc03a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Jan 2025 10:46:16 -0800 Subject: [PATCH 020/408] MINOR: Bump logback.version from 1.5.12 to 1.5.16 (#481) Bumps `logback.version` from 1.5.12 to 1.5.16. Updates `ch.qos.logback:logback-classic` from 1.5.12 to 1.5.16
Commits
  • 74c9ebd prepare release 1.5.16
  • 9308a58 javadocs structure changed
  • 8935470 adapt test to SLF4J version 2.0.16
  • cb60369 addded StubEventEvaluator as default class for evaluator element so as to dir...
  • 1da2f17 bump jxr version
  • 5bde644 bump slf4j version to 2.0.16
  • aa2ebae remove stax related code
  • 80db86b fix issues/860
  • a8a2303 start work on 1.5.16-SNAPSHOT
  • bf14c2c minor javadoc update
  • Additional commits viewable in compare view

Updates `ch.qos.logback:logback-core` from 1.5.12 to 1.5.16
Commits
  • 74c9ebd prepare release 1.5.16
  • 9308a58 javadocs structure changed
  • 8935470 adapt test to SLF4J version 2.0.16
  • cb60369 addded StubEventEvaluator as default class for evaluator element so as to dir...
  • 1da2f17 bump jxr version
  • 5bde644 bump slf4j version to 2.0.16
  • aa2ebae remove stax related code
  • 80db86b fix issues/860
  • a8a2303 start work on 1.5.16-SNAPSHOT
  • bf14c2c minor javadoc update
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 961b1e5a69..6aeacdc348 100644 --- a/pom.xml +++ b/pom.xml @@ -110,7 +110,7 @@ under the License. 2.31.0 5.15.2 3.48.2 - 1.5.12 + 1.5.16 none -Xdoclint:none From adf51bbd463dc9d60814b56b695f414fcb695283 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 9 Jan 2025 20:52:26 +0900 Subject: [PATCH 021/408] GH-497: [Dev] Use apache-arrow-java.tar.gz for RAT (#498) Fixes GH-497. --- .gitignore | 1 + .pre-commit-config.yaml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index ca0ac32463..8171952dad 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ .settings/ /*-build/ /.mvn/.develocity/ +/apache-arrow-java.tar.gz /build/ /dev/release/apache-rat-0.16.1.jar /dev/release/filtered_rat.txt diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 64f9e1cda9..ff0d85448f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,9 +33,9 @@ repos: entry: | bash -c " \ git archive HEAD \ - --prefix=apache-arrow-go/ \ - --output=apache-arrow-go.tar.gz && \ - dev/release/run_rat.sh apache-arrow-go.tar.gz" + --prefix=apache-arrow-java/ \ + --output=apache-arrow-java.tar.gz && \ + dev/release/run_rat.sh apache-arrow-java.tar.gz" always_run: true pass_filenames: false - repo: https://github.com/koalaman/shellcheck-precommit From 6d2116d23a005d05ad7437858cb004bdd9cce189 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 20:52:49 -0500 Subject: [PATCH 022/408] MINOR: Bump parquet.version from 1.14.4 to 1.15.0 (#478) Bumps `parquet.version` from 1.14.4 to 1.15.0. Updates `org.apache.parquet:parquet-avro` from 1.14.4 to 1.15.0 Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dataset/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataset/pom.xml b/dataset/pom.xml index 1618b2b3b9..7d074f0508 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -32,7 +32,7 @@ under the License. ../../../cpp/release-build/ - 1.14.4 + 1.15.0 1.12.0 From 4a535af9e92b11fc618f15e3dd9cdfad8c554ca2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 20:54:09 -0500 Subject: [PATCH 023/408] MINOR: Bump com.google.guava:guava-bom from 33.3.1-jre to 33.4.0-jre (#479) Bumps [com.google.guava:guava-bom](https://github.com/google/guava) from 33.3.1-jre to 33.4.0-jre. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6aeacdc348..c691b0b859 100644 --- a/pom.xml +++ b/pom.xml @@ -95,7 +95,7 @@ under the License. 1.9.0 5.11.3 2.0.16 - 33.3.1-jre + 33.4.0-jre 4.1.115.Final 1.65.0 3.25.5 From de80cef9e7d81748cffc57f2438762a0c1d30369 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 10 Jan 2025 14:47:09 +0900 Subject: [PATCH 024/408] MINOR: Bump version to 18.2.0 (#502) --- adapter/avro/pom.xml | 2 +- adapter/jdbc/pom.xml | 2 +- adapter/orc/pom.xml | 2 +- algorithm/pom.xml | 2 +- bom/pom.xml | 4 ++-- c/pom.xml | 2 +- compression/pom.xml | 2 +- dataset/pom.xml | 2 +- flight/flight-core/pom.xml | 2 +- flight/flight-integration-tests/pom.xml | 2 +- flight/flight-sql-jdbc-core/pom.xml | 2 +- flight/flight-sql-jdbc-driver/pom.xml | 2 +- flight/flight-sql/pom.xml | 2 +- flight/pom.xml | 2 +- format/pom.xml | 2 +- gandiva/pom.xml | 2 +- memory/memory-core/pom.xml | 2 +- memory/memory-netty-buffer-patch/pom.xml | 2 +- memory/memory-netty/pom.xml | 2 +- memory/memory-unsafe/pom.xml | 2 +- memory/pom.xml | 2 +- performance/pom.xml | 2 +- pom.xml | 4 ++-- tools/pom.xml | 2 +- vector/pom.xml | 2 +- 25 files changed, 27 insertions(+), 27 deletions(-) diff --git a/adapter/avro/pom.xml b/adapter/avro/pom.xml index 827d19f2a2..2eb77d7539 100644 --- a/adapter/avro/pom.xml +++ b/adapter/avro/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 18.2.0 ../../pom.xml diff --git a/adapter/jdbc/pom.xml b/adapter/jdbc/pom.xml index 2f621d7a05..57a728124b 100644 --- a/adapter/jdbc/pom.xml +++ b/adapter/jdbc/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 18.2.0 ../../pom.xml diff --git a/adapter/orc/pom.xml b/adapter/orc/pom.xml index 6bde7119fa..6b15a34a77 100644 --- a/adapter/orc/pom.xml +++ b/adapter/orc/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 18.2.0 ../../pom.xml diff --git a/algorithm/pom.xml b/algorithm/pom.xml index 898c2605b6..3bcbfa5062 100644 --- a/algorithm/pom.xml +++ b/algorithm/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 18.2.0 arrow-algorithm Arrow Algorithms diff --git a/bom/pom.xml b/bom/pom.xml index 61441048ff..192b494eb2 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -29,7 +29,7 @@ under the License. org.apache.arrow arrow-bom - 19.0.0-SNAPSHOT + 18.2.0 pom Arrow Bill of Materials @@ -68,7 +68,7 @@ under the License. scm:git:https://github.com/apache/arrow-java.git scm:git:https://github.com/apache/arrow-java.git - main + v18.2.0 https://github.com/apache/arrow-java/tree/${project.scm.tag} diff --git a/c/pom.xml b/c/pom.xml index c90b6dc0ef..48e9fafdf5 100644 --- a/c/pom.xml +++ b/c/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 18.2.0 arrow-c-data diff --git a/compression/pom.xml b/compression/pom.xml index 3c339d557a..6ef36fe8ab 100644 --- a/compression/pom.xml +++ b/compression/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 18.2.0 arrow-compression Arrow Compression diff --git a/dataset/pom.xml b/dataset/pom.xml index 7d074f0508..52ad0c056e 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 18.2.0 arrow-dataset diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml index 9dac97dd9c..340004041e 100644 --- a/flight/flight-core/pom.xml +++ b/flight/flight-core/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 19.0.0-SNAPSHOT + 18.2.0 flight-core diff --git a/flight/flight-integration-tests/pom.xml b/flight/flight-integration-tests/pom.xml index e43bcd0571..704a26b549 100644 --- a/flight/flight-integration-tests/pom.xml +++ b/flight/flight-integration-tests/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 19.0.0-SNAPSHOT + 18.2.0 flight-integration-tests diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 204e17efe8..ab3560459b 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 19.0.0-SNAPSHOT + 18.2.0 flight-sql-jdbc-core diff --git a/flight/flight-sql-jdbc-driver/pom.xml b/flight/flight-sql-jdbc-driver/pom.xml index ae8c543fbf..f8d829a142 100644 --- a/flight/flight-sql-jdbc-driver/pom.xml +++ b/flight/flight-sql-jdbc-driver/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 19.0.0-SNAPSHOT + 18.2.0 flight-sql-jdbc-driver diff --git a/flight/flight-sql/pom.xml b/flight/flight-sql/pom.xml index 9cbc8430fe..668fcb148b 100644 --- a/flight/flight-sql/pom.xml +++ b/flight/flight-sql/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 19.0.0-SNAPSHOT + 18.2.0 flight-sql diff --git a/flight/pom.xml b/flight/pom.xml index 2fc3e89ef8..35dfe90468 100644 --- a/flight/pom.xml +++ b/flight/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 18.2.0 arrow-flight diff --git a/format/pom.xml b/format/pom.xml index d3578b63d2..292f23bcd2 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 18.2.0 arrow-format diff --git a/gandiva/pom.xml b/gandiva/pom.xml index 5367bfdedf..384dcb8019 100644 --- a/gandiva/pom.xml +++ b/gandiva/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 18.2.0 org.apache.arrow.gandiva diff --git a/memory/memory-core/pom.xml b/memory/memory-core/pom.xml index 72ee69d60a..9432dd2461 100644 --- a/memory/memory-core/pom.xml +++ b/memory/memory-core/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 19.0.0-SNAPSHOT + 18.2.0 arrow-memory-core diff --git a/memory/memory-netty-buffer-patch/pom.xml b/memory/memory-netty-buffer-patch/pom.xml index 07dc7d2403..696fadeba3 100644 --- a/memory/memory-netty-buffer-patch/pom.xml +++ b/memory/memory-netty-buffer-patch/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 19.0.0-SNAPSHOT + 18.2.0 arrow-memory-netty-buffer-patch diff --git a/memory/memory-netty/pom.xml b/memory/memory-netty/pom.xml index 6d660da117..b81e13cd90 100644 --- a/memory/memory-netty/pom.xml +++ b/memory/memory-netty/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 19.0.0-SNAPSHOT + 18.2.0 arrow-memory-netty diff --git a/memory/memory-unsafe/pom.xml b/memory/memory-unsafe/pom.xml index 92dc0c9fe5..a476da71b1 100644 --- a/memory/memory-unsafe/pom.xml +++ b/memory/memory-unsafe/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 19.0.0-SNAPSHOT + 18.2.0 arrow-memory-unsafe diff --git a/memory/pom.xml b/memory/pom.xml index bc34c26050..af6babe841 100644 --- a/memory/pom.xml +++ b/memory/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 18.2.0 arrow-memory pom diff --git a/performance/pom.xml b/performance/pom.xml index 3f18188e3a..507222706d 100644 --- a/performance/pom.xml +++ b/performance/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 18.2.0 arrow-performance jar diff --git a/pom.xml b/pom.xml index c691b0b859..0fd76bd2ed 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 18.2.0 pom Apache Arrow Java Root POM @@ -81,7 +81,7 @@ under the License. scm:git:https://github.com/apache/arrow-java.git scm:git:https://github.com/apache/arrow-java.git - main + v18.2.0 https://github.com/apache/arrow-java/tree/${project.scm.tag} diff --git a/tools/pom.xml b/tools/pom.xml index f06ded294a..a64b84857c 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 18.2.0 arrow-tools Arrow Tools diff --git a/vector/pom.xml b/vector/pom.xml index 7cd25cd43e..6ad28bbe9c 100644 --- a/vector/pom.xml +++ b/vector/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 18.2.0 arrow-vector Arrow Vectors From 14dcd50579b1b4c42092b87ba1d43f86007eb79f Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 13 Jan 2025 18:51:15 +0900 Subject: [PATCH 025/408] GH-20: Add release scripts (#496) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes GH-20. This is based on https://github.com/apache/arrow-go/pull/122 . Workflow: Cut a RC: 1. Bump version by `dev/release/bump_version.sh 19.0.0` 2. Run `dev/release/release_rc.sh` 1. `dev/release/release_rc.sh` pushes `v${version}-rc${rc}` tag 2. `.github/workflows/rc.yml` creates `apache-arrow-java-${version}.tar.gz{,.sha256,.sha512}` 3. `.github/workflows/rc.yml` uploads `apache-arrow-java-${version}.tar.gz{,.sha256,.sha512}` to GitHub Releases 4. `dev/release/release_rc.sh` downloads `apache-arrow-java-${version}.tar.gz` from GitHub Releases 5. `dev/release/release_rc.sh` signs `apache-arrow-java-${version}.tar.gz` as `apache-arrow-go-${version}.tar.gz.asc` 6. `dev/release/release_rc.sh` uploads `apache-arrow-java-${version}.tar.gz.asc` to GitHub Releases 3. Start a vote (GitHub Actions instead of https://dist.apache.org/repos/dist/dev/arrow/ is used like ADBC.) Verify a RC: 1. Run `dev/release/verify_rc.sh` Release an approved RC: 1. Run `dev/release/release.sh` 1. `dev/release/release.sh` pushes `v${version}` tag that refers that same commit ID as `v${version}-rc${rc}` 2. `.github/workflows/release.yml` create a GitHub Releases for `v${version}` tag by copying the RC GitHub Releases 3. `dev/release/release.sh` downloads `apache-arrow-java-${version}.tar.gz{,.asc,.sha256,.sha512}` from GitHub Releases 4. `dev/release/release.sh` uploads `apache-arrow-java-${version}.tar.gz{,.asc,.sha256,.sha512}` to https://dist.apache.org/repos/dist/release/arrow 5. `dev/release/release.sh` removes old releases from https://dist.apache.org/repos/dist/release/arrow 2. Add this release to ASF's report database: https://reporter.apache.org/addrelease.html?arrow Follow-up tasks: * Add support for building binary packages (.jar) * Add support for releasing binary packages (.jar) * Add support for running JNI test in the verification script * Add support for running integration test in the verification script * Add support for verifying binary packages (.jar) in the verification script --------- Co-authored-by: Raúl Cumplido --- .github/workflows/rc.yml | 141 +++++++++++++++++++++++++++ .github/workflows/release.yml | 54 +++++++++++ dev/release/README.md | 174 ++++++++++++++++++++++++++++++++++ dev/release/bump_version.sh | 64 +++++++++++++ dev/release/release.sh | 89 +++++++++++++++++ dev/release/release_rc.sh | 157 ++++++++++++++++++++++++++++++ dev/release/verify_rc.sh | 158 ++++++++++++++++++++++++++++++ 7 files changed, 837 insertions(+) create mode 100644 .github/workflows/rc.yml create mode 100644 .github/workflows/release.yml create mode 100644 dev/release/README.md create mode 100755 dev/release/bump_version.sh create mode 100755 dev/release/release.sh create mode 100755 dev/release/release_rc.sh create mode 100755 dev/release/verify_rc.sh diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml new file mode 100644 index 0000000000..5a82d686e8 --- /dev/null +++ b/.github/workflows/rc.yml @@ -0,0 +1,141 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: RC +on: + push: + branches: + - "**" + - "!dependabot/**" + tags: + - "*-rc*" + pull_request: + schedule: + - cron: "0 0 * * *" +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true +permissions: + contents: read +jobs: + archive: + name: Archive + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: recursive + - name: Prepare for tag + if: github.ref_type == 'tag' + run: | + version=${GITHUB_REF_NAME%-rc*} + version=${version#v} + rc=${GITHUB_REF_NAME#*-rc} + echo "VERSION=${version}" >> ${GITHUB_ENV} + echo "RC=${rc}" >> ${GITHUB_ENV} + - name: Prepare for branch + if: github.ref_type == 'branch' + run: | + version=$(grep -o '^ .*' "pom.xml" | + sed \ + -e 's,^ ,,' \ + -e 's,$,,') + rc=$(date +%Y%m%d) + echo "VERSION=${version}" >> ${GITHUB_ENV} + echo "RC=${rc}" >> ${GITHUB_ENV} + - name: Archive + run: | + id="apache-arrow-java-${VERSION}" + tar_gz="${id}.tar.gz" + echo "TAR_GZ=${tar_gz}" >> ${GITHUB_ENV} + git archive HEAD --prefix "${id}/" --output "${tar_gz}" + sha256sum "${tar_gz}" > "${tar_gz}.sha256" + sha512sum "${tar_gz}" > "${tar_gz}.sha512" + - name: Audit + run: | + dev/release/run_rat.sh "${TAR_GZ}" + - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 + with: + name: archive + path: | + apache-arrow-java-* + verify: + name: Verify + needs: + - archive + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - macos-latest + - ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: recursive + - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: archive + - name: Verify + run: | + tar_gz=$(echo apache-arrow-java-*.tar.gz) + version=${tar_gz#apache-arrow-java-} + version=${version%.tar.gz} + # rc isn't used with VERIFY_DOWNLOAD=0 + if [ "${GITHUB_REF_TYPE}" = "tag" ]; then + rc="${GITHUB_REF_NAME#*-rc}" + else + rc=$(date +%Y%m%d) + fi + VERIFY_DEFAULT=0 \ + VERIFY_SOURCE=1 \ + dev/release/verify_rc.sh "${version}" "${rc}" + upload: + name: Upload + if: github.ref_type == 'tag' + needs: + - verify + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: recursive + - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: archive + - name: Upload + run: | + # GH-499: How to create release notes? + version=${GITHUB_REF_NAME%-rc*} + version=${version#v} + rc=${GITHUB_REF_NAME#*-rc} + gh release create ${GITHUB_REF_NAME} \ + --generate-notes \ + --prerelease \ + --title "Apache Arrow Java ${version} RC${rc}" \ + --verify-tag \ + apache-arrow-java-*.tar.gz \ + apache-arrow-java-*.tar.gz.sha* + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000000..4982c6b343 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Release +on: + push: + tags: + - "*" + - "!*-rc*" +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true +permissions: + contents: write +env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} +jobs: + publish: + name: Publish + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Download RC contents + run: | + set -x + latest_rc_tag=$(gh release list --json tagName --jq '.[].tagName' | \ + grep -F "${GITHUB_REF_NAME}-rc" | \ + head -n1) + gh release download ${latest_rc_tag} --dir dists + - name: Create GitHub Release + run: | + # GH-499: How to create release notes? + version=${GITHUB_REF_NAME#v} + gh release create ${GITHUB_REF_NAME} \ + --generate-notes \ + --title "Apache Arrow Java ${version}" \ + --verify-tag \ + dists/* diff --git a/dev/release/README.md b/dev/release/README.md new file mode 100644 index 0000000000..eafbb61b8c --- /dev/null +++ b/dev/release/README.md @@ -0,0 +1,174 @@ + + +# Release + +## Overview + + 1. Test the revision to be released + 2. Bump version for new release (detailed later) + 3. Prepare RC and vote (detailed later) + 4. Publish (detailed later) + 5. Bump version for new development (detailed later) + +### Bump version for new release + +Run `dev/release/bump_version.sh` on a working copy of your fork not +`git@github.com:apache/arrow-java`: + +```console +$ git clone git@github.com:${YOUR_GITHUB_ACCOUNT}/arrow-java.git arrow-java.${YOUR_GITHUB_ACCOUNT} +$ cd arrow-java.${YOUR_GITHUB_ACCOUNT} +$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/bump_version.sh ${NEW_VERSION} +``` + +Here is an example to bump version to 19.0.0: + +``` +$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/bump_version.sh 19.0.0 +``` + +It creates a feature branch and adds a commit that bumps version. This +opens a pull request from the feature branch by `gh pr create`. So you +need `gh` command and GitHub personal access token. + +See also: +https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens + +We need to merge the pull request before we cut a RC. If we try cut a +RC without merging the pull request, the script to cut a RC is failed. + +### Prepare RC and vote + +Run `dev/release/release_rc.sh` on a working copy of +`git@github.com:apache/arrow-java` not your fork: + +```console +$ git clone git@github.com:apache/arrow-java.git +$ cd arrow-java +$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release_rc.sh ${RC} +(Send a vote email to dev@arrow.apache.org. + You can use a draft shown by release_rc.sh for the email.) +``` + +Here is an example to release RC1: + +```console +$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release_rc.sh 1 +``` + +The argument of `release_rc.sh` is the RC number. If RC1 has a +problem, we'll increment the RC number such as RC2, RC3 and so on. + +Requirements to run `release_rc.sh`: + + * You must be an Apache Arrow committer or PMC member + * You must prepare your PGP key for signing + +If you don't have a PGP key, +https://infra.apache.org/release-signing.html#generate may be helpful. + +Your PGP key must be registered to the followings: + + * https://dist.apache.org/repos/dist/dev/arrow/KEYS + * https://dist.apache.org/repos/dist/release/arrow/KEYS + +See the header comment of them how to add a PGP key. + +Apache arrow committers can update them by Subversion client with +their ASF account. e.g.: + +```console +$ svn co https://dist.apache.org/repos/dist/dev/arrow +$ cd arrow +$ head KEYS +(This shows how to update KEYS) +$ svn ci KEYS +``` + +### Publish + +We need to do the followings to publish a new release: + + * Publish to apache.org + +Run `dev/release/release.sh` on a working copy of +`git@github.com:apache/arrow-java` not your fork to publish to +apache.org: + +```console +$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release.sh ${VERSION} ${RC} +``` + +Here is an example to release 19.0.0 RC1: + +```console +$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release.sh 19.0.0 1 +``` + +Add the release to ASF's report database via [Apache Committee Report +Helper](https://reporter.apache.org/addrelease.html?arrow). + +### Bump version for new development + +We should bump version in the main branch for new development after we +release a new version. + +Run `dev/release/bump_version.sh` on a working copy of your fork not +`git@github.com:apache/arrow-java`: + +```console +$ git clone git@github.com:${YOUR_GITHUB_ACCOUNT}/arrow-java.git arrow-java.${YOUR_GITHUB_ACCOUNT} +$ cd arrow-java.${YOUR_GITHUB_ACCOUNT} +$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/bump_version.sh ${NEW_VERSION}-SNAPSHOT +``` + +Here is an example to bump version to 19.0.1-SNAPSHOT: + +``` +$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/bump_version.sh 19.0.0-SNAPSHOT +``` + +It creates a feature branch and adds a commit that bumps version. This +opens a pull request from the feature branch by `gh pr create`. + +## Verify + +We have a script to verify a RC. + +You must install the following commands to use the script: + + * `curl` + * `gpg` + * `shasum` or `sha256sum`/`sha512sum` + * `tar` + +To verify a RC, run the following command line: + +```console +$ dev/release/verify_rc.sh ${VERSION} ${RC} +``` + +Here is an example to verify the release 19.0.0 RC1: + +```console +$ dev/release/verify_rc.sh 19.0.0 1 +``` + +If the verification is successful, the message `RC looks good!` is shown. diff --git a/dev/release/bump_version.sh b/dev/release/bump_version.sh new file mode 100755 index 0000000000..d1c127de35 --- /dev/null +++ b/dev/release/bump_version.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eu + +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)" + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + echo " e.g.: $0 19.0.1" + echo " e.g.: $0 20.0.0-SNAPSHOT" + exit 1 +fi + +version=$1 + +cd "${SOURCE_TOP_DIR}" + +git_origin_url="$(git remote get-url origin)" +case "${git_origin_url}" in +*apache/arrow-java*) + echo "You must use your fork: ${git_origin_url}" + exit 1 + ;; +*) + : # OK + ;; +esac + +branch="bump-version-${version}" +git switch -c "${branch}" main +mvn versions:set "-DnewVersion=${version}" -DprocessAllModules -DgenerateBackupPoms=false +case "${version}" in +*-SNAPSHOT) + tag=main + ;; +*) + tag=v${version} + ;; +esac +mvn versions:set-scm-tag "-DnewTag=${tag}" -DgenerateBackupPoms=false -pl :arrow-java-root +mvn versions:set-scm-tag "-DnewTag=${tag}" -DgenerateBackupPoms=false -pl :arrow-bom +git add "pom.xml" +git add "**/pom.xml" +git commit -m "MINOR: Bump version to ${version}" +git push --set-upstream origin "${branch}" +gh pr create --fill --repo apache/arrow-java diff --git a/dev/release/release.sh b/dev/release/release.sh new file mode 100755 index 0000000000..df671108d6 --- /dev/null +++ b/dev/release/release.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eu + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + echo " e.g.: $0 19.0.1 1" + exit 1 +fi + +version=$1 +rc=$2 + +git_origin_url="$(git remote get-url origin)" +repository="${git_origin_url#*github.com?}" +repository="${repository%.git}" +case "${git_origin_url}" in +git@github.com:apache/arrow-java.git | https://github.com/apache/arrow-java.git) + : # OK + ;; +*) + echo "This script must be ran with working copy of apache/arrow-java." + echo "The origin's URL: ${git_origin_url}" + exit 1 + ;; +esac + +tag="v${version}" +rc_tag="${tag}-rc${rc}" +echo "Tagging for release: ${tag}" +git tag -a -m "${version}" "${tag}" "${rc_tag}" +git push origin "${tag}" + +release_id="apache-arrow-java-${version}" +dist_url="https://dist.apache.org/repos/dist/release/arrow" +dist_base_dir="dev/release/dist" +dist_dir="dev/release/dist/${release_id}" +echo "Checking out ${dist_url}" +rm -rf "${dist_base_dir}" +svn co --depth=empty "${dist_url}" "${dist_base_dir}" +gh release download "${rc_tag}" \ + --repo "${repository}" \ + --dir "${dist_dir}" \ + --skip-existing + +echo "Uploading to release/" +pushd "${dist_base_dir}" +svn add . +svn ci -m "Apache Arrow Java ${version}" +popd +rm -rf "${dist_base_dir}" + +echo "Keep only the latest versions" +old_releases=$( + svn ls https://dist.apache.org/repos/dist/release/arrow/ | + grep -E '^apache-arrow-java-' | + sort --version-sort --reverse | + tail -n +2 +) +for old_release_version in ${old_releases}; do + echo "Remove old release ${old_release_version}" + svn \ + delete \ + -m "Remove old Apache Arrow Java release: ${old_release_version}" \ + "https://dist.apache.org/repos/dist/release/arrow/${old_release_version}" +done + +echo "Success! The release is available here:" +echo " https://dist.apache.org/repos/dist/release/arrow/${release_id}" +echo +echo "Add this release to ASF's report database:" +echo " https://reporter.apache.org/addrelease.html?arrow" diff --git a/dev/release/release_rc.sh b/dev/release/release_rc.sh new file mode 100755 index 0000000000..c8f1024a3f --- /dev/null +++ b/dev/release/release_rc.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eu + +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)" + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + echo " e.g.: $0 1" + exit 1 +fi + +rc=$1 + +: "${RELEASE_DEFAULT:=1}" +: "${RELEASE_PULL:=${RELEASE_DEFAULT}}" +: "${RELEASE_PUSH_TAG:=${RELEASE_DEFAULT}}" +: "${RELEASE_SIGN:=${RELEASE_DEFAULT}}" +: "${RELEASE_UPLOAD:=${RELEASE_DEFAULT}}" + +cd "${SOURCE_TOP_DIR}" + +if [ "${RELEASE_PULL}" -gt 0 ] || [ "${RELEASE_PUSH_TAG}" -gt 0 ]; then + git_origin_url="$(git remote get-url origin)" + case "${git_origin_url}" in + git@github.com:apache/arrow-java.git | https://github.com/apache/arrow-java.git) + : # OK + ;; + *) + echo "This script must be ran with working copy of apache/arrow-java." + echo "The origin's URL: ${git_origin_url}" + exit 1 + ;; + esac +fi + +if [ "${RELEASE_PULL}" -gt 0 ]; then + echo "Ensure using the latest commit" + git checkout main + git pull --ff-only +fi + +version=$(grep -o '^ .*' "pom.xml" | + sed \ + -e 's,^ ,,' \ + -e 's,$,,') + +case "${version}" in +*-SNAPSHOT) + echo "Version isn't bumped: ${version}" + echo "Run dev/release/bump_version.sh before you run this script." + exit 1 + ;; +esac + +rc_tag="v${version}-rc${rc}" +if [ "${RELEASE_PUSH_TAG}" -gt 0 ]; then + echo "Tagging for RC: ${rc_tag}" + git tag -a -m "${version} RC${rc}" "${rc_tag}" + git push origin "${rc_tag}" +fi + +rc_hash="$(git rev-list --max-count=1 "${rc_tag}")" + +id="apache-arrow-java-${version}" +tar_gz="${id}.tar.gz" + +if [ "${RELEASE_SIGN}" -gt 0 ]; then + git_origin_url="$(git remote get-url origin)" + repository="${git_origin_url#*github.com?}" + repository="${repository%.git}" + + echo "Looking for GitHub Actions workflow on ${repository}:${rc_tag}" + run_id="" + while [ -z "${run_id}" ]; do + echo "Waiting for run to start..." + run_id=$(gh run list \ + --branch "${rc_tag}" \ + --jq ".[].databaseId" \ + --json 'databaseId' \ + --limit 1 \ + --repo "${repository}" \ + --workflow rc.yml) + sleep 1 + done + + echo "Found GitHub Actions workflow with ID: ${run_id}" + gh run watch --repo "${repository}" --exit-status "${run_id}" + + echo "Downloading .tar.gz from GitHub Releases" + gh release download "${rc_tag}" \ + --dir . \ + --pattern "${tar_gz}" \ + --repo "${repository}" \ + --skip-existing + + echo "Signing tar.gz and creating checksums" + gpg --armor --output "${tar_gz}.asc" --detach-sig "${tar_gz}" +fi + +if [ "${RELEASE_UPLOAD}" -gt 0 ]; then + echo "Uploading signature" + gh release upload "${rc_tag}" \ + --clobber \ + --repo "${repository}" \ + "${tar_gz}.asc" +fi + +echo "Draft email for dev@arrow.apache.org mailing list" +echo "" +echo "---------------------------------------------------------" +cat < " + echo " e.g.: $0 19.0.1 1" + exit 1 +fi + +set -o pipefail +set -x + +VERSION="$1" +RC="$2" + +ARROW_DIST_BASE_URL="https://dist.apache.org/repos/dist/release/arrow" +DOWNLOAD_RC_BASE_URL="https://github.com/apache/arrow-java/releases/download/v${VERSION}-rc${RC}" +ARCHIVE_BASE_NAME="apache-arrow-java-${VERSION}" + +: "${VERIFY_DEFAULT:=1}" +: "${VERIFY_DOWNLOAD:=${VERIFY_DEFAULT}}" +: "${VERIFY_SIGN:=${VERIFY_DEFAULT}}" +: "${VERIFY_SOURCE:=${VERIFY_DEFAULT}}" +: "${VERIFY_BINARY:=${VERIFY_DEFAULT}}" + +VERIFY_SUCCESS=no + +setup_tmpdir() { + cleanup() { + if [ "${VERIFY_SUCCESS}" = "yes" ]; then + rm -rf "${VERIFY_TMPDIR}" + else + echo "Failed to verify release candidate. See ${VERIFY_TMPDIR} for details." + fi + } + + if [ -z "${VERIFY_TMPDIR:-}" ]; then + VERIFY_TMPDIR="$(mktemp -d -t "$1.XXXXX")" + trap cleanup EXIT + else + mkdir -p "${VERIFY_TMPDIR}" + fi +} + +download() { + curl \ + --fail \ + --location \ + --remote-name \ + --show-error \ + --silent \ + "$1" +} + +download_rc_file() { + if [ "${VERIFY_DOWNLOAD}" -gt 0 ]; then + download "${DOWNLOAD_RC_BASE_URL}/$1" + else + cp "${TOP_SOURCE_DIR}/$1" "$1" + fi +} + +import_gpg_keys() { + if [ "${VERIFY_SIGN}" -gt 0 ]; then + download "${ARROW_DIST_BASE_URL}/KEYS" + gpg --import KEYS + fi +} + +if type shasum >/dev/null 2>&1; then + sha256_verify="shasum -a 256 -c" + sha512_verify="shasum -a 512 -c" +else + sha256_verify="sha256sum -c" + sha512_verify="sha512sum -c" +fi + +fetch_archive() { + download_rc_file "${ARCHIVE_BASE_NAME}.tar.gz" + if [ "${VERIFY_SIGN}" -gt 0 ]; then + download_rc_file "${ARCHIVE_BASE_NAME}.tar.gz.asc" + gpg --verify "${ARCHIVE_BASE_NAME}.tar.gz.asc" "${ARCHIVE_BASE_NAME}.tar.gz" + fi + download_rc_file "${ARCHIVE_BASE_NAME}.tar.gz.sha256" + ${sha256_verify} "${ARCHIVE_BASE_NAME}.tar.gz.sha256" + download_rc_file "${ARCHIVE_BASE_NAME}.tar.gz.sha512" + ${sha512_verify} "${ARCHIVE_BASE_NAME}.tar.gz.sha512" +} + +ensure_source_directory() { + tar xf "${ARCHIVE_BASE_NAME}".tar.gz + + if [ -d "${TOP_SOURCE_DIR}/testing/data" ]; then + cp -a "${TOP_SOURCE_DIR}/testing" "${ARCHIVE_BASE_NAME}/" + else + git clone \ + https://github.com/apache/arrow-testing.git \ + "${ARCHIVE_BASE_NAME}/testing" + fi + + ARROW_TEST_DATA="$(pwd)/${ARCHIVE_BASE_NAME}/testing/data" + export ARROW_TEST_DATA +} + +test_source_distribution() { + if [ "${VERIFY_SOURCE}" -le 0 ]; then + return 0 + fi + + "${TOP_SOURCE_DIR}/ci/scripts/build.sh" "$(pwd)" build jni_dist + "${TOP_SOURCE_DIR}/ci/scripts/test.sh" "$(pwd)" build jni_dist + # TODO: JNI test + # TODO: Integration test +} + +test_binary_distribution() { + if [ "${VERIFY_BINARY}" -le 0 ]; then + return 0 + fi + + # TODO: jar test +} + +setup_tmpdir "arrow-java-${VERSION}-${RC}" +echo "Working in sandbox ${VERIFY_TMPDIR}" +cd "${VERIFY_TMPDIR}" + +import_gpg_keys +fetch_archive +ensure_source_directory +pushd "${ARCHIVE_BASE_NAME}" +test_source_distribution +test_binary_distribution +popd + +VERIFY_SUCCESS=yes +echo "RC looks good!" From 3375a5c078829d58d56c4c90d4b9d16059dc25ce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jan 2025 18:45:51 -0500 Subject: [PATCH 026/408] MINOR: Bump org.assertj:assertj-core from 3.26.3 to 3.27.2 (#505) Bumps [org.assertj:assertj-core](https://github.com/assertj/assertj) from 3.26.3 to 3.27.2. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0fd76bd2ed..8f2d867356 100644 --- a/pom.xml +++ b/pom.xml @@ -172,7 +172,7 @@ under the License. org.assertj assertj-core - 3.26.3 + 3.27.2 test From 1d4b75785246f0f2fafad373c0781985f8886f99 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 14 Jan 2025 10:46:31 +0900 Subject: [PATCH 027/408] GH-512: [Release] Fix `svn add` target (#513) Fixes GH-512. We can't use `svn add .` here. If we use `.`, we'll get the following error: svn: warning: W150002: '/tmp/arrow-java/dev/release/dist' is already under version control svn: E200009: Could not add all targets because some targets are already versioned svn: E200009: Illegal target for the requested operation --- dev/release/release.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/release/release.sh b/dev/release/release.sh index df671108d6..c0c7d3328a 100755 --- a/dev/release/release.sh +++ b/dev/release/release.sh @@ -51,7 +51,7 @@ git push origin "${tag}" release_id="apache-arrow-java-${version}" dist_url="https://dist.apache.org/repos/dist/release/arrow" dist_base_dir="dev/release/dist" -dist_dir="dev/release/dist/${release_id}" +dist_dir="${dist_base_dir}/${release_id}" echo "Checking out ${dist_url}" rm -rf "${dist_base_dir}" svn co --depth=empty "${dist_url}" "${dist_base_dir}" @@ -62,7 +62,7 @@ gh release download "${rc_tag}" \ echo "Uploading to release/" pushd "${dist_base_dir}" -svn add . +svn add "${release_id}" svn ci -m "Apache Arrow Java ${version}" popd rm -rf "${dist_base_dir}" From acbed51f051450cc7607a59fb324be1ecff078b8 Mon Sep 17 00:00:00 2001 From: An Qi Date: Tue, 14 Jan 2025 13:41:43 +0800 Subject: [PATCH 028/408] GH-469: Unify `ValueVector.getObject` and `VariableWidthFieldVector.get` behavior about null value (#486) Fixes #469. --- .../apache/arrow/vector/FixedSizeBinaryVector.java | 5 ++++- .../apache/arrow/vector/LargeVarBinaryVector.java | 13 +++++++++++-- .../org/apache/arrow/vector/LargeVarCharVector.java | 4 ++-- .../java/org/apache/arrow/vector/ValueVector.java | 2 +- .../org/apache/arrow/vector/VarBinaryVector.java | 10 +++++++++- .../java/org/apache/arrow/vector/VarCharVector.java | 2 +- .../apache/arrow/vector/ViewVarBinaryVector.java | 5 ++++- .../org/apache/arrow/vector/ViewVarCharVector.java | 2 +- 8 files changed, 33 insertions(+), 10 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/FixedSizeBinaryVector.java b/vector/src/main/java/org/apache/arrow/vector/FixedSizeBinaryVector.java index 4add729358..2005036c78 100644 --- a/vector/src/main/java/org/apache/arrow/vector/FixedSizeBinaryVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/FixedSizeBinaryVector.java @@ -149,7 +149,10 @@ public void get(int index, NullableFixedSizeBinaryHolder holder) { */ @Override public byte[] getObject(int index) { - return get(index); + if (isSet(index) == 0) { + return null; + } + return get(valueBuffer, index, byteWidth); } public int getByteWidth() { diff --git a/vector/src/main/java/org/apache/arrow/vector/LargeVarBinaryVector.java b/vector/src/main/java/org/apache/arrow/vector/LargeVarBinaryVector.java index f38627b933..fe798494c9 100644 --- a/vector/src/main/java/org/apache/arrow/vector/LargeVarBinaryVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/LargeVarBinaryVector.java @@ -16,6 +16,8 @@ */ package org.apache.arrow.vector; +import static org.apache.arrow.vector.NullCheckingForGet.NULL_CHECKING_ENABLED; + import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.ReusableBuffer; import org.apache.arrow.vector.complex.impl.LargeVarBinaryReaderImpl; @@ -95,7 +97,7 @@ public MinorType getMinorType() { */ public byte[] get(int index) { assert index >= 0; - if (isSet(index) == 0) { + if (NULL_CHECKING_ENABLED && isSet(index) == 0) { return null; } final long startOffset = getStartOffset(index); @@ -127,7 +129,14 @@ public void read(int index, ReusableBuffer buffer) { */ @Override public byte[] getObject(int index) { - return get(index); + if (isSet(index) == 0) { + return null; + } + final long startOffset = getStartOffset(index); + final long dataLength = getEndOffset(index) - startOffset; + final byte[] result = new byte[(int) dataLength]; + valueBuffer.getBytes(startOffset, result, 0, (int) dataLength); + return result; } /** diff --git a/vector/src/main/java/org/apache/arrow/vector/LargeVarCharVector.java b/vector/src/main/java/org/apache/arrow/vector/LargeVarCharVector.java index 07a9a172f0..b7a765f310 100644 --- a/vector/src/main/java/org/apache/arrow/vector/LargeVarCharVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/LargeVarCharVector.java @@ -101,7 +101,7 @@ public Types.MinorType getMinorType() { @Override public byte[] get(int index) { assert index >= 0; - if (isSet(index) == 0) { + if (NULL_CHECKING_ENABLED && isSet(index) == 0) { return null; } final long startOffset = getStartOffset(index); @@ -120,7 +120,7 @@ public byte[] get(int index) { @Override public Text getObject(int index) { assert index >= 0; - if (NULL_CHECKING_ENABLED && isSet(index) == 0) { + if (isSet(index) == 0) { return null; } diff --git a/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/vector/src/main/java/org/apache/arrow/vector/ValueVector.java index 0a45409eb9..3a5058256c 100644 --- a/vector/src/main/java/org/apache/arrow/vector/ValueVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/ValueVector.java @@ -264,7 +264,7 @@ public interface ValueVector extends Closeable, Iterable { * Get friendly type object from the vector. * * @param index index of object to get - * @return friendly type object + * @return friendly type object, null if value is unset */ Object getObject(int index); diff --git a/vector/src/main/java/org/apache/arrow/vector/VarBinaryVector.java b/vector/src/main/java/org/apache/arrow/vector/VarBinaryVector.java index 7196e9c910..ad76504f0f 100644 --- a/vector/src/main/java/org/apache/arrow/vector/VarBinaryVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/VarBinaryVector.java @@ -128,7 +128,15 @@ public void read(int index, ReusableBuffer buffer) { */ @Override public byte[] getObject(int index) { - return get(index); + if (isSet(index) == 0) { + return null; + } + + final int startOffset = getStartOffset(index); + final int dataLength = getEndOffset(index) - startOffset; + final byte[] result = new byte[dataLength]; + valueBuffer.getBytes(startOffset, result, 0, dataLength); + return result; } /** diff --git a/vector/src/main/java/org/apache/arrow/vector/VarCharVector.java b/vector/src/main/java/org/apache/arrow/vector/VarCharVector.java index c81e34558c..5ddc8b84d2 100644 --- a/vector/src/main/java/org/apache/arrow/vector/VarCharVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/VarCharVector.java @@ -117,7 +117,7 @@ public byte[] get(int index) { @Override public Text getObject(int index) { assert index >= 0; - if (NULL_CHECKING_ENABLED && isSet(index) == 0) { + if (isSet(index) == 0) { return null; } diff --git a/vector/src/main/java/org/apache/arrow/vector/ViewVarBinaryVector.java b/vector/src/main/java/org/apache/arrow/vector/ViewVarBinaryVector.java index 80d6952e00..c41854bb5f 100644 --- a/vector/src/main/java/org/apache/arrow/vector/ViewVarBinaryVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/ViewVarBinaryVector.java @@ -122,7 +122,10 @@ public void read(int index, ReusableBuffer buffer) { */ @Override public byte[] getObject(int index) { - return get(index); + if (isSet(index) == 0) { + return null; + } + return getData(index); } /** diff --git a/vector/src/main/java/org/apache/arrow/vector/ViewVarCharVector.java b/vector/src/main/java/org/apache/arrow/vector/ViewVarCharVector.java index dc474b68e3..9ce7f85ef6 100644 --- a/vector/src/main/java/org/apache/arrow/vector/ViewVarCharVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/ViewVarCharVector.java @@ -115,7 +115,7 @@ public byte[] get(int index) { @Override public Text getObject(int index) { assert index >= 0; - if (NULL_CHECKING_ENABLED && isSet(index) == 0) { + if (isSet(index) == 0) { return null; } From 7953e4483808fbb95bf86b4fe6a8ce3db5c1c017 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Jan 2025 00:49:26 -0500 Subject: [PATCH 029/408] MINOR: Bump checker.framework.version from 3.48.2 to 3.48.4 (#506) Bumps `checker.framework.version` from 3.48.2 to 3.48.4. Updates `org.checkerframework:checker-qual` from 3.48.2 to 3.48.4 Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8f2d867356..1d71bb9511 100644 --- a/pom.xml +++ b/pom.xml @@ -109,7 +109,7 @@ under the License. true 2.31.0 5.15.2 - 3.48.2 + 3.48.4 1.5.16 none -Xdoclint:none From 16406d31ec36d7e7edeb660985b5b2e1e9e9e4b0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Jan 2025 00:50:12 -0500 Subject: [PATCH 030/408] MINOR: Bump org.mockito:mockito-junit-jupiter from 5.14.2 to 5.15.2 (#508) Bumps [org.mockito:mockito-junit-jupiter](https://github.com/mockito/mockito) from 5.14.2 to 5.15.2. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1d71bb9511..8660ff39b5 100644 --- a/pom.xml +++ b/pom.xml @@ -278,7 +278,7 @@ under the License. org.mockito mockito-junit-jupiter - 5.14.2 + 5.15.2 test From 3878927025364343a1dd5aa9004ddcdaf8ceef82 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Jan 2025 01:01:48 -0500 Subject: [PATCH 031/408] MINOR: Bump com.gradle:develocity-maven-extension from 1.22.2 to 1.23 (#507) Bumps com.gradle:develocity-maven-extension from 1.22.2 to 1.23. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .mvn/extensions.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index 0836fc47d0..47cffa9b57 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -23,7 +23,7 @@ com.gradle develocity-maven-extension - 1.22.2 + 1.23 com.gradle From f4a0adfb56ffbd4d94df907d6dcf18b113c6e542 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 15 Jan 2025 11:34:27 +0900 Subject: [PATCH 032/408] GH-518: Cleanup apache-arrow-java.tar.gz created by pre-commit (#519) Fixes GH-518. --- .pre-commit-config.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ff0d85448f..dd365b15db 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,7 +35,8 @@ repos: git archive HEAD \ --prefix=apache-arrow-java/ \ --output=apache-arrow-java.tar.gz && \ - dev/release/run_rat.sh apache-arrow-java.tar.gz" + dev/release/run_rat.sh apache-arrow-java.tar.gz && \ + rm -f apache-arrow-java.tar.gz" always_run: true pass_filenames: false - repo: https://github.com/koalaman/shellcheck-precommit From 4f9b45213cec5607a250f264271f512d5b27dd59 Mon Sep 17 00:00:00 2001 From: Clay Johnson Date: Tue, 14 Jan 2025 21:47:46 -0600 Subject: [PATCH 033/408] GH-515: Publish build scans to develocity.apache.org (#516) This PR migrates the Arrow project to publish Build Scans to the the new Develocity instance at develocity.apache.org. Additionally, this PR sets a projectId for use by Develocity. Fixes #515. --- .github/workflows/test.yml | 10 +++++----- .mvn/develocity.xml | 15 +++++++-------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a61578572e..7841dd89ca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -71,7 +71,7 @@ jobs: - name: Execute Docker Build env: # Enables build caching, but not strictly required - DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} run: | docker compose run \ -e CI=true \ @@ -107,12 +107,12 @@ jobs: - name: Build shell: bash env: - DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} run: ci/scripts/build.sh $(pwd) $(pwd)/build - name: Test shell: bash env: - DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} run: ci/scripts/test.sh $(pwd) $(pwd)/build windows: @@ -138,12 +138,12 @@ jobs: - name: Build shell: bash env: - DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} run: ci/scripts/build.sh $(pwd) $(pwd)/build - name: Test shell: bash env: - DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} run: ci/scripts/test.sh $(pwd) $(pwd)/build integration: diff --git a/.mvn/develocity.xml b/.mvn/develocity.xml index df3cbccd2b..298f1efcff 100644 --- a/.mvn/develocity.xml +++ b/.mvn/develocity.xml @@ -20,19 +20,18 @@ --> + arrow - https://ge.apache.org + https://develocity.apache.org false - - true - true - true - #{isFalse(env['CI'])} - true - true + + + + + #{{'0.0.0.0'}} From 89217c8ed7222e4f30e0190c65cf64ee06d9ec06 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 15 Jan 2025 12:51:05 +0900 Subject: [PATCH 034/408] GH-520: [CI] Add cmake-format to pre-commit (#521) Fixes GH-520. The configuration file is borrowed from apache/arrow. --- .cmake-format.py | 76 +++++++++++++++++++++++++++++++++++++++++ .pre-commit-config.yaml | 4 +++ 2 files changed, 80 insertions(+) create mode 100644 .cmake-format.py diff --git a/.cmake-format.py b/.cmake-format.py new file mode 100644 index 0000000000..b8fc893969 --- /dev/null +++ b/.cmake-format.py @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# cmake-format configuration file +# Use `archery lint --cmake-format --fix` to reformat all cmake files in the +# source tree + +# ----------------------------- +# Options affecting formatting. +# ----------------------------- +with section("format"): + # How wide to allow formatted cmake files + line_width = 90 + + # How many spaces to tab for indent + tab_size = 2 + + # If a positional argument group contains more than this many arguments, + # then force it to a vertical layout. + max_pargs_hwrap = 4 + + # If the statement spelling length (including space and parenthesis) is + # smaller than this amount, then force reject nested layouts. + # This value only comes into play when considering whether or not to nest + # arguments below their parent. If the number of characters in the parent + # is less than this value, we will not nest. + min_prefix_chars = 32 + + # If true, separate flow control names from their parentheses with a space + separate_ctrl_name_with_space = False + + # If true, separate function names from parentheses with a space + separate_fn_name_with_space = False + + # If a statement is wrapped to more than one line, than dangle the closing + # parenthesis on it's own line + dangle_parens = False + + # What style line endings to use in the output. + line_ending = 'unix' + + # Format command names consistently as 'lower' or 'upper' case + command_case = 'lower' + + # Format keywords consistently as 'lower' or 'upper' case + keyword_case = 'unchanged' + +# ------------------------------------------------ +# Options affecting comment reflow and formatting. +# ------------------------------------------------ +with section("markup"): + # enable comment markup parsing and reflow + enable_markup = False + + # If comment markup is enabled, don't reflow the first comment block in + # eachlistfile. Use this to preserve formatting of your + # copyright/licensestatements. + first_comment_is_literal = True + + # If comment markup is enabled, don't reflow any comment block which + # matches this (regex) pattern. Default is `None` (disabled). + literal_comment_pattern = None diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dd365b15db..0a762414fd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,6 +16,10 @@ # under the License. repos: + - repo: https://github.com/cheshirekow/cmake-format-precommit + rev: v0.6.13 + hooks: + - id: cmake-format - repo: https://github.com/pre-commit/pre-commit-hooks rev: cef0300fd0fc4d2a87a85fa2093c6b283ea36f4b # v5.0.0 hooks: From 6e3c8ba99f545e273063a6ab6a351c957cd872bc Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 16 Jan 2025 09:14:42 +0900 Subject: [PATCH 035/408] GH-523: [Release] Add support for building .jar including JNI binaries (#517) Fixes GH-523. Merge `test_jni.yml` to `rc.yml` so that .jar including JNI binaries can be built and uploaded to GitHub Releases for RC. ccache support is also enabled for fast CI. --- .github/workflows/rc.yml | 299 +++++++++++++++++++++++++++--- .github/workflows/test_jni.yml | 267 -------------------------- CMakeLists.txt | 5 +- ci/scripts/build.sh | 1 + ci/scripts/download_cpp.sh | 52 ++++++ ci/scripts/jni_build.sh | 21 +-- ci/scripts/jni_full_build.sh | 25 ++- ci/scripts/jni_macos_build.sh | 31 ++-- ci/scripts/jni_manylinux_build.sh | 35 ++-- ci/scripts/test.sh | 1 + docker-compose.yml | 6 +- 11 files changed, 398 insertions(+), 345 deletions(-) delete mode 100644 .github/workflows/test_jni.yml create mode 100755 ci/scripts/download_cpp.sh diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 5a82d686e8..666767ff71 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -32,8 +32,8 @@ concurrency: permissions: contents: read jobs: - archive: - name: Archive + source: + name: Source runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -70,15 +70,269 @@ jobs: - name: Audit run: | dev/release/run_rat.sh "${TAR_GZ}" - - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 + - name: Upload source archive + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 with: - name: archive + name: release-source path: | apache-arrow-java-* + jni-ubuntu: + name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }} + runs-on: ${{ matrix.platform.runs_on }} + needs: + - source + strategy: + fail-fast: false + matrix: + platform: + - runs_on: ubuntu-latest + arch: "x86_64" + archery_arch: "amd64" + env: + # architecture name used for archery build + ARCH: ${{ matrix.platform.archery_arch }} + DOCKER_VOLUME_PREFIX: .docker/ + permissions: + contents: read + packages: write + steps: + - name: Download source archive + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: release-source + - name: Extract Download the latest Apache Arrow C++ + run: | + tar -xf apache-arrow-java-*.tar.gz --strip-components=1 + - name: Download the latest Apache Arrow C++ + run: | + ci/scripts/download_cpp.sh + - name: Checkout apache/arrow-testing + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: apache/arrow-testing + path: arrow/testing + - name: Checkout apache/parquet-testing + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: apache/parquet-testing + path: arrow/cpp/submodules/parquet-testing + - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Cache + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + with: + path: .docker + key: jni-linux-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} + restore-keys: jni-linux-${{ matrix.platform.arch }}- + - name: Build C++ libraries + run: | + docker compose run vcpkg-jni + - name: Push Docker image + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow-java' && github.ref_name == 'main' + run: | + docker compose push vcpkg-jni + - name: Compress into single artifact to keep directory structure + run: tar -cvzf jni-linux-${{ matrix.platform.arch }}.tar.gz jni/ + - name: Upload artifacts + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: jni-linux-${{ matrix.platform.arch }} + path: jni-linux-${{ matrix.platform.arch }}.tar.gz + jni-macos: + name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }} + runs-on: ${{ matrix.platform.runs_on }} + needs: + - source + strategy: + fail-fast: false + matrix: + platform: + - { runs_on: macos-13, arch: "x86_64"} + - { runs_on: macos-14, arch: "aarch_64" } + env: + MACOSX_DEPLOYMENT_TARGET: "14.0" + steps: + - name: Download source archive + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: release-source + - name: Extract Download the latest Apache Arrow C++ + run: | + tar -xf apache-arrow-java-*.tar.gz --strip-components=1 + # We need 19.0.0 for latest Boost support + - name: Download the latest RC Apache Arrow C++ + run: | + ci/scripts/download_cpp.sh latest-rc + - name: Checkout apache/arrow-testing + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: apache/arrow-testing + path: arrow/testing + - name: Checkout apache/parquet-testing + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: apache/parquet-testing + path: arrow/cpp/submodules/parquet-testing + - name: Set up Python + uses: actions/setup-python@v5 + with: + cache: 'pip' + python-version: 3.12 + - name: Install Archery + run: pip install -e arrow/dev/archery[all] + - name: Install dependencies + run: | + # We want to use llvm@14 to avoid shared z3 + # dependency. llvm@14 doesn't depend on z3 and llvm depends + # on z3. And Homebrew's z3 provides only shared library. It + # doesn't provides static z3 because z3's CMake doesn't accept + # building both shared and static libraries at once. + # See also: Z3_BUILD_LIBZ3_SHARED in + # https://github.com/Z3Prover/z3/blob/master/README-CMake.md + # + # If llvm is installed, Apache Arrow C++ uses llvm rather than + # llvm@14 because llvm is newer than llvm@14. + brew uninstall llvm || : + + # Ensure updating python@XXX with the "--overwrite" option. + # If python@XXX is updated without "--overwrite", it causes + # a conflict error. Because Python 3 installed not by + # Homebrew exists in /usr/local on GitHub Actions. If + # Homebrew's python@XXX is updated without "--overwrite", it + # tries to replace /usr/local/bin/2to3 and so on and causes + # a conflict error. + brew update + for python_package in $(brew list | grep python@); do + brew install --overwrite ${python_package} + done + brew install --overwrite python + + if [ "$(uname -m)" = "arm64" ]; then + # pkg-config formula is deprecated but it's still installed + # in GitHub Actions runner now. We can remove this once + # pkg-config formula is removed from GitHub Actions runner. + brew uninstall pkg-config || : + brew uninstall pkg-config@0.29.2 || : + fi + + brew bundle --file=arrow/cpp/Brewfile + # We want to link aws-sdk-cpp statically but Homebrew's + # aws-sdk-cpp provides only shared library. If we have + # Homebrew's aws-sdk-cpp, our build mix Homebrew's + # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's + # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp. + brew uninstall aws-sdk-cpp + # We want to use bundled RE2 for static linking. If + # Homebrew's RE2 is installed, its header file may be used. + # We uninstall Homebrew's RE2 to ensure using bundled RE2. + brew uninstall grpc || : # gRPC depends on RE2 + brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too + brew uninstall re2 + # We want to use bundled Protobuf for static linking. If + # Homebrew's Protobuf is installed, its library file may be + # used on test We uninstall Homebrew's Protobuf to ensure using + # bundled Protobuf. + brew uninstall protobuf + + brew bundle --file=Brewfile + - name: Prepare ccache + run: | + echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} + - name: Cache ccache + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + with: + path: ccache + key: jni-macos-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} + restore-keys: jni-macos-${{ matrix.platform.arch }}- + - name: Build C++ libraries + run: | + set -e + # make brew Java available to CMake + export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home + ci/scripts/jni_macos_build.sh . arrow build jni + - name: Compress into single artifact to keep directory structure + run: tar -cvzf jni-macos-${{ matrix.platform.arch }}.tar.gz jni/ + - name: Upload artifacts + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: jni-macos-${{ matrix.platform.arch }} + path: jni-macos-${{ matrix.platform.arch }}.tar.gz + binaries: + name: Binaries + runs-on: ubuntu-latest + needs: + - jni-ubuntu + - jni-macos + steps: + - name: Download artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + path: artifacts + - name: Decompress artifacts + run: | + mv artifacts/*/*.tar.gz . + tar -xf apache-arrow-java-*.tar.gz --strip-components=1 + tar -xvzf jni-linux-x86_64.tar.gz + # tar -xvzf jni-linux-aarch_64.tar.gz + tar -xvzf jni-macos-x86_64.tar.gz + tar -xvzf jni-macos-aarch_64.tar.gz + # tar -xvzf jni-windows.tar.gz + - name: Test that shared libraries exist + run: | + set -x + + test -f jni/arrow_cdata_jni/x86_64/libarrow_cdata_jni.so + test -f jni/arrow_dataset_jni/x86_64/libarrow_dataset_jni.so + test -f jni/arrow_orc_jni/x86_64/libarrow_orc_jni.so + test -f jni/gandiva_jni/x86_64/libgandiva_jni.so + + # test -f jni/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.so + # test -f jni/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.so + # test -f jni/arrow_orc_jni/aarch_64/libarrow_orc_jni.so + # test -f jni/gandiva_jni/aarch_64/libgandiva_jni.so + + test -f jni/arrow_cdata_jni/x86_64/libarrow_cdata_jni.dylib + test -f jni/arrow_dataset_jni/x86_64/libarrow_dataset_jni.dylib + test -f jni/arrow_orc_jni/x86_64/libarrow_orc_jni.dylib + test -f jni/gandiva_jni/x86_64/libgandiva_jni.dylib + + test -f jni/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.dylib + test -f jni/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.dylib + test -f jni/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib + test -f jni/gandiva_jni/aarch_64/libgandiva_jni.dylib + + # test -f jni/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll + # test -f jni/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll + # test -f jni/arrow_orc_jni/x86_64/arrow_orc_jni.dll + - name: Checkout apache/arrow-testing + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: apache/arrow-testing + path: testing + - name: Cache ~/.m2 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + with: + path: ~/.m2 + key: binaries-build-${{ hashFiles('**/*.java', '**/pom.xml') }} + restore-keys: binaries-build- + - name: Build bundled JAR + env: + MAVEN_ARGS: >- + --no-transfer-progress + run: | + ci/scripts/jni_full_build.sh . jni binaries + - name: Upload artifacts + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: release-binaries + path: binaries/* verify: name: Verify needs: - - archive + - binaries runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -87,15 +341,13 @@ jobs: - macos-latest - ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - submodules: recursive - - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + - name: Download release artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: - name: archive + pattern: release-* - name: Verify run: | + mv release-source/* ./ tar_gz=$(echo apache-arrow-java-*.tar.gz) version=${tar_gz#apache-arrow-java-} version=${version%.tar.gz} @@ -105,9 +357,14 @@ jobs: else rc=$(date +%Y%m%d) fi - VERIFY_DEFAULT=0 \ - VERIFY_SOURCE=1 \ - dev/release/verify_rc.sh "${version}" "${rc}" + tar -xf ${tar_gz} + export VERIFY_DEFAULT=0 + export VERIFY_BINARY=1 + export VERIFY_SOURCE=1 + cd apache-arrow-java-${version} + mv ../${tar_gz}* ./ + mv ../release-binaries binaries + dev/release/verify_rc.sh "${version}" "${rc}" upload: name: Upload if: github.ref_type == 'tag' @@ -117,13 +374,11 @@ jobs: permissions: contents: write steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - submodules: recursive - - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + - name: Download release artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: - name: archive + pattern: release-* + path: artifacts - name: Upload run: | # GH-499: How to create release notes? @@ -133,9 +388,9 @@ jobs: gh release create ${GITHUB_REF_NAME} \ --generate-notes \ --prerelease \ + --repo ${{ github.repository }} \ --title "Apache Arrow Java ${version} RC${rc}" \ --verify-tag \ - apache-arrow-java-*.tar.gz \ - apache-arrow-java-*.tar.gz.sha* + artifacts/*/* env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test_jni.yml b/.github/workflows/test_jni.yml deleted file mode 100644 index 15b535e17f..0000000000 --- a/.github/workflows/test_jni.yml +++ /dev/null @@ -1,267 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Test (JNI) - -on: - push: - branches: - - '**' - - '!dependabot/**' - tags: - - '**' - pull_request: - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -env: - DOCKER_VOLUME_PREFIX: ".docker/" - -jobs: - cpp-ubuntu: - name: Build C++ libraries ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }} - runs-on: ${{ matrix.platform.runs_on }} - strategy: - fail-fast: false - matrix: - platform: - - runs_on: ubuntu-latest - arch: "x86_64" - archery_arch: "amd64" - env: - # architecture name used for archery build - ARCH: ${{ matrix.platform.archery_arch }} - permissions: - contents: read - packages: write - steps: - - name: Checkout apache/arrow-java - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - with: - fetch-depth: 0 - submodules: recursive - - name: Checkout apache/arrow - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - with: - repository: apache/arrow - fetch-depth: 0 - path: arrow - submodules: recursive - - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Build C++ libraries - run: | - docker compose run vcpkg-jni - - name: Push Docker image - if: success() && github.event_name == 'push' && github.repository == 'apache/arrow-java' && github.ref_name == 'main' - run: | - docker compose push vcpkg-jni - - name: Compress into single artifact to keep directory structure - run: tar -cvzf arrow-shared-libs-linux-${{ matrix.platform.arch }}.tar.gz dist/ - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: ubuntu-shared-lib-${{ matrix.platform.arch }} - path: arrow-shared-libs-linux-${{ matrix.platform.arch }}.tar.gz - - cpp-macos: - name: Build C++ libraries macOS ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }} - runs-on: ${{ matrix.platform.runs_on }} - strategy: - fail-fast: false - matrix: - platform: - - { runs_on: macos-13, arch: "x86_64"} - - { runs_on: macos-14, arch: "aarch_64" } - env: - MACOSX_DEPLOYMENT_TARGET: "14.0" - steps: - - name: Checkout apache/arrow-java - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - with: - fetch-depth: 0 - submodules: recursive - - name: Checkout apache/arrow - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - with: - repository: apache/arrow - fetch-depth: 0 - path: arrow - submodules: recursive - - name: Set up Python - uses: actions/setup-python@v5 - with: - cache: 'pip' - python-version: 3.12 - - name: Install Archery - run: pip install -e arrow/dev/archery[all] - - name: Install dependencies - run: | - # We want to use llvm@14 to avoid shared z3 - # dependency. llvm@14 doesn't depend on z3 and llvm depends - # on z3. And Homebrew's z3 provides only shared library. It - # doesn't provides static z3 because z3's CMake doesn't accept - # building both shared and static libraries at once. - # See also: Z3_BUILD_LIBZ3_SHARED in - # https://github.com/Z3Prover/z3/blob/master/README-CMake.md - # - # If llvm is installed, Apache Arrow C++ uses llvm rather than - # llvm@14 because llvm is newer than llvm@14. - brew uninstall llvm || : - - # Ensure updating python@XXX with the "--overwrite" option. - # If python@XXX is updated without "--overwrite", it causes - # a conflict error. Because Python 3 installed not by - # Homebrew exists in /usr/local on GitHub Actions. If - # Homebrew's python@XXX is updated without "--overwrite", it - # tries to replace /usr/local/bin/2to3 and so on and causes - # a conflict error. - brew update - for python_package in $(brew list | grep python@); do - brew install --overwrite ${python_package} - done - brew install --overwrite python - - if [ "$(uname -m)" = "arm64" ]; then - # pkg-config formula is deprecated but it's still installed - # in GitHub Actions runner now. We can remove this once - # pkg-config formula is removed from GitHub Actions runner. - brew uninstall pkg-config || : - brew uninstall pkg-config@0.29.2 || : - fi - - brew bundle --file=arrow/cpp/Brewfile - # We want to link aws-sdk-cpp statically but Homebrew's - # aws-sdk-cpp provides only shared library. If we have - # Homebrew's aws-sdk-cpp, our build mix Homebrew's - # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's - # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp. - brew uninstall aws-sdk-cpp - # We want to use bundled RE2 for static linking. If - # Homebrew's RE2 is installed, its header file may be used. - # We uninstall Homebrew's RE2 to ensure using bundled RE2. - brew uninstall grpc || : # gRPC depends on RE2 - brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too - brew uninstall re2 - # We want to use bundled Protobuf for static linking. If - # Homebrew's Protobuf is installed, its library file may be - # used on test We uninstall Homebrew's Protobuf to ensure using - # bundled Protobuf. - brew uninstall protobuf - - brew bundle --file=Brewfile - - name: Build C++ libraries - run: | - set -e - # make brew Java available to CMake - export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home - ./ci/scripts/jni_macos_build.sh \ - $GITHUB_WORKSPACE \ - $GITHUB_WORKSPACE/arrow \ - $GITHUB_WORKSPACE/arrow-java/cpp-build \ - $GITHUB_WORKSPACE/dist - - name: Compress into single artifact to keep directory structure - run: tar -cvzf arrow-shared-libs-macos-${{ matrix.platform.arch }}.tar.gz dist/ - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: macos-shared-lib-${{ matrix.platform.arch }} - path: arrow-shared-libs-macos-${{ matrix.platform.arch }}.tar.gz - - java-jars: - name: Build JAR files - runs-on: ubuntu-latest - needs: - - cpp-ubuntu - - cpp-macos - steps: - - name: Checkout apache/arrow-java - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - with: - fetch-depth: 0 - submodules: recursive - - name: Checkout apache/arrow - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - with: - repository: apache/arrow - fetch-depth: 0 - path: arrow - submodules: recursive - - name: Download Libraries - uses: actions/download-artifact@v4 - with: - path: artifacts - - name: Decompress artifacts - run: | - mv artifacts/*/*.tar.gz . - tar -xvzf arrow-shared-libs-linux-x86_64.tar.gz - # tar -xvzf arrow-shared-libs-linux-aarch_64.tar.gz - tar -xvzf arrow-shared-libs-macos-x86_64.tar.gz - tar -xvzf arrow-shared-libs-macos-aarch_64.tar.gz - # tar -xvzf arrow-shared-libs-windows.tar.gz - - name: Test that shared libraries exist - run: | - set -x - - test -f dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.so - test -f dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.so - test -f dist/arrow_orc_jni/x86_64/libarrow_orc_jni.so - test -f dist/gandiva_jni/x86_64/libgandiva_jni.so - - # test -f dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.so - # test -f dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.so - # test -f dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.so - # test -f dist/gandiva_jni/aarch_64/libgandiva_jni.so - - test -f dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.dylib - test -f dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.dylib - test -f dist/arrow_orc_jni/x86_64/libarrow_orc_jni.dylib - test -f dist/gandiva_jni/x86_64/libgandiva_jni.dylib - - test -f dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.dylib - test -f dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.dylib - test -f dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib - test -f dist/gandiva_jni/aarch_64/libgandiva_jni.dylib - - # test -f dist/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll - # test -f dist/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll - # test -f dist/arrow_orc_jni/x86_64/arrow_orc_jni.dll - - name: Build bundled jar - env: - MAVEN_ARGS: >- - --no-transfer-progress - run: | - set -e - # mvn versions:set -DnewVersion={{ arrow.no_rc_snapshot_version }} - # mvn versions:set -DnewVersion={{ arrow.no_rc_snapshot_version }} -f bom - ./ci/scripts/jni_full_build.sh \ - $GITHUB_WORKSPACE \ - $GITHUB_WORKSPACE/arrow \ - $GITHUB_WORKSPACE/dist - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: java-jars - path: ${{ github.workspace }}/arrow-java/java-dist diff --git a/CMakeLists.txt b/CMakeLists.txt index 8b29f37d80..318bd4d10c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -66,7 +66,10 @@ add_library(jni INTERFACE IMPORTED) set_target_properties(jni PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${JNI_INCLUDE_DIRS}") include(CTest) -if(BUILD_TESTING) +if(BUILD_TESTING + AND (ARROW_JAVA_JNI_ENABLE_DATASET + OR ARROW_JAVA_JNI_ENABLE_GANDIVA + OR ARROW_JAVA_JNI_ENABLE_ORC)) find_package(ArrowTesting REQUIRED) find_package(GTest REQUIRED) add_library(arrow_java_test INTERFACE IMPORTED) diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index f3f2023759..309f28126a 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/ci/scripts/download_cpp.sh b/ci/scripts/download_cpp.sh new file mode 100755 index 0000000000..4d801a47f9 --- /dev/null +++ b/ci/scripts/download_cpp.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euxo pipefail + +if [ $# -eq 1 ]; then + version="${1}" +else + version="latest-release" +fi + +url="" + +if [ "${version}" = "latest-release" ]; then + version=$(curl \ + https://raw.githubusercontent.com/apache/arrow-site/refs/heads/main/_data/versions.yml | + grep '^ number:' | + sed -E -e "s/^ number: '|'$//g") +elif [ "${version}" = "latest-rc" ]; then + rc_archive_name=$(curl \ + https://dist.apache.org/repos/dist/dev/arrow/ | + grep -E -o 'apache-arrow-[0-9]+\.[0-9]+\.[0-9]+\-rc[0-9]' | + sort | + uniq | + tail -n1) + rc_version="${rc_archive_name#apache-arrow-}" + version="${rc_version%-rc*}" + url="https://dist.apache.org/repos/dist/dev/arrow/apache-arrow-${rc_version}/apache-arrow-${version}.tar.gz" +fi + +if [ -z "${url}" ]; then + url="https://www.apache.org/dyn/closer.lua?action=download&filename=arrow/arrow-${version}/apache-arrow-${version}.tar.gz" +fi +wget --output-document "apache-arrow-${version}.tar.gz" "${url}" +tar xf "apache-arrow-${version}.tar.gz" +mv "apache-arrow-${version}" arrow diff --git a/ci/scripts/jni_build.sh b/ci/scripts/jni_build.sh index 4462646347..a77b0da02a 100755 --- a/ci/scripts/jni_build.sh +++ b/ci/scripts/jni_build.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,9 +17,9 @@ # specific language governing permissions and limitations # under the License. -set -exo pipefail +set -euxo pipefail -arrow_dir=${1} +source_dir=${1} arrow_install_dir=${2} build_dir=${3}/java_jni # The directory where the final binaries will be stored when scripts finish @@ -30,8 +31,6 @@ echo "=== Clear output directories and leftovers ===" rm -rf "${build_dir}" echo "=== Building Arrow Java C Data Interface native library ===" -mkdir -p "${build_dir}" -pushd "${build_dir}" case "$(uname)" in Linux) @@ -45,10 +44,12 @@ Darwin) ;; esac -: "${ARROW_JAVA_BUILD_TESTS:=${ARROW_BUILD_TESTS:-OFF}}" +: "${ARROW_JAVA_BUILD_TESTS:=${ARROW_BUILD_TESTS:-ON}}" : "${CMAKE_BUILD_TYPE:=release}" read -ra EXTRA_CMAKE_OPTIONS <<<"${JAVA_JNI_CMAKE_ARGS:-}" cmake \ + -S "${source_dir}" \ + -B "${build_dir}" \ -DARROW_JAVA_JNI_ENABLE_DATASET="${ARROW_DATASET:-OFF}" \ -DARROW_JAVA_JNI_ENABLE_GANDIVA="${ARROW_GANDIVA:-OFF}" \ -DARROW_JAVA_JNI_ENABLE_ORC="${ARROW_ORC:-OFF}" \ @@ -59,18 +60,16 @@ cmake \ -DCMAKE_UNITY_BUILD="${CMAKE_UNITY_BUILD:-OFF}" \ -DProtobuf_USE_STATIC_LIBS=ON \ -GNinja \ - "${EXTRA_CMAKE_OPTIONS[@]}" \ - "${arrow_dir}" -export CMAKE_BUILD_PARALLEL_LEVEL=${n_jobs} -cmake --build . --config "${CMAKE_BUILD_TYPE}" + "${EXTRA_CMAKE_OPTIONS[@]}" +cmake --build "${build_dir}" if [ "${ARROW_JAVA_BUILD_TESTS}" = "ON" ]; then ctest \ --output-on-failure \ --parallel "${n_jobs}" \ + --test-dir "${build_dir}" \ --timeout 300 fi -cmake --build . --config "${CMAKE_BUILD_TYPE}" --target install -popd +cmake --build "${build_dir}" --target install mkdir -p "${dist_dir}" # For Windows. *.dll are installed into bin/ on Windows. diff --git a/ci/scripts/jni_full_build.sh b/ci/scripts/jni_full_build.sh index 1a39aeb510..15cf72f21b 100755 --- a/ci/scripts/jni_full_build.sh +++ b/ci/scripts/jni_full_build.sh @@ -19,16 +19,13 @@ set -e -arrow_java_dir="${1}" -arrow_dir="${2}" +source_dir="$(cd "${1}" && pwd)" +jni_build_dir="$(cd "${2}" && pwd)" dist_dir="${3}" +mkdir -p "${dist_dir}" +dist_dir="$(cd "${dist_dir}" && pwd)" -export ARROW_TEST_DATA="${arrow_dir}/testing/data" - -pushd "${arrow_java_dir}" - -# Ensure that there is no old jar -# inside the maven repository +# Ensure that there is no old artifacts inside the maven repository maven_repo=~/.m2/repository/org/apache/arrow if [ -d "$maven_repo" ]; then find "$maven_repo" \ @@ -37,7 +34,7 @@ if [ -d "$maven_repo" ]; then -exec rm -rf {} ";" fi -# generate dummy GPG key for -Papache-release. +# Generate dummy GPG key for -Papache-release. # -Papache-release generates signs (*.asc) of artifacts. # We don't use these signs in our release process. ( @@ -49,15 +46,17 @@ fi ) | gpg --full-generate-key --batch +pushd "${source_dir}" # build the entire project mvn clean \ install \ -Papache-release \ -Parrow-c-data \ -Parrow-jni \ - -Darrow.cpp.build.dir="$dist_dir" \ - -Darrow.c.jni.dist.dir="$dist_dir" \ + -Darrow.cpp.build.dir="${jni_build_dir}" \ + -Darrow.c.jni.dist.dir="${jni_build_dir}" \ --no-transfer-progress +popd # copy all jar, zip and pom files to the distribution folder find ~/.m2/repository/org/apache/arrow \ @@ -69,6 +68,4 @@ find ~/.m2/repository/org/apache/arrow \ -name "*.zip" \ ")" \ -exec echo "{}" ";" \ - -exec cp "{}" "$dist_dir" ";" - -popd + -exec cp "{}" "${dist_dir}" ";" diff --git a/ci/scripts/jni_macos_build.sh b/ci/scripts/jni_macos_build.sh index eeabfd1334..5a693031c3 100755 --- a/ci/scripts/jni_macos_build.sh +++ b/ci/scripts/jni_macos_build.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -21,8 +22,8 @@ set -ex -arrow_java_dir="${1}" -arrow_dir="${2}" +source_dir="$(cd "${1}" && pwd)" +arrow_dir="$(cd "${2}" && pwd)" build_dir="${3}" normalized_arch="$(arch)" case "${normalized_arch}" in @@ -39,6 +40,10 @@ dist_dir="${4}" echo "=== Clear output directories and leftovers ===" # Clear output directories and leftovers rm -rf "${build_dir}" +rm -rf "${dist_dir}" + +mkdir -p "${build_dir}" +build_dir="$(cd "${build_dir}" && pwd)" echo "=== Building Arrow C++ libraries ===" install_dir="${build_dir}/cpp-install" @@ -53,7 +58,7 @@ export ARROW_GANDIVA export ARROW_ORC : "${ARROW_PARQUET:=ON}" : "${ARROW_S3:=ON}" -: "${ARROW_USE_CCACHE:=OFF}" +: "${ARROW_USE_CCACHE:=ON}" : "${CMAKE_BUILD_TYPE:=Release}" : "${CMAKE_UNITY_BUILD:=ON}" @@ -66,10 +71,9 @@ export ARROW_TEST_DATA="${arrow_dir}/testing/data" export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data" export AWS_EC2_METADATA_DISABLED=TRUE -mkdir -p "${build_dir}/cpp" -pushd "${build_dir}/cpp" - cmake \ + -S "${arrow_dir}/cpp" \ + -B "${build_dir}/cpp" \ -DARROW_ACERO="${ARROW_ACERO}" \ -DARROW_BUILD_SHARED=OFF \ -DARROW_BUILD_TESTS="${ARROW_BUILD_TESTS}" \ @@ -84,6 +88,7 @@ cmake \ -DARROW_PARQUET="${ARROW_PARQUET}" \ -DARROW_S3="${ARROW_S3}" \ -DARROW_USE_CCACHE="${ARROW_USE_CCACHE}" \ + -DAWSSDK_SOURCE=BUNDLED \ -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" \ -DCMAKE_INSTALL_PREFIX="${install_dir}" \ -DCMAKE_UNITY_BUILD="${CMAKE_UNITY_BUILD}" \ @@ -92,11 +97,10 @@ cmake \ -DPARQUET_BUILD_EXECUTABLES=OFF \ -DPARQUET_REQUIRE_ENCRYPTION=OFF \ -Dre2_SOURCE=BUNDLED \ - -GNinja \ - "${arrow_dir}/cpp" -cmake --build . --target install + -GNinja +cmake --build "${build_dir}/cpp" --target install -if [ "${ARROW_BUILD_TESTS}" == "ON" ]; then +if [ "${ARROW_RUN_TESTS:-}" == "ON" ]; then # MinIO is required exclude_tests="arrow-s3fs-test" # unstable @@ -107,14 +111,13 @@ if [ "${ARROW_BUILD_TESTS}" == "ON" ]; then --label-regex unittest \ --output-on-failure \ --parallel "$(sysctl -n hw.ncpu)" \ + --test-dir "${build_dir}/cpp" \ --timeout 300 fi -popd - export JAVA_JNI_CMAKE_ARGS="-DProtobuf_ROOT=${build_dir}/cpp/protobuf_ep-install" -"${arrow_java_dir}/ci/scripts/jni_build.sh" \ - "${arrow_java_dir}" \ +"${source_dir}/ci/scripts/jni_build.sh" \ + "${source_dir}" \ "${install_dir}" \ "${build_dir}" \ "${dist_dir}" diff --git a/ci/scripts/jni_manylinux_build.sh b/ci/scripts/jni_manylinux_build.sh index 2551d67239..0251c71c65 100755 --- a/ci/scripts/jni_manylinux_build.sh +++ b/ci/scripts/jni_manylinux_build.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -21,8 +22,8 @@ set -exo pipefail -arrow_java_dir="${1}" -arrow_dir="${2}" +source_dir="$(cd "${1}" && pwd)" +arrow_dir="$(cd "${2}" && pwd)" build_dir="${3}" normalized_arch="$(arch)" case "${normalized_arch}" in @@ -41,12 +42,15 @@ echo "=== Clear output directories and leftovers ===" rm -rf "${build_dir}" rm -rf "${dist_dir}" +mkdir -p "${build_dir}" +build_dir="$(cd "${build_dir}" && pwd)" + echo "=== Building Arrow C++ libraries ===" devtoolset_version="$(rpm -qa "devtoolset-*-gcc" --queryformat '%{VERSION}' | grep -o "^[0-9]*")" devtoolset_include_cpp="/opt/rh/devtoolset-${devtoolset_version}/root/usr/include/c++/${devtoolset_version}" : "${ARROW_ACERO:=ON}" export ARROW_ACERO -: "${ARROW_BUILD_TESTS:=OFF}" +: "${ARROW_BUILD_TESTS:=ON}" : "${ARROW_DATASET:=ON}" export ARROW_DATASET : "${ARROW_GANDIVA:=ON}" @@ -59,7 +63,7 @@ export ARROW_GANDIVA export ARROW_ORC : "${ARROW_PARQUET:=ON}" : "${ARROW_S3:=ON}" -: "${ARROW_USE_CCACHE:=OFF}" +: "${ARROW_USE_CCACHE:=ON}" : "${CMAKE_BUILD_TYPE:=release}" : "${CMAKE_UNITY_BUILD:=ON}" : "${VCPKG_ROOT:=/opt/vcpkg}" @@ -77,9 +81,11 @@ export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data" export AWS_EC2_METADATA_DISABLED=TRUE mkdir -p "${build_dir}/cpp" -pushd "${build_dir}/cpp" +install_dir="${build_dir}/cpp-install" cmake \ + -S "${arrow_dir}/cpp" \ + -B "${build_dir}/cpp" \ -DARROW_ACERO="${ARROW_ACERO}" \ -DARROW_BUILD_SHARED=OFF \ -DARROW_BUILD_TESTS="${ARROW_BUILD_TESTS}" \ @@ -100,7 +106,7 @@ cmake \ -DARROW_S3="${ARROW_S3}" \ -DARROW_USE_CCACHE="${ARROW_USE_CCACHE}" \ -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" \ - -DCMAKE_INSTALL_PREFIX="${ARROW_HOME}" \ + -DCMAKE_INSTALL_PREFIX="${install_dir}" \ -DCMAKE_UNITY_BUILD="${CMAKE_UNITY_BUILD}" \ -DGTest_SOURCE=BUNDLED \ -DORC_SOURCE=BUNDLED \ @@ -110,11 +116,11 @@ cmake \ -DPARQUET_REQUIRE_ENCRYPTION=OFF \ -DVCPKG_MANIFEST_MODE=OFF \ -DVCPKG_TARGET_TRIPLET="${VCPKG_TARGET_TRIPLET}" \ - -GNinja \ - "${arrow_dir}/cpp" -ninja install + -GNinja +cmake --build "${build_dir}/cpp" +cmake --install "${build_dir}/cpp" -if [ "${ARROW_BUILD_TESTS}" = "ON" ]; then +if [ "${ARROW_RUN_TESTS:-OFF}" = "ON" ]; then # MinIO is required exclude_tests="arrow-s3fs-test" case $(arch) in @@ -138,17 +144,16 @@ if [ "${ARROW_BUILD_TESTS}" = "ON" ]; then --label-regex unittest \ --output-on-failure \ --parallel "$(nproc)" \ + --test-dir "${build_dir}/cpp" \ --timeout 300 fi -popd - JAVA_JNI_CMAKE_ARGS="-DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" JAVA_JNI_CMAKE_ARGS="${JAVA_JNI_CMAKE_ARGS} -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET}" export JAVA_JNI_CMAKE_ARGS -"${arrow_java_dir}/ci/scripts/jni_build.sh" \ - "${arrow_java_dir}" \ - "${ARROW_HOME}" \ +"${source_dir}/ci/scripts/jni_build.sh" \ + "${source_dir}" \ + "${install_dir}" \ "${build_dir}" \ "${dist_dir}" diff --git a/ci/scripts/test.sh b/ci/scripts/test.sh index 6449bbc2c5..f65f630182 100755 --- a/ci/scripts/test.sh +++ b/ci/scripts/test.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/docker-compose.yml b/docker-compose.yml index c1d270287d..5decbfc86c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,6 +26,8 @@ # $ ARCH=arm64v8 docker compose run java volumes: + ccache-cache: + name: ccache-cache maven-cache: name: maven-cache @@ -101,9 +103,11 @@ services: volumes: - .:/arrow-java:delegated - ${ARROW_REPO_ROOT}:/arrow:delegated + - ${DOCKER_VOLUME_PREFIX}ccache:/ccache:delegated - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated environment: ARROW_JAVA_CDATA: "ON" + CCACHE_DIR: "/ccache" command: ["git config --global --add safe.directory /arrow-java && \ - /arrow-java/ci/scripts/jni_manylinux_build.sh /arrow-java /arrow /build /arrow-java/dist"] + /arrow-java/ci/scripts/jni_manylinux_build.sh /arrow-java /arrow /build /arrow-java/jni"] From a523df7c07169f5d25c52e8f5b0abe3afaf5cb11 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 16 Jan 2025 12:10:01 +0900 Subject: [PATCH 036/408] GH-522: [Release] Add support for signing .jar (#524) Fixes GH-522. --- .gitignore | 1 + ci/scripts/jni_full_build.sh | 8 +++++++- dev/release/release_rc.sh | 27 +++++++++++++++++++-------- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 8171952dad..8c7bd135e1 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ .settings/ /*-build/ /.mvn/.develocity/ +/apache-arrow-java-* /apache-arrow-java.tar.gz /build/ /dev/release/apache-rat-0.16.1.jar diff --git a/ci/scripts/jni_full_build.sh b/ci/scripts/jni_full_build.sh index 15cf72f21b..49cbdc138f 100755 --- a/ci/scripts/jni_full_build.sh +++ b/ci/scripts/jni_full_build.sh @@ -17,11 +17,12 @@ # specific language governing permissions and limitations # under the License. -set -e +set -eu source_dir="$(cd "${1}" && pwd)" jni_build_dir="$(cd "${2}" && pwd)" dist_dir="${3}" +rm -rf "${dist_dir}" mkdir -p "${dist_dir}" dist_dir="$(cd "${dist_dir}" && pwd)" @@ -69,3 +70,8 @@ find ~/.m2/repository/org/apache/arrow \ ")" \ -exec echo "{}" ";" \ -exec cp "{}" "${dist_dir}" ";" + +for artifact in "${dist_dir}"/*; do + sha256sum "${artifact}" >"${artifact}.sha256" + sha512sum "${artifact}" >"${artifact}.sha512" +done diff --git a/dev/release/release_rc.sh b/dev/release/release_rc.sh index c8f1024a3f..de053578e0 100755 --- a/dev/release/release_rc.sh +++ b/dev/release/release_rc.sh @@ -80,8 +80,8 @@ fi rc_hash="$(git rev-list --max-count=1 "${rc_tag}")" -id="apache-arrow-java-${version}" -tar_gz="${id}.tar.gz" +artifacts_dir="apache-arrow-java-${version}-rc${rc}" +signed_artifacts_dir="${artifacts_dir}-signed" if [ "${RELEASE_SIGN}" -gt 0 ]; then git_origin_url="$(git remote get-url origin)" @@ -105,15 +105,26 @@ if [ "${RELEASE_SIGN}" -gt 0 ]; then echo "Found GitHub Actions workflow with ID: ${run_id}" gh run watch --repo "${repository}" --exit-status "${run_id}" - echo "Downloading .tar.gz from GitHub Releases" + echo "Downloading artifacts from GitHub Releases" gh release download "${rc_tag}" \ - --dir . \ - --pattern "${tar_gz}" \ + --dir "${artifacts_dir}" \ --repo "${repository}" \ --skip-existing - echo "Signing tar.gz and creating checksums" - gpg --armor --output "${tar_gz}.asc" --detach-sig "${tar_gz}" + echo "Signing artifacts" + rm -rf "${signed_artifacts_dir}" + mkdir -p "${signed_artifacts_dir}" + for artifact in "${artifacts_dir}"/*; do + case "${artifact}" in + *.asc | *.sha256 | *.sha512) + continue + ;; + esac + gpg --armor \ + --detach-sig \ + --output "${signed_artifacts_dir}/$(basename "${artifact}").asc" \ + "${artifact}" + done fi if [ "${RELEASE_UPLOAD}" -gt 0 ]; then @@ -121,7 +132,7 @@ if [ "${RELEASE_UPLOAD}" -gt 0 ]; then gh release upload "${rc_tag}" \ --clobber \ --repo "${repository}" \ - "${tar_gz}.asc" + "${signed_artifacts_dir}"/*.asc fi echo "Draft email for dev@arrow.apache.org mailing list" From 37988c5d9786c9e98a48ff749d1fc561850fb752 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 16 Jan 2025 12:30:29 +0900 Subject: [PATCH 037/408] GH-525: [Release] Use released Apache Arrow C++ for RC CI (#526) Apache Arrow 19.0.0 has been released. --- .github/workflows/rc.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 666767ff71..60a8a61d77 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -162,10 +162,9 @@ jobs: - name: Extract Download the latest Apache Arrow C++ run: | tar -xf apache-arrow-java-*.tar.gz --strip-components=1 - # We need 19.0.0 for latest Boost support - - name: Download the latest RC Apache Arrow C++ + - name: Download the latest Apache Arrow C++ run: | - ci/scripts/download_cpp.sh latest-rc + ci/scripts/download_cpp.sh - name: Checkout apache/arrow-testing uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: From aee890f967e951e82ae95fbd43eb6aed4e0d7483 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 17 Jan 2025 14:51:37 +0900 Subject: [PATCH 038/408] GH-527: [Release] Add support for .jar for Windows (#528) Fixes GH-527. It seems that arrow-testing with ARROW_BUILD_STATIC=ON and GTest_SOURCE=BUNDLED is broken. We'll enable JNI test after we fix it in apache/arrow. --- .github/workflows/rc.yml | 93 +++++++++++++++++--- .pre-commit-config.yaml | 2 + ci/scripts/download_cpp.sh | 2 +- ci/scripts/jni_build.sh | 17 +++- ci/scripts/jni_macos_build.sh | 36 +++++--- ci/scripts/jni_manylinux_build.sh | 52 ++++++----- ci/scripts/jni_windows_build.sh | 139 ++++++++++++++++++++++++++++++ ci/scripts/util_log.sh | 28 ++++++ 8 files changed, 321 insertions(+), 48 deletions(-) create mode 100755 ci/scripts/jni_windows_build.sh create mode 100644 ci/scripts/util_log.sh diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 60a8a61d77..4c206625c2 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -76,7 +76,7 @@ jobs: name: release-source path: | apache-arrow-java-* - jni-ubuntu: + jni-linux: name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }} runs-on: ${{ matrix.platform.runs_on }} needs: @@ -100,7 +100,7 @@ jobs: uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: name: release-source - - name: Extract Download the latest Apache Arrow C++ + - name: Extract source archive run: | tar -xf apache-arrow-java-*.tar.gz --strip-components=1 - name: Download the latest Apache Arrow C++ @@ -127,7 +127,7 @@ jobs: path: .docker key: jni-linux-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} restore-keys: jni-linux-${{ matrix.platform.arch }}- - - name: Build C++ libraries + - name: Build run: | docker compose run vcpkg-jni - name: Push Docker image @@ -159,7 +159,7 @@ jobs: uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: name: release-source - - name: Extract Download the latest Apache Arrow C++ + - name: Extract source archive run: | tar -xf apache-arrow-java-*.tar.gz --strip-components=1 - name: Download the latest Apache Arrow C++ @@ -176,7 +176,7 @@ jobs: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: cache: 'pip' python-version: 3.12 @@ -246,7 +246,7 @@ jobs: path: ccache key: jni-macos-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} restore-keys: jni-macos-${{ matrix.platform.arch }}- - - name: Build C++ libraries + - name: Build run: | set -e # make brew Java available to CMake @@ -259,12 +259,83 @@ jobs: with: name: jni-macos-${{ matrix.platform.arch }} path: jni-macos-${{ matrix.platform.arch }}.tar.gz + jni-windows: + name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }} + runs-on: ${{ matrix.platform.runs_on }} + needs: + - source + strategy: + fail-fast: false + matrix: + platform: + - runs_on: windows-2019 + arch: "x86_64" + steps: + - name: Download source archive + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: release-source + - name: Extract source archive + shell: bash + run: | + tar -xf apache-arrow-java-*.tar.gz --strip-components=1 + - name: Download the latest Apache Arrow C++ + shell: bash + run: | + ci/scripts/download_cpp.sh + - name: Set up Java + uses: actions/setup-java@7a6d8a8234af8eb26422e24e3006232cccaa061b # v4.6.0 + with: + java-version: '11' + distribution: 'temurin' + - name: Download Timezone Database + shell: bash + run: | + arrow/ci/scripts/download_tz_database.sh + - name: Install ccache + shell: bash + run: | + env | sort + version=4.10.2 + base_name="ccache-${version}-windows-x86_64" + url="https://github.com/ccache/ccache/releases/download/v${version}/${base_name}.zip" + curl --fail --location --remote-name "${url}" + unzip "${base_name}.zip" + chmod +x "${base_name}/ccache.exe" + mv "${base_name}/ccache.exe" /usr/bin/ + rm -rf "${base_name}"{,.zip} + - name: Prepare ccache + shell: bash + run: | + echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} + - name: Cache ccache + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + with: + path: ccache + key: jni-windows-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} + restore-keys: jni-windows-${{ matrix.platform.arch }}- + - name: Build + shell: cmd + run: | + call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + REM For ORC + set TZDIR=/c/msys64/usr/share/zoneinfo + bash -c "ci/scripts/jni_windows_build.sh . arrow build jni" + - name: Compress into single artifact to keep directory structure + shell: bash + run: tar -cvzf jni-windows-${{ matrix.platform.arch }}.tar.gz jni/ + - name: Upload artifacts + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: jni-windows-${{ matrix.platform.arch }} + path: jni-windows-${{ matrix.platform.arch }}.tar.gz binaries: name: Binaries runs-on: ubuntu-latest needs: - - jni-ubuntu + - jni-linux - jni-macos + - jni-windows steps: - name: Download artifacts uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 @@ -278,7 +349,7 @@ jobs: # tar -xvzf jni-linux-aarch_64.tar.gz tar -xvzf jni-macos-x86_64.tar.gz tar -xvzf jni-macos-aarch_64.tar.gz - # tar -xvzf jni-windows.tar.gz + tar -xvzf jni-windows-x86_64.tar.gz - name: Test that shared libraries exist run: | set -x @@ -303,9 +374,9 @@ jobs: test -f jni/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib test -f jni/gandiva_jni/aarch_64/libgandiva_jni.dylib - # test -f jni/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll - # test -f jni/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll - # test -f jni/arrow_orc_jni/x86_64/arrow_orc_jni.dll + test -f jni/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll + test -f jni/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll + test -f jni/arrow_orc_jni/x86_64/arrow_orc_jni.dll - name: Checkout apache/arrow-testing uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0a762414fd..ca3cacec3b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,6 +47,8 @@ repos: rev: v0.10.0 hooks: - id: shellcheck + args: + - "--external-sources" - repo: https://github.com/scop/pre-commit-shfmt rev: v3.9.0-1 hooks: diff --git a/ci/scripts/download_cpp.sh b/ci/scripts/download_cpp.sh index 4d801a47f9..3721f62ad5 100755 --- a/ci/scripts/download_cpp.sh +++ b/ci/scripts/download_cpp.sh @@ -47,6 +47,6 @@ fi if [ -z "${url}" ]; then url="https://www.apache.org/dyn/closer.lua?action=download&filename=arrow/arrow-${version}/apache-arrow-${version}.tar.gz" fi -wget --output-document "apache-arrow-${version}.tar.gz" "${url}" +curl --location --output "apache-arrow-${version}.tar.gz" "${url}" tar xf "apache-arrow-${version}.tar.gz" mv "apache-arrow-${version}" arrow diff --git a/ci/scripts/jni_build.sh b/ci/scripts/jni_build.sh index a77b0da02a..aec6fc325c 100755 --- a/ci/scripts/jni_build.sh +++ b/ci/scripts/jni_build.sh @@ -17,20 +17,25 @@ # specific language governing permissions and limitations # under the License. -set -euxo pipefail +set -euo pipefail +# shellcheck source=ci/scripts/util_log.sh +. "$(dirname "${0}")/util_log.sh" + +github_actions_group_begin "Prepare arguments" source_dir=${1} arrow_install_dir=${2} build_dir=${3}/java_jni # The directory where the final binaries will be stored when scripts finish dist_dir=${4} prefix_dir="${build_dir}/java-jni" +github_actions_group_end -echo "=== Clear output directories and leftovers ===" -# Clear output directories and leftovers +github_actions_group_begin "Clear output directories and leftovers" rm -rf "${build_dir}" +github_actions_group_end -echo "=== Building Arrow Java C Data Interface native library ===" +github_actions_group_begin "Building Arrow Java C Data Interface native library" case "$(uname)" in Linux) @@ -71,6 +76,9 @@ if [ "${ARROW_JAVA_BUILD_TESTS}" = "ON" ]; then fi cmake --build "${build_dir}" --target install +github_actions_group_end + +github_actions_group_begin "Copying artifacts" mkdir -p "${dist_dir}" # For Windows. *.dll are installed into bin/ on Windows. if [ -d "${prefix_dir}/bin" ]; then @@ -78,3 +86,4 @@ if [ -d "${prefix_dir}/bin" ]; then else mv "${prefix_dir}"/lib/* "${dist_dir}"/ fi +github_actions_group_end diff --git a/ci/scripts/jni_macos_build.sh b/ci/scripts/jni_macos_build.sh index 5a693031c3..1ad76266ee 100755 --- a/ci/scripts/jni_macos_build.sh +++ b/ci/scripts/jni_macos_build.sh @@ -20,8 +20,12 @@ # This script is like java_jni_build.sh, but is meant for release artifacts # and hardcodes assumptions about the environment it is being run in. -set -ex +set -euo pipefail +# shellcheck source=ci/scripts/util_log.sh +. "$(dirname "${0}")/util_log.sh" + +github_actions_group_begin "Prepare arguments" source_dir="$(cd "${1}" && pwd)" arrow_dir="$(cd "${2}" && pwd)" build_dir="${3}" @@ -36,20 +40,29 @@ i386) esac # The directory where the final binaries will be stored when scripts finish dist_dir="${4}" +github_actions_group_end -echo "=== Clear output directories and leftovers ===" -# Clear output directories and leftovers +github_actions_group_begin "Clear output directories and leftovers" rm -rf "${build_dir}" rm -rf "${dist_dir}" mkdir -p "${build_dir}" build_dir="$(cd "${build_dir}" && pwd)" +github_actions_group_end + +: "${ARROW_USE_CCACHE:=ON}" +if [ "${ARROW_USE_CCACHE}" == "ON" ]; then + github_actions_group_begin "ccache statistics before build" + ccache -sv 2>/dev/null || ccache -s + github_actions_group_end +fi -echo "=== Building Arrow C++ libraries ===" +github_actions_group_begin "Building Arrow C++ libraries" install_dir="${build_dir}/cpp-install" : "${ARROW_ACERO:=ON}" export ARROW_ACERO : "${ARROW_BUILD_TESTS:=ON}" +export ARROW_BUILD_TESTS : "${ARROW_DATASET:=ON}" export ARROW_DATASET : "${ARROW_GANDIVA:=ON}" @@ -58,15 +71,9 @@ export ARROW_GANDIVA export ARROW_ORC : "${ARROW_PARQUET:=ON}" : "${ARROW_S3:=ON}" -: "${ARROW_USE_CCACHE:=ON}" : "${CMAKE_BUILD_TYPE:=Release}" : "${CMAKE_UNITY_BUILD:=ON}" -if [ "${ARROW_USE_CCACHE}" == "ON" ]; then - echo "=== ccache statistics before build ===" - ccache -sv 2>/dev/null || ccache -s -fi - export ARROW_TEST_DATA="${arrow_dir}/testing/data" export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data" export AWS_EC2_METADATA_DISABLED=TRUE @@ -99,8 +106,10 @@ cmake \ -Dre2_SOURCE=BUNDLED \ -GNinja cmake --build "${build_dir}/cpp" --target install +github_actions_group_end if [ "${ARROW_RUN_TESTS:-}" == "ON" ]; then + github_actions_group_begin "Running Arrow C++ libraries tests" # MinIO is required exclude_tests="arrow-s3fs-test" # unstable @@ -113,6 +122,7 @@ if [ "${ARROW_RUN_TESTS:-}" == "ON" ]; then --parallel "$(sysctl -n hw.ncpu)" \ --test-dir "${build_dir}/cpp" \ --timeout 300 + github_actions_group_end fi export JAVA_JNI_CMAKE_ARGS="-DProtobuf_ROOT=${build_dir}/cpp/protobuf_ep-install" @@ -123,11 +133,12 @@ export JAVA_JNI_CMAKE_ARGS="-DProtobuf_ROOT=${build_dir}/cpp/protobuf_ep-install "${dist_dir}" if [ "${ARROW_USE_CCACHE}" == "ON" ]; then - echo "=== ccache statistics after build ===" + github_actions_group_begin "ccache statistics after build" ccache -sv 2>/dev/null || ccache -s + github_actions_group_end fi -echo "=== Checking shared dependencies for libraries ===" +github_actions_group_begin "Checking shared dependencies for libraries" pushd "${dist_dir}" archery linking check-dependencies \ --allow CoreFoundation \ @@ -147,3 +158,4 @@ archery linking check-dependencies \ "arrow_orc_jni/${normalized_arch}/libarrow_orc_jni.dylib" \ "gandiva_jni/${normalized_arch}/libgandiva_jni.dylib" popd +github_actions_group_end diff --git a/ci/scripts/jni_manylinux_build.sh b/ci/scripts/jni_manylinux_build.sh index 0251c71c65..148d2e02f6 100755 --- a/ci/scripts/jni_manylinux_build.sh +++ b/ci/scripts/jni_manylinux_build.sh @@ -20,37 +20,45 @@ # This script is like java_jni_build.sh, but is meant for release artifacts # and hardcodes assumptions about the environment it is being run in. -set -exo pipefail +set -euo pipefail +# shellcheck source=ci/scripts/util_log.sh +. "$(dirname "${0}")/util_log.sh" + +github_actions_group_begin "Prepare arguments" source_dir="$(cd "${1}" && pwd)" arrow_dir="$(cd "${2}" && pwd)" build_dir="${3}" -normalized_arch="$(arch)" -case "${normalized_arch}" in -aarch64) - normalized_arch=aarch_64 - ;; -esac # The directory where the final binaries will be stored when scripts finish dist_dir="${4}" +github_actions_group_end -echo "=== Install Archery ===" +github_actions_group_begin "Install Archery" pip install -e "${arrow_dir}/dev/archery[all]" +github_actions_group_end -echo "=== Clear output directories and leftovers ===" -# Clear output directories and leftovers +github_actions_group_begin "Clear output directories and leftovers" rm -rf "${build_dir}" rm -rf "${dist_dir}" mkdir -p "${build_dir}" build_dir="$(cd "${build_dir}" && pwd)" +github_actions_group_end + +: "${ARROW_USE_CCACHE:=ON}" +if [ "${ARROW_USE_CCACHE}" == "ON" ]; then + github_actions_group_begin "ccache statistics before build" + ccache -sv 2>/dev/null || ccache -s + github_actions_group_end +fi -echo "=== Building Arrow C++ libraries ===" +github_actions_group_begin "Building Arrow C++ libraries" devtoolset_version="$(rpm -qa "devtoolset-*-gcc" --queryformat '%{VERSION}' | grep -o "^[0-9]*")" devtoolset_include_cpp="/opt/rh/devtoolset-${devtoolset_version}/root/usr/include/c++/${devtoolset_version}" : "${ARROW_ACERO:=ON}" export ARROW_ACERO : "${ARROW_BUILD_TESTS:=ON}" +export ARROW_BUILD_TESTS : "${ARROW_DATASET:=ON}" export ARROW_DATASET : "${ARROW_GANDIVA:=ON}" @@ -63,7 +71,6 @@ export ARROW_GANDIVA export ARROW_ORC : "${ARROW_PARQUET:=ON}" : "${ARROW_S3:=ON}" -: "${ARROW_USE_CCACHE:=ON}" : "${CMAKE_BUILD_TYPE:=release}" : "${CMAKE_UNITY_BUILD:=ON}" : "${VCPKG_ROOT:=/opt/vcpkg}" @@ -71,16 +78,10 @@ export ARROW_ORC : "${VCPKG_TARGET_TRIPLET:=${VCPKG_DEFAULT_TRIPLET:-x64-linux-static-${CMAKE_BUILD_TYPE}}}" : "${GANDIVA_CXX_FLAGS:=-isystem;${devtoolset_include_cpp};-isystem;${devtoolset_include_cpp}/x86_64-redhat-linux;-lpthread}" -if [ "${ARROW_USE_CCACHE}" == "ON" ]; then - echo "=== ccache statistics before build ===" - ccache -sv 2>/dev/null || ccache -s -fi - export ARROW_TEST_DATA="${arrow_dir}/testing/data" export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data" export AWS_EC2_METADATA_DISABLED=TRUE -mkdir -p "${build_dir}/cpp" install_dir="${build_dir}/cpp-install" cmake \ @@ -119,8 +120,10 @@ cmake \ -GNinja cmake --build "${build_dir}/cpp" cmake --install "${build_dir}/cpp" +github_actions_group_end if [ "${ARROW_RUN_TESTS:-OFF}" = "ON" ]; then + github_actions_group_begin "Running Arrow C++ libraries tests" # MinIO is required exclude_tests="arrow-s3fs-test" case $(arch) in @@ -146,6 +149,7 @@ if [ "${ARROW_RUN_TESTS:-OFF}" = "ON" ]; then --parallel "$(nproc)" \ --test-dir "${build_dir}/cpp" \ --timeout 300 + github_actions_group_end fi JAVA_JNI_CMAKE_ARGS="-DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" @@ -158,11 +162,18 @@ export JAVA_JNI_CMAKE_ARGS "${dist_dir}" if [ "${ARROW_USE_CCACHE}" == "ON" ]; then - echo "=== ccache statistics after build ===" + github_actions_group_begin "ccache statistics after build" ccache -sv 2>/dev/null || ccache -s + github_actions_group_end fi -echo "=== Checking shared dependencies for libraries ===" +github_actions_group_begin "Checking shared dependencies for libraries" +normalized_arch="$(arch)" +case "${normalized_arch}" in +aarch64) + normalized_arch=aarch_64 + ;; +esac pushd "${dist_dir}" archery linking check-dependencies \ --allow ld-linux-aarch64 \ @@ -181,3 +192,4 @@ archery linking check-dependencies \ arrow_orc_jni/"${normalized_arch}"/libarrow_orc_jni.so \ gandiva_jni/"${normalized_arch}"/libgandiva_jni.so popd +github_actions_group_end diff --git a/ci/scripts/jni_windows_build.sh b/ci/scripts/jni_windows_build.sh new file mode 100755 index 0000000000..d01ef45f5a --- /dev/null +++ b/ci/scripts/jni_windows_build.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +# shellcheck source=ci/scripts/util_log.sh +. "$(dirname "${0}")/util_log.sh" + +github_actions_group_begin "Prepare arguments" +source_dir="$(cd "${1}" && pwd)" +arrow_dir="$(cd "${2}" && pwd)" +build_dir="${3}" +# The directory where the final binaries will be stored when scripts finish +dist_dir="${4}" +github_actions_group_end + +github_actions_group_begin "Clear output directories and leftovers" +rm -rf "${build_dir}" + +mkdir -p "${build_dir}" +build_dir="$(cd "${build_dir}" && pwd)" +github_actions_group_end + +: "${ARROW_USE_CCACHE:=ON}" +if [ "${ARROW_USE_CCACHE}" == "ON" ]; then + github_actions_group_begin "ccache statistics before build" + ccache -sv 2>/dev/null || ccache -s + github_actions_group_end +fi + +github_actions_group_begin "Building Arrow C++ libraries" +install_dir="${build_dir}/cpp-install" +: "${ARROW_ACERO:=ON}" +export ARROW_ACERO +: "${ARROW_BUILD_TESTS:=OFF}" # TODO: ON +export ARROW_BUILD_TESTS +: "${ARROW_DATASET:=ON}" +export ARROW_DATASET +: "${ARROW_ORC:=ON}" +export ARROW_ORC +: "${ARROW_PARQUET:=ON}" +: "${ARROW_S3:=ON}" +: "${CMAKE_BUILD_TYPE:=release}" +: "${CMAKE_UNITY_BUILD:=ON}" + +export ARROW_TEST_DATA="${arrow_dir}/testing/data" +export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data" +export AWS_EC2_METADATA_DISABLED=TRUE + +cmake \ + -S "${arrow_dir}/cpp" \ + -B "${build_dir}/cpp" \ + -DARROW_ACERO="${ARROW_ACERO}" \ + -DARROW_BUILD_SHARED=OFF \ + -DARROW_BUILD_TESTS=ON \ + -DARROW_CSV="${ARROW_DATASET}" \ + -DARROW_DATASET="${ARROW_DATASET}" \ + -DARROW_SUBSTRAIT="${ARROW_DATASET}" \ + -DARROW_DEPENDENCY_USE_SHARED=OFF \ + -DARROW_ORC="${ARROW_ORC}" \ + -DARROW_PARQUET="${ARROW_PARQUET}" \ + -DARROW_S3="${ARROW_S3}" \ + -DARROW_USE_CCACHE="${ARROW_USE_CCACHE}" \ + -DARROW_WITH_BROTLI=ON \ + -DARROW_WITH_LZ4=ON \ + -DARROW_WITH_SNAPPY=ON \ + -DARROW_WITH_ZSTD=ON \ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" \ + -DCMAKE_INSTALL_PREFIX="${install_dir}" \ + -DCMAKE_UNITY_BUILD="${CMAKE_UNITY_BUILD}" \ + -GNinja +cmake --build "${build_dir}/cpp" +cmake --install "${build_dir}/cpp" +github_actions_group_end + +if [ "${ARROW_RUN_TESTS:-OFF}" = "ON" ]; then + github_actions_group_begin "Running Arrow C++ libraries tests" + # MinIO is required + exclude_tests="arrow-s3fs-test" + # unstable + exclude_tests="${exclude_tests}|arrow-compute-hash-join-node-test" + exclude_tests="${exclude_tests}|arrow-dataset-scanner-test" + # strptime + exclude_tests="${exclude_tests}|arrow-utility-test" + ctest \ + --exclude-regex "${exclude_tests}" \ + --label-regex unittest \ + --output-on-failure \ + --parallel "$(nproc)" \ + --timeout 300 + github_actions_group_end +fi + +"${source_dir}/ci/scripts/jni_build.sh" \ + "${source_dir}" \ + "${install_dir}" \ + "${build_dir}" \ + "${dist_dir}" + +if [ "${ARROW_USE_CCACHE}" == "ON" ]; then + github_actions_group_begin "ccache statistics after build" + ccache -sv 2>/dev/null || ccache -s + github_actions_group_end +fi + +github_actions_group_begin "Checking shared dependencies for libraries" +normalized_arch="$(arch)" +case "${normalized_arch}" in +aarch64) + normalized_arch=aarch_64 + ;; +esac +pushd "${dist_dir}" +# TODO +# archery linking check-dependencies \ +# --allow libm \ +# --allow librt \ +# --allow libz \ +# arrow_cdata_jni/"${normalized_arch}"/libarrow_cdata_jni.dll \ +# arrow_dataset_jni/"${normalized_arch}"/libarrow_dataset_jni.dll \ +# arrow_orc_jni/"${normalized_arch}"/libarrow_orc_jni.dll +popd +github_actions_group_end diff --git a/ci/scripts/util_log.sh b/ci/scripts/util_log.sh new file mode 100644 index 0000000000..c8ee48bbb2 --- /dev/null +++ b/ci/scripts/util_log.sh @@ -0,0 +1,28 @@ +# shellcheck shell=bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +github_actions_group_begin() { + echo "::group::$1" + set -x +} + +github_actions_group_end() { + set +x + echo "::endgroup::" +} From 34f4ffe81ed8b933be54ef0a91802ac3a377d0dd Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 17 Jan 2025 20:21:54 +0900 Subject: [PATCH 039/408] GH-529: [Release] Add support for uploading binary artifacts to repository.apache.org (#530) Fixes GH-529. --- .gitignore | 1 + dev/release/.env.example | 27 ++++++++++++ dev/release/README.md | 52 ++++++++++++++-------- dev/release/release_rc.sh | 91 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 151 insertions(+), 20 deletions(-) create mode 100644 dev/release/.env.example diff --git a/.gitignore b/.gitignore index 8c7bd135e1..205be77eaf 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ /apache-arrow-java-* /apache-arrow-java.tar.gz /build/ +/dev/release/.env /dev/release/apache-rat-0.16.1.jar /dev/release/filtered_rat.txt /dev/release/rat.xml diff --git a/dev/release/.env.example b/dev/release/.env.example new file mode 100644 index 0000000000..cc7fd58bfc --- /dev/null +++ b/dev/release/.env.example @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# The GPG key ID to sign artifacts. The GPG key ID must be registered +# to both of the followings: +# +# * https://dist.apache.org/repos/dist/dev/arrow/KEYS +# * https://dist.apache.org/repos/dist/release/arrow/KEYS +# +# See these files how to import your GPG key ID to these files. +# +# You must set this. +#GPG_KEY_ID=08D3564B7C6A9CAFBFF6A66791D18FCF079F8007 diff --git a/dev/release/README.md b/dev/release/README.md index eafbb61b8c..0294ebedc9 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -56,30 +56,13 @@ RC without merging the pull request, the script to cut a RC is failed. ### Prepare RC and vote -Run `dev/release/release_rc.sh` on a working copy of -`git@github.com:apache/arrow-java` not your fork: - -```console -$ git clone git@github.com:apache/arrow-java.git -$ cd arrow-java -$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release_rc.sh ${RC} -(Send a vote email to dev@arrow.apache.org. - You can use a draft shown by release_rc.sh for the email.) -``` - -Here is an example to release RC1: - -```console -$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release_rc.sh 1 -``` - -The argument of `release_rc.sh` is the RC number. If RC1 has a -problem, we'll increment the RC number such as RC2, RC3 and so on. +You can use `dev/release/release_rc.sh`. Requirements to run `release_rc.sh`: * You must be an Apache Arrow committer or PMC member * You must prepare your PGP key for signing + * You must configure Maven If you don't have a PGP key, https://infra.apache.org/release-signing.html#generate may be helpful. @@ -102,6 +85,37 @@ $ head KEYS $ svn ci KEYS ``` +Configure Maven to publish artifacts to Apache repositories. You will +need to setup a master password at `~/.m2/settings-security.xml` and +`~/.m2/settings.xml` as specified on [the Apache +guide](https://infra.apache.org/publishing-maven-artifacts.html). It +can be tested with the following command: + +```bash +# You might need to export GPG_TTY=$(tty) to properly prompt for a passphrase +mvn clean install -Papache-release +``` + +Run `dev/release/release_rc.sh` on a working copy of +`git@github.com:apache/arrow-java` not your fork: + +```console +$ git clone git@github.com:apache/arrow-java.git +$ cd arrow-java +$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release_rc.sh ${RC} +(Send a vote email to dev@arrow.apache.org. + You can use a draft shown by release_rc.sh for the email.) +``` + +Here is an example to release RC1: + +```console +$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release_rc.sh 1 +``` + +The argument of `release_rc.sh` is the RC number. If RC1 has a +problem, we'll increment the RC number such as RC2, RC3 and so on. + ### Publish We need to do the followings to publish a new release: diff --git a/dev/release/release_rc.sh b/dev/release/release_rc.sh index de053578e0..dd61f9c9e6 100755 --- a/dev/release/release_rc.sh +++ b/dev/release/release_rc.sh @@ -84,6 +84,13 @@ artifacts_dir="apache-arrow-java-${version}-rc${rc}" signed_artifacts_dir="${artifacts_dir}-signed" if [ "${RELEASE_SIGN}" -gt 0 ]; then + if [ ! -f "${SOURCE_DIR}/.env" ]; then + echo "You must create ${SOURCE_DIR}/.env" + echo "You can use ${SOURCE_DIR}/.env.example as template" + exit 1 + fi + . "${SOURCE_DIR}/.env" + git_origin_url="$(git remote get-url origin)" repository="${git_origin_url#*github.com?}" repository="${repository%.git}" @@ -120,21 +127,103 @@ if [ "${RELEASE_SIGN}" -gt 0 ]; then continue ;; esac - gpg --armor \ + gpg \ + --armor \ --detach-sig \ + --local-user "${GPG_KEY_ID}" \ --output "${signed_artifacts_dir}/$(basename "${artifact}").asc" \ "${artifact}" done fi +# arrow-c-data-18.2.0-sources.jar -> +# jar +extract_type() { + local path="$1" + echo "${path}" | grep -o "[^.]*$" +} + +# arrow-c-data-18.2.0-sources.jar arrow-c-data-18.2.0 -> +# sources +extract_classifier() { + local path="$1" + local base="$2" + basename "${path}" | sed -e "s/^${base}-//g" -e "s/\.[^.]*$//g" +} + if [ "${RELEASE_UPLOAD}" -gt 0 ]; then echo "Uploading signature" gh release upload "${rc_tag}" \ --clobber \ --repo "${repository}" \ "${signed_artifacts_dir}"/*.asc + + echo "Uploading packages" + for pom in "${artifacts_dir}"/*.pom; do + base=$(basename "${pom}" .pom) + files=() + types=() + classifiers=() + args=() + args+=(deploy:deploy-file) + args+=(-Durl=https://repository.apache.org/service/local/staging/deploy/maven2) + args+=(-DrepositoryId=apache.releases.https) + args+=(-DretryFailedDeploymentCount=10) + args+=(-DpomFile="${pom}") + if [ -f "${artifacts_dir}/${base}.jar" ]; then + jar="${artifacts_dir}/${base}.jar" + args+=(-Dfile="${jar}") + files+=("${signed_artifacts_dir}/${base}.jar.asc") + types+=("jar.asc") + classifiers+=("") + else + args+=(-Dfile="${pom}") + fi + files+=("${signed_artifacts_dir}/${base}.pom.asc") + types+=("pom.asc") + classifiers+=("") + if [ "$(echo "${artifacts_dir}/${base}"-*)" != "${artifacts_dir}/${base}-*" ]; then + for other_file in "${artifacts_dir}/${base}"-*; do + type="$(extract_type "${other_file}")" + case "${type}" in + sha256 | sha512) + continue + ;; + esac + classifier=$(extract_classifier "${other_file}" "${base}") + files+=("${other_file}") + types+=("${type}") + classifiers+=("${classifier}") + other_file_base="$(basename "${other_file}")" + files+=("${signed_artifacts_dir}/${other_file_base}.asc") + types+=("${type}.asc") + classifiers+=("${classifier}") + done + fi + args+=(-Dfiles="$( + IFS=, + echo "${files[*]}" + )") + args+=(-Dtypes="$( + IFS=, + echo "${types[*]}" + )") + args+=(-Dclassifiers="$( + IFS=, + echo "${classifiers[*]}" + )") + mvn "${args[@]}" + done + + echo + echo "Success!" + echo "Press the 'Close' button manually by Web interface:" + echo " https://repository.apache.org/#stagingRepositories" + echo "It publishes the artifacts to the staging repository:" + echo " https://repository.apache.org/content/repositories/staging/org/apache/arrow/" fi +echo echo "Draft email for dev@arrow.apache.org mailing list" echo "" echo "---------------------------------------------------------" From 2e4283a7bb75bd5d54e6e18608158ad4c3546abd Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 18 Jan 2025 15:32:06 +0900 Subject: [PATCH 040/408] GH-531: [Release] Add support for .jar for arm64 Linux (#532) Fixes GH-531. --- .github/workflows/rc.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 4c206625c2..71cb885946 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -79,6 +79,7 @@ jobs: jni-linux: name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }} runs-on: ${{ matrix.platform.runs_on }} + timeout-minutes: 120 needs: - source strategy: @@ -88,6 +89,9 @@ jobs: - runs_on: ubuntu-latest arch: "x86_64" archery_arch: "amd64" + - runs_on: ubuntu-24.04-arm + arch: "aarch_64" + archery_arch: "arm64v8" env: # architecture name used for archery build ARCH: ${{ matrix.platform.archery_arch }} @@ -144,6 +148,7 @@ jobs: jni-macos: name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }} runs-on: ${{ matrix.platform.runs_on }} + timeout-minutes: 45 needs: - source strategy: @@ -262,6 +267,7 @@ jobs: jni-windows: name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }} runs-on: ${{ matrix.platform.runs_on }} + timeout-minutes: 45 needs: - source strategy: @@ -346,7 +352,7 @@ jobs: mv artifacts/*/*.tar.gz . tar -xf apache-arrow-java-*.tar.gz --strip-components=1 tar -xvzf jni-linux-x86_64.tar.gz - # tar -xvzf jni-linux-aarch_64.tar.gz + tar -xvzf jni-linux-aarch_64.tar.gz tar -xvzf jni-macos-x86_64.tar.gz tar -xvzf jni-macos-aarch_64.tar.gz tar -xvzf jni-windows-x86_64.tar.gz @@ -359,10 +365,10 @@ jobs: test -f jni/arrow_orc_jni/x86_64/libarrow_orc_jni.so test -f jni/gandiva_jni/x86_64/libgandiva_jni.so - # test -f jni/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.so - # test -f jni/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.so - # test -f jni/arrow_orc_jni/aarch_64/libarrow_orc_jni.so - # test -f jni/gandiva_jni/aarch_64/libgandiva_jni.so + test -f jni/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.so + test -f jni/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.so + test -f jni/arrow_orc_jni/aarch_64/libarrow_orc_jni.so + test -f jni/gandiva_jni/aarch_64/libgandiva_jni.so test -f jni/arrow_cdata_jni/x86_64/libarrow_cdata_jni.dylib test -f jni/arrow_dataset_jni/x86_64/libarrow_dataset_jni.dylib From b32b0c0750462f49030c78753126f76500300364 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 20 Jan 2025 11:39:13 +0900 Subject: [PATCH 041/408] GH-535: [Dev] Use `compose.yaml` not `docker-compose.yml` (#536) Fixes GH-535. Because Docker Compose recommends `compose.yaml`: https://docs.docker.com/compose/intro/compose-application-model/#the-compose-file > The default path for a Compose file is compose.yaml (preferred) or > compose.yml that is placed in the working directory. Compose also > supports docker-compose.yaml and docker-compose.yml for backwards > compatibility of earlier versions. If both files exist, Compose > prefers the canonical compose.yaml. --- .github/workflows/test.yml | 2 +- docker-compose.yml => compose.yaml | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename docker-compose.yml => compose.yaml (100%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7841dd89ca..a9ed7eb68f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -66,7 +66,7 @@ jobs: uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: path: .docker - key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('**/docker-compose.yml', '**/pom.xml') }} + key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('**/compose.yaml', '**/pom.xml') }} restore-keys: maven-${{ matrix.jdk }}-${{ matrix.maven }}- - name: Execute Docker Build env: diff --git a/docker-compose.yml b/compose.yaml similarity index 100% rename from docker-compose.yml rename to compose.yaml From 7580bf5e231ecc8c3db79345c4b2ec704e6c5d6f Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 20 Jan 2025 11:39:29 +0900 Subject: [PATCH 042/408] GH-533: [Release] Add a post release task for publishing staged .jar (#534) Fixes GH-533. This is borrowed from ADBC: https://arrow.apache.org/adbc/current/development/releasing.html --- dev/release/README.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/dev/release/README.md b/dev/release/README.md index 0294ebedc9..f62befba03 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -120,11 +120,12 @@ problem, we'll increment the RC number such as RC2, RC3 and so on. We need to do the followings to publish a new release: - * Publish to apache.org + * Publish the source archive to apache.org + * Publish the binary artifacts to repository.apache.org Run `dev/release/release.sh` on a working copy of -`git@github.com:apache/arrow-java` not your fork to publish to -apache.org: +`git@github.com:apache/arrow-java` not your fork to publish the source +archive to apache.org: ```console $ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release.sh ${VERSION} ${RC} @@ -139,6 +140,15 @@ $ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release.sh 19.0.0 1 Add the release to ASF's report database via [Apache Committee Report Helper](https://reporter.apache.org/addrelease.html?arrow). +You need to do the followings to publish the binary artifacts to +repository.apache.org: + +* Logon to the Apache repository: + https://repository.apache.org/#stagingRepositories +* Select the Arrow staging repository you created for RC: + `orgapachearrow-XXXX` +* Click the `release` button + ### Bump version for new development We should bump version in the main branch for new development after we From b1fc2c09c20d2fa80039cc082e94a3bf3d6307c4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jan 2025 02:39:56 -0500 Subject: [PATCH 043/408] MINOR: Bump dep.junit.jupiter.version from 5.11.3 to 5.11.4 (#541) Bumps `dep.junit.jupiter.version` from 5.11.3 to 5.11.4. Updates `org.junit.jupiter:junit-jupiter-engine` from 5.11.3 to 5.11.4 Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8660ff39b5..cc89746189 100644 --- a/pom.xml +++ b/pom.xml @@ -93,7 +93,7 @@ under the License. ${project.build.directory}/generated-sources 1.9.0 - 5.11.3 + 5.11.4 2.0.16 33.4.0-jre 4.1.115.Final From 36f9b04acebbe578c8188b59dfbbf8978348dd34 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jan 2025 02:40:40 -0500 Subject: [PATCH 044/408] MINOR: Bump org.apache.orc:orc-core from 2.0.3 to 2.1.0 (#540) Bumps org.apache.orc:orc-core from 2.0.3 to 2.1.0. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- adapter/orc/pom.xml | 2 +- dataset/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/adapter/orc/pom.xml b/adapter/orc/pom.xml index 6b15a34a77..b8ff0aa409 100644 --- a/adapter/orc/pom.xml +++ b/adapter/orc/pom.xml @@ -61,7 +61,7 @@ under the License. org.apache.orc orc-core - 2.0.3 + 2.1.0 test diff --git a/dataset/pom.xml b/dataset/pom.xml index 52ad0c056e..ad21c7754a 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -130,7 +130,7 @@ under the License. org.apache.orc orc-core - 2.0.3 + 2.1.0 test From c377e3b86832b80fa4e5c3f2cb02f5c698c94141 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jan 2025 03:20:56 -0500 Subject: [PATCH 045/408] MINOR: Bump io.grpc:grpc-bom from 1.65.0 to 1.69.1 (#539) Bumps [io.grpc:grpc-bom](https://github.com/grpc/grpc-java) from 1.65.0 to 1.69.1. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index cc89746189..b07797252e 100644 --- a/pom.xml +++ b/pom.xml @@ -97,7 +97,7 @@ under the License. 2.0.16 33.4.0-jre 4.1.115.Final - 1.65.0 + 1.69.1 3.25.5 2.18.2 3.4.1 From 1960085a95cbd5992e82de014254798081f9e681 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 20 Jan 2025 20:14:22 +0900 Subject: [PATCH 046/408] GH-537: [Release][Docs] Generate API reference (#538) The followings are out-of-scope: * Generate Sphinx based documents * Publish this to https://arrow.apache.org/java/reference/ in release process * Publish this to https://arrow.apache.org/dev/reference/ nightly --- .github/workflows/rc.yml | 17 ++++++++++++----- .github/workflows/test.yml | 12 ++++++------ ci/scripts/build.sh | 37 +++++++++++++++++++++++------------- ci/scripts/jni_full_build.sh | 37 +++++++++++++++++++++++++++++++----- ci/scripts/test.sh | 6 +++++- compose.yaml | 8 ++++---- 6 files changed, 83 insertions(+), 34 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 71cb885946..3d6120dc24 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -394,17 +394,24 @@ jobs: path: ~/.m2 key: binaries-build-${{ hashFiles('**/*.java', '**/pom.xml') }} restore-keys: binaries-build- - - name: Build bundled JAR - env: - MAVEN_ARGS: >- - --no-transfer-progress + - name: Build bundled JAR and docs run: | ci/scripts/jni_full_build.sh . jni binaries - - name: Upload artifacts + - name: Prepare docs + run: | + mkdir -p docs + cp -a target/site/apidocs docs/reference + tar -cvzf docs.tar.gz docs + - name: Upload binaries uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 with: name: release-binaries path: binaries/* + - name: Upload docs + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: release-docs + path: docs.tar.gz verify: name: Verify needs: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a9ed7eb68f..3ea5d8a4b6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -66,7 +66,7 @@ jobs: uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: path: .docker - key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('**/compose.yaml', '**/pom.xml') }} + key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('compose.yaml', '**/pom.xml', '**/*.java') }} restore-keys: maven-${{ matrix.jdk }}-${{ matrix.maven }}- - name: Execute Docker Build env: @@ -108,12 +108,12 @@ jobs: shell: bash env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - run: ci/scripts/build.sh $(pwd) $(pwd)/build + run: ci/scripts/build.sh . build jni - name: Test shell: bash env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - run: ci/scripts/test.sh $(pwd) $(pwd)/build + run: ci/scripts/test.sh . build jni windows: name: AMD64 Windows Server 2022 Java JDK ${{ matrix.jdk }} @@ -139,12 +139,12 @@ jobs: shell: bash env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - run: ci/scripts/build.sh $(pwd) $(pwd)/build + run: ci/scripts/build.sh . build jni - name: Test shell: bash env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - run: ci/scripts/test.sh $(pwd) $(pwd)/build + run: ci/scripts/test.sh . build jni integration: name: AMD64 integration @@ -184,7 +184,7 @@ jobs: with: path: .docker key: integration-conda-${{ hashFiles('cpp/**') }} - restore-keys: conda- + restore-keys: integration-conda- - name: Setup Python uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index 309f28126a..146a40cf7a 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -17,7 +17,7 @@ # specific language governing permissions and limitations # under the License. -set -eo pipefail +set -euo pipefail if [[ "${ARROW_JAVA_BUILD:-ON}" != "ON" ]]; then exit @@ -27,11 +27,9 @@ source_dir=${1} build_dir=${2} java_jni_dist_dir=${3} -: "${BUILD_DOCS_JAVA:=OFF}" - mvn="mvn -B -DskipTests -Drat.skip=true -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" -if [ "$ARROW_JAVA_SKIP_GIT_PLUGIN" ]; then +if [ "${ARROW_JAVA_SKIP_GIT_PLUGIN:-OFF}" = "ON" ]; then mvn="${mvn} -Dmaven.gitcommitid.skip=true" fi @@ -61,27 +59,40 @@ done pushd "${build_dir}" -if [ "${ARROW_JAVA_SHADE_FLATBUFFERS}" == "ON" ]; then +# TODO: ARROW_JAVA_SHADE_FLATBUFFERS isn't used for our artifacts. Do +# we need this? +# See also: +# * https://github.com/apache/arrow/issues/22021 +# * https://github.com/apache/arrow-java/issues/67 +if [ "${ARROW_JAVA_SHADE_FLATBUFFERS:-OFF}" == "ON" ]; then mvn="${mvn} -Pshade-flatbuffers" fi -if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then +if [ "${ARROW_JAVA_CDATA:-OFF}" = "ON" ]; then mvn="${mvn} -Darrow.c.jni.dist.dir=${java_jni_dist_dir} -Parrow-c-data" fi -if [ "${ARROW_JAVA_JNI}" = "ON" ]; then +if [ "${ARROW_JAVA_JNI:-OFF}" = "ON" ]; then mvn="${mvn} -Darrow.cpp.build.dir=${java_jni_dist_dir} -Parrow-jni" fi # Use `2 * ncores` threads ${mvn} -T 2C clean install -if [ "${BUILD_DOCS_JAVA}" == "ON" ]; then - # HTTP pooling is turned of to avoid download issues https://issues.apache.org/jira/browse/ARROW-11633 - # GH-43378: Maven site plugins not compatible with multithreading - mkdir -p "${build_dir}"/docs/java/reference - ${mvn} -Dcheckstyle.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false clean install site - rsync -a target/site/apidocs/ "${build_dir}"/docs/java/reference +if [ "${ARROW_JAVA_BUILD_DOCS:-OFF}" == "ON" ]; then + # HTTP pooling is turned off to avoid download issues: + # https://github.com/apache/arrow/issues/27496 + # + # Maven site plugins not compatible with multithreading: + # https://github.com/apache/arrow/issues/43378 + ${mvn} \ + -Dcheckstyle.skip=true \ + -Dhttp.keepAlive=false \ + -Dmaven.wagon.http.pool=false \ + site + rm -rf docs/reference + mkdir -p docs + cp -a target/site/apidocs/ docs/reference fi popd diff --git a/ci/scripts/jni_full_build.sh b/ci/scripts/jni_full_build.sh index 49cbdc138f..e9ad0ddbda 100755 --- a/ci/scripts/jni_full_build.sh +++ b/ci/scripts/jni_full_build.sh @@ -17,15 +17,21 @@ # specific language governing permissions and limitations # under the License. -set -eu +set -euo pipefail +# shellcheck source=ci/scripts/util_log.sh +. "$(dirname "${0}")/util_log.sh" + +github_actions_group_begin "Prepare arguments" source_dir="$(cd "${1}" && pwd)" jni_build_dir="$(cd "${2}" && pwd)" dist_dir="${3}" rm -rf "${dist_dir}" mkdir -p "${dist_dir}" dist_dir="$(cd "${dist_dir}" && pwd)" +github_actions_group_end +github_actions_group_begin "Clear old artifacts" # Ensure that there is no old artifacts inside the maven repository maven_repo=~/.m2/repository/org/apache/arrow if [ -d "$maven_repo" ]; then @@ -34,7 +40,9 @@ if [ -d "$maven_repo" ]; then -exec echo {} ";" \ -exec rm -rf {} ";" fi +github_actions_group_end +github_actions_group_begin "Generate dummy GPG key" # Generate dummy GPG key for -Papache-release. # -Papache-release generates signs (*.asc) of artifacts. # We don't use these signs in our release process. @@ -46,19 +54,37 @@ fi echo "%no-protection" ) | gpg --full-generate-key --batch +github_actions_group_end pushd "${source_dir}" +github_actions_group_begin "Build .jar" # build the entire project -mvn clean \ - install \ +mvn \ + --no-transfer-progress \ + -Darrow.c.jni.dist.dir="${jni_build_dir}" \ + -Darrow.cpp.build.dir="${jni_build_dir}" \ -Papache-release \ -Parrow-c-data \ -Parrow-jni \ - -Darrow.cpp.build.dir="${jni_build_dir}" \ + clean \ + install +github_actions_group_end +github_actions_group_begin "Build docs" +# build docs +mvn \ + --no-transfer-progress \ -Darrow.c.jni.dist.dir="${jni_build_dir}" \ - --no-transfer-progress + -Darrow.cpp.build.dir="${jni_build_dir}" \ + -Dcheckstyle.skip=true \ + -Dhttp.keepAlive=false \ + -Dmaven.wagon.http.pool=false \ + -Parrow-c-data \ + -Parrow-jni \ + site +github_actions_group_end popd +github_actions_group_begin "Prepare artifacts" # copy all jar, zip and pom files to the distribution folder find ~/.m2/repository/org/apache/arrow \ "(" \ @@ -75,3 +101,4 @@ for artifact in "${dist_dir}"/*; do sha256sum "${artifact}" >"${artifact}.sha256" sha512sum "${artifact}" >"${artifact}.sha512" done +github_actions_group_end diff --git a/ci/scripts/test.sh b/ci/scripts/test.sh index f65f630182..cacc20034e 100755 --- a/ci/scripts/test.sh +++ b/ci/scripts/test.sh @@ -23,10 +23,14 @@ if [[ "${ARROW_JAVA_TEST:-ON}" != "ON" ]]; then exit fi -source_dir=${1} +source_dir="$(cd "${1}" && pwd)" build_dir=${2} java_jni_dist_dir=${3} +if [ -d "${java_jni_dist_dir}" ]; then + java_jni_dist_dir="$(cd "${java_jni_dist_dir}" && pwd)" +fi + mvn="mvn -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" # Use `2 * ncores` threads mvn="${mvn} -T 2C" diff --git a/compose.yaml b/compose.yaml index 5decbfc86c..58a19676a1 100644 --- a/compose.yaml +++ b/compose.yaml @@ -47,8 +47,8 @@ services: - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated command: /bin/bash -c " - /arrow-java/ci/scripts/build.sh /arrow-java /build && - /arrow-java/ci/scripts/test.sh /arrow-java /build" + /arrow-java/ci/scripts/build.sh /arrow-java /build /jni && + /arrow-java/ci/scripts/test.sh /arrow-java /build /jni" conda-jni-cdata: # Builds and tests just the C Data Interface JNI library and JARs. @@ -103,11 +103,11 @@ services: volumes: - .:/arrow-java:delegated - ${ARROW_REPO_ROOT}:/arrow:delegated - - ${DOCKER_VOLUME_PREFIX}ccache:/ccache:delegated + - ${DOCKER_VOLUME_PREFIX}ccache-cache:/ccache:delegated - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated environment: ARROW_JAVA_CDATA: "ON" CCACHE_DIR: "/ccache" command: ["git config --global --add safe.directory /arrow-java && \ - /arrow-java/ci/scripts/jni_manylinux_build.sh /arrow-java /arrow /build /arrow-java/jni"] + /arrow-java/ci/scripts/jni_manylinux_build.sh /arrow-java /arrow /build/java /arrow-java/jni"] From ae395dfe4235642dd641bd5ff8256f6315497793 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Jan 2025 05:09:11 -0500 Subject: [PATCH 047/408] MINOR: Bump com.puppycrawl.tools:checkstyle from 10.20.1 to 10.21.1 (#543) Bumps [com.puppycrawl.tools:checkstyle](https://github.com/checkstyle/checkstyle) from 10.20.1 to 10.21.1. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b07797252e..0bbcba911c 100644 --- a/pom.xml +++ b/pom.xml @@ -105,7 +105,7 @@ under the License. 1.12.0 2 - 10.20.1 + 10.21.1 true 2.31.0 5.15.2 From c69ae1a593d51d8b4c2851ea20b34c7cd9668d66 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Wed, 22 Jan 2025 07:59:24 +0800 Subject: [PATCH 048/408] GH-549: Fix Incorrect NOTICE File and Update LICENSE Headers for Third-Party Code Compliance (#550) The files **IntObjectMap.java** and **IntObjectHashMap.java** are sourced from: https://github.com/netty/netty/tree/netty-4.1.117.Final/common/src/main/templates/io/netty/util/collection/ (Specific types will be generated during the compilation phase, but fundamentally, no change in intellectual property ownership has occurred.) The Guava code is sourced from: https://github.com/google/guava/blob/v33.4.0/guava/src/com/google/common/base/Preconditions.java Given the possibility of changes to the license, the specific version has been noted to help reduce ambiguity. close #549 --- LICENSE.txt | 4 ++-- NOTICE.txt | 12 +++++++++- dev/checkstyle/suppressions.xml | 2 +- .../org/apache/arrow/util/Preconditions.java | 21 ++++++++---------- pom.xml | 5 +++++ .../arrow/vector/util/IntObjectHashMap.java | 22 +++++++++---------- .../arrow/vector/util/IntObjectMap.java | 22 +++++++++---------- 7 files changed, 48 insertions(+), 40 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index 7ae0e2080e..c45ec81a93 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -205,13 +205,13 @@ vector/src/main/java/org/apache/arrow/vector/util/IntObjectHashMap.java vector/src/main/java/org/apache/arrow/vector/util/IntObjectMap.java -These file are derived from code from Netty, which is made available under the +These file are derived from code from Netty(4.1.117), which is made available under the Apache License 2.0. -------------------------------------------------------------------------------- memory/memory-core/src/main/java/org/apache/arrow/util/Preconditions.java -This product includes software from Google Guava, which is made available under the +This product includes software from Google Guava(33.4.0), which is made available under the Apache License 2.0. * Copyright (C) 2007 The Guava Authors * https://github.com/google/guava diff --git a/NOTICE.txt b/NOTICE.txt index a595306b89..da061b7926 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1,5 +1,15 @@ Apache Arrow Java -Copyright 2016-2024 The Apache Software Foundation +Copyright 2016-2025 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). + +This product contains code from the Netty Project: +--------------------------------------------------- +The Netty Project +================= +Please visit the Netty web site for more information: + * http://netty.io/ + +Copyright 2014 The Netty Project +--------------------------------------------------- diff --git a/dev/checkstyle/suppressions.xml b/dev/checkstyle/suppressions.xml index e8669c54e6..b1841d7fe6 100644 --- a/dev/checkstyle/suppressions.xml +++ b/dev/checkstyle/suppressions.xml @@ -37,7 +37,7 @@ - + diff --git a/memory/memory-core/src/main/java/org/apache/arrow/util/Preconditions.java b/memory/memory-core/src/main/java/org/apache/arrow/util/Preconditions.java index 71e622b45f..16d763afba 100644 --- a/memory/memory-core/src/main/java/org/apache/arrow/util/Preconditions.java +++ b/memory/memory-core/src/main/java/org/apache/arrow/util/Preconditions.java @@ -1,18 +1,15 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * Copyright (C) 2007 The Guava Authors * - * http://www.apache.org/licenses/LICENSE-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. */ package org.apache.arrow.util; diff --git a/pom.xml b/pom.xml index 0bbcba911c..af7f5bd06e 100644 --- a/pom.xml +++ b/pom.xml @@ -739,6 +739,11 @@ under the License. ${maven.multiModuleProjectDirectory}/dev/license/asf-java.license package + + **/Preconditions.java + **/IntObjectMap.java + **/IntObjectHashMap.java + diff --git a/vector/src/main/java/org/apache/arrow/vector/util/IntObjectHashMap.java b/vector/src/main/java/org/apache/arrow/vector/util/IntObjectHashMap.java index b625f602ca..2fa16c66b8 100644 --- a/vector/src/main/java/org/apache/arrow/vector/util/IntObjectHashMap.java +++ b/vector/src/main/java/org/apache/arrow/vector/util/IntObjectHashMap.java @@ -1,18 +1,16 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * Copyright 2014 The Netty Project * - * http://www.apache.org/licenses/LICENSE-2.0 + * The Netty Project licenses this file to you under the Apache License, version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at: * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. */ package org.apache.arrow.vector.util; diff --git a/vector/src/main/java/org/apache/arrow/vector/util/IntObjectMap.java b/vector/src/main/java/org/apache/arrow/vector/util/IntObjectMap.java index 0de31b56df..362c60b5c3 100644 --- a/vector/src/main/java/org/apache/arrow/vector/util/IntObjectMap.java +++ b/vector/src/main/java/org/apache/arrow/vector/util/IntObjectMap.java @@ -1,18 +1,16 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * Copyright 2014 The Netty Project * - * http://www.apache.org/licenses/LICENSE-2.0 + * The Netty Project licenses this file to you under the Apache License, version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at: * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. */ package org.apache.arrow.vector.util; From db8b6aabd26d6526f10fed780a42791c755cac23 Mon Sep 17 00:00:00 2001 From: ViggoC Date: Wed, 22 Jan 2025 14:27:49 +0800 Subject: [PATCH 049/408] GH-48: Implement VectorAppender for BaseVariableWidthViewVector (#454) Fixes #48. --- .../arrow/vector/util/VectorAppender.java | 65 +++++++++- .../org/apache/arrow/vector/TestUtils.java | 10 ++ .../vector/TestVariableWidthViewVector.java | 41 +++--- .../testing/ValueVectorDataPopulator.java | 13 ++ .../arrow/vector/util/TestVectorAppender.java | 117 ++++++++++++++++++ 5 files changed, 218 insertions(+), 28 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java b/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java index e703571b37..0dc96a4d4b 100644 --- a/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java +++ b/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java @@ -19,6 +19,8 @@ import static org.apache.arrow.memory.util.LargeMemoryUtil.checkedCastToInt; import java.util.HashSet; +import java.util.List; +import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.util.MemoryUtil; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.BaseFixedWidthVector; @@ -91,7 +93,6 @@ public ValueVector visit(BaseFixedWidthVector deltaVector, Void value) { deltaVector.getDataBuffer(), deltaVector.getValueCount(), targetVector.getDataBuffer()); - } else { MemoryUtil.copyMemory( deltaVector.getDataBuffer().memoryAddress(), @@ -247,8 +248,66 @@ public ValueVector visit(BaseLargeVariableWidthVector deltaVector, Void value) { } @Override - public ValueVector visit(BaseVariableWidthViewVector left, Void value) { - throw new UnsupportedOperationException("View vectors are not supported."); + public ValueVector visit(BaseVariableWidthViewVector deltaVector, Void value) { + Preconditions.checkArgument( + typeVisitor.equals(deltaVector), + "The targetVector to append must have the same type as the targetVector being appended"); + + if (deltaVector.getValueCount() == 0) { + return targetVector; // nothing to append, return + } + + int oldTargetValueCount = targetVector.getValueCount(); + int newValueCount = oldTargetValueCount + deltaVector.getValueCount(); + + // make sure there is enough capacity + while (targetVector.getValueCapacity() < newValueCount) { + // Do not call BaseVariableWidthViewVector#reAlloc() here, + // because reallocViewDataBuffer() is always unnecessary + ((BaseVariableWidthViewVector) targetVector).reallocValidityBuffer(); + ((BaseVariableWidthViewVector) targetVector).reallocViewBuffer(); + } + + // append validity buffer + BitVectorHelper.concatBits( + targetVector.getValidityBuffer(), + oldTargetValueCount, + deltaVector.getValidityBuffer(), + deltaVector.getValueCount(), + targetVector.getValidityBuffer()); + + // append data buffers + BaseVariableWidthViewVector targetViewVector = (BaseVariableWidthViewVector) targetVector; + List targetDataBuffers = targetViewVector.getDataBuffers(); + final int oldTargetDataBufferCount = targetDataBuffers.size(); + List deltaVectorDataBuffers = deltaVector.getDataBuffers(); + deltaVectorDataBuffers.forEach(buf -> buf.getReferenceManager().retain()); + targetDataBuffers.addAll(deltaVectorDataBuffers); + + // append view buffer + ArrowBuf targetViewBuffer = targetVector.getDataBuffer(); + MemoryUtil.copyMemory( + deltaVector.getDataBuffer().memoryAddress(), + targetViewBuffer.memoryAddress() + + (long) BaseVariableWidthViewVector.ELEMENT_SIZE * oldTargetValueCount, + (long) BaseVariableWidthViewVector.ELEMENT_SIZE * deltaVector.getValueCount()); + + // update view buffer + for (int i = oldTargetValueCount; i < newValueCount; i++) { + if (targetViewVector.isSet(i) > 0 + && targetViewVector.getValueLength(i) > BaseVariableWidthViewVector.INLINE_SIZE) { + long start = + (long) i * BaseVariableWidthViewVector.ELEMENT_SIZE + + BaseVariableWidthViewVector.LENGTH_WIDTH + + BaseVariableWidthViewVector.PREFIX_WIDTH; + // shift buf id + int bufferId = targetViewBuffer.getInt(start); + targetViewBuffer.setInt(start, bufferId + oldTargetDataBufferCount); + } + } + + targetVector.setValueCount(newValueCount); + return targetVector; } @Override diff --git a/vector/src/test/java/org/apache/arrow/vector/TestUtils.java b/vector/src/test/java/org/apache/arrow/vector/TestUtils.java index 3845652ad0..82295f8037 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestUtils.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestUtils.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.vector; +import java.util.Random; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -52,4 +53,13 @@ public static T newVector( Class c, String name, MinorType type, BufferAllocator allocator) { return c.cast(FieldType.nullable(type.getType()).createNewSingleVector(name, allocator, null)); } + + public static String generateRandomString(int length) { + Random random = new Random(); + StringBuilder sb = new StringBuilder(length); + for (int i = 0; i < length; i++) { + sb.append(random.nextInt(10)); // 0-9 + } + return sb.toString(); + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestVariableWidthViewVector.java b/vector/src/test/java/org/apache/arrow/vector/TestVariableWidthViewVector.java index a4533dba3b..7a3a1bae63 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestVariableWidthViewVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestVariableWidthViewVector.java @@ -160,7 +160,7 @@ public void testDataBufferBasedAllocationInSameBuffer() { try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator)) { viewVarCharVector.allocateNew(48, 4); final int valueCount = 4; - String str4 = generateRandomString(34); + String str4 = TestUtils.generateRandomString(34); viewVarCharVector.set(0, STR1); viewVarCharVector.set(1, STR2); viewVarCharVector.set(2, STR3); @@ -216,7 +216,7 @@ public void testDataBufferBasedAllocationInOtherBuffer() { try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator)) { viewVarCharVector.allocateNew(48, 4); final int valueCount = 4; - String str4 = generateRandomString(35); + String str4 = TestUtils.generateRandomString(35); viewVarCharVector.set(0, STR1); viewVarCharVector.set(1, STR2); viewVarCharVector.set(2, STR3); @@ -271,7 +271,7 @@ public void testDataBufferBasedAllocationInOtherBuffer() { public void testSetSafe() { try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator)) { viewVarCharVector.allocateNew(1, 1); - byte[] str6 = generateRandomString(40).getBytes(); + byte[] str6 = TestUtils.generateRandomString(40).getBytes(); final List strings = List.of(STR0, STR1, STR2, STR3, STR4, STR5, str6); // set data to a position out of capacity index @@ -305,8 +305,8 @@ public void testMixedAllocation() { try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator)) { viewVarCharVector.allocateNew(128, 6); final int valueCount = 6; - String str4 = generateRandomString(35); - String str6 = generateRandomString(40); + String str4 = TestUtils.generateRandomString(35); + String str6 = TestUtils.generateRandomString(40); viewVarCharVector.set(0, STR1); viewVarCharVector.set(1, STR2); viewVarCharVector.set(2, STR3); @@ -405,7 +405,7 @@ public void testSetNullableViewVarCharHolder() { setAndCheck(viewVarCharVector, i, strings.get(size - i - 1), stringHolder); } - String longString = generateRandomString(128); + String longString = TestUtils.generateRandomString(128); setAndCheck(viewVarCharVector, 6, longString.getBytes(), stringHolder); } } @@ -441,7 +441,7 @@ public void testSetNullableViewVarBinaryHolder() { setAndCheck(viewVarBinaryVector, i, strings.get(size - i - 1), holder); } - String longString = generateRandomString(128); + String longString = TestUtils.generateRandomString(128); setAndCheck(viewVarBinaryVector, 6, longString.getBytes(), holder); } } @@ -1169,7 +1169,7 @@ public void testOverwriteShortFromLongString() { vector.setValueCount(5); // overwrite index 2 with a long string - String longString = generateRandomString(128); + String longString = TestUtils.generateRandomString(128); byte[] longStringBytes = longString.getBytes(StandardCharsets.UTF_8); // since the append-only approach is used and the remaining capacity // is not enough to store the new string; a new buffer will be allocated. @@ -1373,7 +1373,7 @@ public void testOverwriteLongFromALongerLongString() { // since a new buffer is added to the dataBuffers final ArrowBuf currentDataBuf = vector.dataBuffers.get(0); final long remainingCapacity = currentDataBuf.capacity() - currentDataBuf.writerIndex(); - String longerString = generateRandomString(35); + String longerString = TestUtils.generateRandomString(35); byte[] longerStringBytes = longerString.getBytes(StandardCharsets.UTF_8); assertTrue(remainingCapacity < longerStringBytes.length); @@ -1406,7 +1406,7 @@ public void testOverwriteLongFromALongerLongString() { // the remaining capacity is enough to store in the same data buffer final ArrowBuf currentDataBuf = vector.dataBuffers.get(0); final long remainingCapacity = currentDataBuf.capacity() - currentDataBuf.writerIndex(); - String longerString = generateRandomString(24); + String longerString = TestUtils.generateRandomString(24); byte[] longerStringBytes = longerString.getBytes(StandardCharsets.UTF_8); assertTrue(remainingCapacity > longerStringBytes.length); @@ -1505,7 +1505,7 @@ public void testSafeOverwriteShortFromLongString() { vector.setValueCount(5); // overwrite index 2 with a long string - String longString = generateRandomString(128); + String longString = TestUtils.generateRandomString(128); byte[] longStringBytes = longString.getBytes(StandardCharsets.UTF_8); vector.setSafe(2, longStringBytes); @@ -1671,7 +1671,7 @@ public void testSafeOverwriteLongFromALongerLongString() { vector.setSafe(2, STR7); vector.setValueCount(3); - String longerString = generateRandomString(35); + String longerString = TestUtils.generateRandomString(35); byte[] longerStringBytes = longerString.getBytes(StandardCharsets.UTF_8); vector.setSafe(1, longerStringBytes); @@ -1697,7 +1697,7 @@ public void testSafeOverwriteLongFromALongerLongString() { vector.setSafe(4, STR6); vector.setValueCount(5); - String longerString = generateRandomString(24); + String longerString = TestUtils.generateRandomString(24); byte[] longerStringBytes = longerString.getBytes(StandardCharsets.UTF_8); vector.setSafe(2, longerStringBytes); @@ -1869,7 +1869,7 @@ public void testCopyFromWithNulls( // to avoid re-allocation. This is to test copyFrom() without re-allocation. final int numberOfValues = initialCapacity / 2 / ViewVarCharVector.ELEMENT_SIZE; - final String prefixString = generateRandomString(12); + final String prefixString = TestUtils.generateRandomString(12); for (int i = 0; i < numberOfValues; i++) { if (i % 3 == 0) { @@ -1965,7 +1965,7 @@ public void testCopyFromSafeWithNulls( final int numberOfValues = initialCapacity / ViewVarCharVector.ELEMENT_SIZE; - final String prefixString = generateRandomString(12); + final String prefixString = TestUtils.generateRandomString(12); for (int i = 0; i < numberOfValues; i++) { if (i % 3 == 0) { @@ -2746,7 +2746,7 @@ private void testSplitAndTransferWithMultipleDataBuffersHelper( */ @Test public void testSplitAndTransferWithMultipleDataBuffers() { - final String str4 = generateRandomString(35); + final String str4 = TestUtils.generateRandomString(35); final byte[][] data = new byte[][] {STR1, STR2, STR3, str4.getBytes(StandardCharsets.UTF_8)}; final int startIndex = 1; final int length = 3; @@ -2851,13 +2851,4 @@ public void testVectorLoadUnloadOnMixedTypes() { } } } - - private String generateRandomString(int length) { - Random random = new Random(); - StringBuilder sb = new StringBuilder(length); - for (int i = 0; i < length; i++) { - sb.append(random.nextInt(10)); // 0-9 - } - return sb.toString(); - } } diff --git a/vector/src/test/java/org/apache/arrow/vector/testing/ValueVectorDataPopulator.java b/vector/src/test/java/org/apache/arrow/vector/testing/ValueVectorDataPopulator.java index f599dfa539..849fe6d667 100644 --- a/vector/src/test/java/org/apache/arrow/vector/testing/ValueVectorDataPopulator.java +++ b/vector/src/test/java/org/apache/arrow/vector/testing/ValueVectorDataPopulator.java @@ -60,6 +60,7 @@ import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VariableWidthFieldVector; +import org.apache.arrow.vector.ViewVarCharVector; import org.apache.arrow.vector.complex.BaseLargeRepeatedValueViewVector; import org.apache.arrow.vector.complex.BaseRepeatedValueVector; import org.apache.arrow.vector.complex.BaseRepeatedValueViewVector; @@ -606,6 +607,18 @@ public static void setVector(VarCharVector vector, String... values) { vector.setValueCount(length); } + /** Populate values for ViewVarCharVector. */ + public static void setVector(ViewVarCharVector vector, String... values) { + final int length = values.length; + vector.allocateNewSafe(); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.setSafe(i, values[i].getBytes(StandardCharsets.UTF_8)); + } + } + vector.setValueCount(length); + } + /** Populate values for LargeVarCharVector. */ public static void setVector(LargeVarCharVector vector, String... values) { final int length = values.length; diff --git a/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java b/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java index 19eafd1b20..e1b3889d85 100644 --- a/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java +++ b/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java @@ -24,16 +24,21 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.stream.IntStream; +import java.util.stream.Stream; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BaseValueVector; +import org.apache.arrow.vector.BaseVariableWidthViewVector; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.Float4Vector; import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.LargeVarCharVector; +import org.apache.arrow.vector.TestUtils; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.ViewVarCharVector; import org.apache.arrow.vector.compare.Range; import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.compare.TypeEqualsVisitor; @@ -171,6 +176,118 @@ public void testAppendVariableWidthVector() { } } + @Test + public void testAppendVariableWidthViewVector() { + final int length1 = 10; + final int length2 = 5; + try (ViewVarCharVector target = new ViewVarCharVector("", allocator); + ViewVarCharVector delta = new ViewVarCharVector("", allocator)) { + target.allocateNew(5, length1); + delta.allocateNew(5, length2); + + ValueVectorDataPopulator.setVector( + target, "a0", "a1", "a2", "a3", null, "a5", "a6", "a7", "a8", "a9"); + ValueVectorDataPopulator.setVector(delta, "a10", "a11", "a12", "a13", null); + + VectorAppender appender = new VectorAppender(target); + delta.accept(appender, null); + + try (ViewVarCharVector expected = new ViewVarCharVector("expected", allocator)) { + expected.allocateNew(); + ValueVectorDataPopulator.setVector( + expected, "a0", "a1", "a2", "a3", null, "a5", "a6", "a7", "a8", "a9", "a10", "a11", + "a12", "a13", null); + assertVectorsEqual(expected, target); + } + } + } + + @Test + public void testAppendEmptyVariableWidthViewVector() { + try (ViewVarCharVector target = new ViewVarCharVector("", allocator); + ViewVarCharVector delta = new ViewVarCharVector("", allocator)) { + ValueVectorDataPopulator.setVector( + target, "a0", "a1", "a2", "a3", null, "a5", "a6", "a7", "a8", "a9"); + + VectorAppender appender = new VectorAppender(target); + delta.accept(appender, null); + + try (ViewVarCharVector expected = new ViewVarCharVector("expected", allocator)) { + ValueVectorDataPopulator.setVector( + expected, "a0", "a1", "a2", "a3", null, "a5", "a6", "a7", "a8", "a9"); + assertVectorsEqual(expected, target); + } + } + } + + @Test + public void testAppendShortLongVariableWidthViewVector() { + try (ViewVarCharVector target = new ViewVarCharVector("", allocator); + ViewVarCharVector delta = new ViewVarCharVector("", allocator)) { + String[] targetValues = + IntStream.range(-5, 5) + .mapToObj( + i -> TestUtils.generateRandomString(BaseVariableWidthViewVector.INLINE_SIZE + i)) + .toArray(String[]::new); + ValueVectorDataPopulator.setVector(target, targetValues); + + String[] deltaValues = + IntStream.range(-3, 3) + .mapToObj( + i -> TestUtils.generateRandomString(BaseVariableWidthViewVector.INLINE_SIZE + i)) + .toArray(String[]::new); + ValueVectorDataPopulator.setVector(delta, deltaValues); + + VectorAppender appender = new VectorAppender(target); + delta.accept(appender, null); + + assertEquals(2, target.getDataBuffers().size()); + try (ViewVarCharVector expected = new ViewVarCharVector("expected", allocator)) { + ValueVectorDataPopulator.setVector( + expected, + Stream.concat(Arrays.stream(targetValues), Arrays.stream(deltaValues)) + .toArray(String[]::new)); + assertVectorsEqual(expected, target); + } + } + } + + @Test + public void testAppendLongVariableWidthViewVector() { + try (ViewVarCharVector target = new ViewVarCharVector("", allocator); + ViewVarCharVector delta = new ViewVarCharVector("", allocator)) { + + String[] targetValues = randomLongViewVarCharVector(target); + String[] deltaValues = randomLongViewVarCharVector(delta); + + VectorAppender appender = new VectorAppender(target); + delta.accept(appender, null); + + assertEquals(4, target.getDataBuffers().size()); + try (ViewVarCharVector expected = new ViewVarCharVector("expected", allocator)) { + ValueVectorDataPopulator.setVector( + expected, + Stream.concat(Arrays.stream(targetValues), Arrays.stream(deltaValues)) + .toArray(String[]::new)); + assertVectorsEqual(expected, target); + } + } + } + + private static String[] randomLongViewVarCharVector(ViewVarCharVector target) { + assertEquals(0, target.getDataBuffers().size()); + int initial = 64; + int stringCount = 128; + target.setInitialCapacity(initial); + String[] targetValues = + IntStream.range(0, stringCount) + .mapToObj(i -> TestUtils.generateRandomString(BaseVariableWidthViewVector.ELEMENT_SIZE)) + .toArray(String[]::new); + ValueVectorDataPopulator.setVector(target, targetValues); + assertEquals(2, target.getDataBuffers().size()); + return targetValues; + } + @Test public void testAppendEmptyVariableWidthVector() { try (VarCharVector target = new VarCharVector("", allocator); From 7cfc89dce4763d027e3900a0dc80d8e46334c16f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 Jan 2025 01:28:30 -0500 Subject: [PATCH 050/408] MINOR: Bump commons-codec:commons-codec from 1.17.1 to 1.17.2 (#504) Bumps [commons-codec:commons-codec](https://github.com/apache/commons-codec) from 1.17.1 to 1.17.2. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- vector/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vector/pom.xml b/vector/pom.xml index 6ad28bbe9c..673f8eed17 100644 --- a/vector/pom.xml +++ b/vector/pom.xml @@ -60,7 +60,7 @@ under the License. commons-codec commons-codec - 1.17.1 + 1.17.2 org.apache.arrow From 921e6b46f37b44d477bd9ad900515636abeb86e8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 Jan 2025 22:46:01 -0500 Subject: [PATCH 051/408] MINOR: Bump com.google.api.grpc:proto-google-common-protos from 2.49.0 to 2.50.1 (#542) Bumps [com.google.api.grpc:proto-google-common-protos](https://github.com/googleapis/sdk-platform-java) from 2.49.0 to 2.50.1. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml index 340004041e..476b994127 100644 --- a/flight/flight-core/pom.xml +++ b/flight/flight-core/pom.xml @@ -134,7 +134,7 @@ under the License. com.google.api.grpc proto-google-common-protos - 2.49.0 + 2.50.1 test From 1bde279dcca3c856c9bcdb74bea4dfa8df767925 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 23 Jan 2025 17:06:48 +0900 Subject: [PATCH 052/408] GH-545: [CI] Use apache/arrow main for daily RC CI (#556) Fixes #545. --- .github/workflows/rc.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 3d6120dc24..61281964e6 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -108,8 +108,15 @@ jobs: run: | tar -xf apache-arrow-java-*.tar.gz --strip-components=1 - name: Download the latest Apache Arrow C++ + if: github.event_name != 'schedule' run: | ci/scripts/download_cpp.sh + - name: Checkout Apache Arrow C++ + if: github.event_name == 'schedule' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: apache/arrow + path: arrow - name: Checkout apache/arrow-testing uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -168,8 +175,15 @@ jobs: run: | tar -xf apache-arrow-java-*.tar.gz --strip-components=1 - name: Download the latest Apache Arrow C++ + if: github.event_name != 'schedule' run: | ci/scripts/download_cpp.sh + - name: Checkout Apache Arrow C++ + if: github.event_name == 'schedule' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: apache/arrow + path: arrow - name: Checkout apache/arrow-testing uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -286,9 +300,16 @@ jobs: run: | tar -xf apache-arrow-java-*.tar.gz --strip-components=1 - name: Download the latest Apache Arrow C++ + if: github.event_name != 'schedule' shell: bash run: | ci/scripts/download_cpp.sh + - name: Checkout Apache Arrow C++ + if: github.event_name == 'schedule' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: apache/arrow + path: arrow - name: Set up Java uses: actions/setup-java@7a6d8a8234af8eb26422e24e3006232cccaa061b # v4.6.0 with: From 3dd0311592f7ca8db615791b4f8fd3aa8dea71cb Mon Sep 17 00:00:00 2001 From: ViggoC Date: Fri, 24 Jan 2025 12:08:43 +0800 Subject: [PATCH 053/408] GH-41: Add VariableWidthViewVectorBenchmarks (#453) GitHub Issue: https://github.com/apache/arrow-java/issues/41 --- ...VariableWidthVectorBenchmarkConstants.java | 22 +++ .../vector/VariableWidthVectorBenchmarks.java | 13 +- ...iableWidthVectorInlineValueBenchmarks.java | 130 +++++++++++++++++ .../VariableWidthViewVectorBenchmarks.java | 130 +++++++++++++++++ ...eWidthViewVectorInlineValueBenchmarks.java | 131 ++++++++++++++++++ 5 files changed, 422 insertions(+), 4 deletions(-) create mode 100644 performance/src/main/java/org/apache/arrow/vector/VariableWidthVectorBenchmarkConstants.java create mode 100644 performance/src/main/java/org/apache/arrow/vector/VariableWidthVectorInlineValueBenchmarks.java create mode 100644 performance/src/main/java/org/apache/arrow/vector/VariableWidthViewVectorBenchmarks.java create mode 100644 performance/src/main/java/org/apache/arrow/vector/VariableWidthViewVectorInlineValueBenchmarks.java diff --git a/performance/src/main/java/org/apache/arrow/vector/VariableWidthVectorBenchmarkConstants.java b/performance/src/main/java/org/apache/arrow/vector/VariableWidthVectorBenchmarkConstants.java new file mode 100644 index 0000000000..d4ef49d902 --- /dev/null +++ b/performance/src/main/java/org/apache/arrow/vector/VariableWidthVectorBenchmarkConstants.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector; + +public class VariableWidthVectorBenchmarkConstants { + public static final String SHORT_VALUE = "InlineValue"; + public static final String LONG_VALUE = VariableWidthVectorBenchmarks.class.getName(); +} diff --git a/performance/src/main/java/org/apache/arrow/vector/VariableWidthVectorBenchmarks.java b/performance/src/main/java/org/apache/arrow/vector/VariableWidthVectorBenchmarks.java index 0bce6569d2..17d3ac1fee 100644 --- a/performance/src/main/java/org/apache/arrow/vector/VariableWidthVectorBenchmarks.java +++ b/performance/src/main/java/org/apache/arrow/vector/VariableWidthVectorBenchmarks.java @@ -23,8 +23,10 @@ import org.apache.arrow.vector.holders.NullableVarCharHolder; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Level; import org.openjdk.jmh.annotations.Mode; import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; import org.openjdk.jmh.annotations.Scope; import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; @@ -45,15 +47,18 @@ public class VariableWidthVectorBenchmarks { private static final int ALLOCATOR_CAPACITY = 1024 * 1024; - private static byte[] bytes = VariableWidthVectorBenchmarks.class.getName().getBytes(); + private static byte[] bytes = VariableWidthVectorBenchmarkConstants.LONG_VALUE.getBytes(); private ArrowBuf arrowBuff; private BufferAllocator allocator; private VarCharVector vector; + @Param({"1", "2", "10", "40"}) + private int step; + /** Setup benchmarks. */ - @Setup + @Setup(Level.Iteration) public void prepare() { allocator = new RootAllocator(ALLOCATOR_CAPACITY); vector = new VarCharVector("vector", allocator); @@ -63,7 +68,7 @@ public void prepare() { } /** Tear down benchmarks. */ - @TearDown + @TearDown(Level.Iteration) public void tearDown() { arrowBuff.close(); vector.close(); @@ -87,7 +92,7 @@ public int getValueCapacity() { @OutputTimeUnit(TimeUnit.MILLISECONDS) public int setSafeFromArray() { for (int i = 0; i < 500; ++i) { - vector.setSafe(i * 40, bytes); + vector.setSafe(i * step, bytes); } return vector.getBufferSize(); } diff --git a/performance/src/main/java/org/apache/arrow/vector/VariableWidthVectorInlineValueBenchmarks.java b/performance/src/main/java/org/apache/arrow/vector/VariableWidthVectorInlineValueBenchmarks.java new file mode 100644 index 0000000000..354d1b1479 --- /dev/null +++ b/performance/src/main/java/org/apache/arrow/vector/VariableWidthVectorInlineValueBenchmarks.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector; + +import java.util.concurrent.TimeUnit; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.holders.NullableVarCharHolder; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +/** Benchmarks for {@link BaseVariableWidthVector}. */ +@State(Scope.Benchmark) +public class VariableWidthVectorInlineValueBenchmarks { + // checkstyle:off: MissingJavadocMethod + + private static final int VECTOR_CAPACITY = 16 * 1024; + + private static final int VECTOR_LENGTH = 1024; + + private static final int ALLOCATOR_CAPACITY = 1024 * 1024; + + private static final byte[] bytes = VariableWidthVectorBenchmarkConstants.SHORT_VALUE.getBytes(); + private ArrowBuf arrowBuff; + + private BufferAllocator allocator; + + private VarCharVector vector; + + @Param({"1", "2", "10", "40"}) + private int step; + + /** Setup benchmarks. */ + @Setup(Level.Iteration) + public void prepare() { + allocator = new RootAllocator(ALLOCATOR_CAPACITY); + vector = new VarCharVector("vector", allocator); + vector.allocateNew(VECTOR_CAPACITY, VECTOR_LENGTH); + arrowBuff = allocator.buffer(VECTOR_LENGTH); + arrowBuff.setBytes(0, bytes, 0, bytes.length); + } + + /** Tear down benchmarks. */ + @TearDown(Level.Iteration) + public void tearDown() { + arrowBuff.close(); + vector.close(); + allocator.close(); + } + + /** + * Test {@link BaseVariableWidthVector#getValueCapacity()}. + * + * @return useless. To avoid DCE by JIT. + */ + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.NANOSECONDS) + public int getValueCapacity() { + return vector.getValueCapacity(); + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public int setSafeFromArray() { + for (int i = 0; i < 500; ++i) { + vector.setSafe(i * step, bytes); + } + return vector.getBufferSize(); + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public int setSafeFromNullableVarcharHolder() { + NullableVarCharHolder nvch = new NullableVarCharHolder(); + nvch.buffer = arrowBuff; + nvch.start = 0; + nvch.end = bytes.length; + for (int i = 0; i < 50; ++i) { + nvch.isSet = 0; + for (int j = 0; j < 9; ++j) { + int idx = 10 * i + j; + vector.setSafe(idx, nvch); + } + nvch.isSet = 1; + vector.setSafe(10 * (i + 1), nvch); + } + return vector.getBufferSize(); + } + + public static void main(String[] args) throws RunnerException { + Options opt = + new OptionsBuilder() + .include(VariableWidthVectorInlineValueBenchmarks.class.getSimpleName()) + .forks(1) + .build(); + + new Runner(opt).run(); + } + // checkstyle:on: MissingJavadocMethod +} diff --git a/performance/src/main/java/org/apache/arrow/vector/VariableWidthViewVectorBenchmarks.java b/performance/src/main/java/org/apache/arrow/vector/VariableWidthViewVectorBenchmarks.java new file mode 100644 index 0000000000..b85377dcd3 --- /dev/null +++ b/performance/src/main/java/org/apache/arrow/vector/VariableWidthViewVectorBenchmarks.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector; + +import java.util.concurrent.TimeUnit; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.holders.NullableViewVarCharHolder; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +/** Benchmarks for {@link BaseVariableWidthVector}. */ +@State(Scope.Benchmark) +public class VariableWidthViewVectorBenchmarks { + // checkstyle:off: MissingJavadocMethod + + private static final int VECTOR_CAPACITY = 16 * 1024; + + private static final int VECTOR_LENGTH = 1024; + + private static final int ALLOCATOR_CAPACITY = 1024 * 1024; + + private static byte[] bytes = VariableWidthVectorBenchmarkConstants.LONG_VALUE.getBytes(); + private ArrowBuf arrowBuff; + + private BufferAllocator allocator; + + private ViewVarCharVector vector; + + @Param({"1", "2", "10", "40"}) + private int step; + + /** Setup benchmarks. */ + @Setup(Level.Iteration) + public void prepare() { + allocator = new RootAllocator(); + vector = new ViewVarCharVector("vector", allocator); + vector.allocateNew(VECTOR_CAPACITY, VECTOR_LENGTH); + arrowBuff = allocator.buffer(VECTOR_LENGTH); + arrowBuff.setBytes(0, bytes, 0, bytes.length); + } + + /** Tear down benchmarks. */ + @TearDown(Level.Iteration) + public void tearDown() { + arrowBuff.close(); + vector.close(); + allocator.close(); + } + + /** + * Test {@link BaseVariableWidthVector#getValueCapacity()}. + * + * @return useless. To avoid DCE by JIT. + */ + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.NANOSECONDS) + public int getValueCapacity() { + return vector.getValueCapacity(); + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public int setSafeFromArray() { + for (int i = 0; i < 500; ++i) { + vector.setSafe(i * step, bytes); + } + return vector.getBufferSize(); + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public int setSafeFromNullableVarcharHolder() { + NullableViewVarCharHolder nvch = new NullableViewVarCharHolder(); + nvch.buffer = arrowBuff; + nvch.start = 0; + nvch.end = bytes.length; + for (int i = 0; i < 50; ++i) { + nvch.isSet = 0; + for (int j = 0; j < 9; ++j) { + int idx = 10 * i + j; + vector.setSafe(idx, nvch); + } + nvch.isSet = 1; + vector.setSafe(10 * (i + 1), nvch); + } + return vector.getBufferSize(); + } + + public static void main(String[] args) throws RunnerException { + Options opt = + new OptionsBuilder() + .include(VariableWidthViewVectorBenchmarks.class.getSimpleName()) + .forks(1) + .build(); + + new Runner(opt).run(); + } + // checkstyle:on: MissingJavadocMethod +} diff --git a/performance/src/main/java/org/apache/arrow/vector/VariableWidthViewVectorInlineValueBenchmarks.java b/performance/src/main/java/org/apache/arrow/vector/VariableWidthViewVectorInlineValueBenchmarks.java new file mode 100644 index 0000000000..32143d4773 --- /dev/null +++ b/performance/src/main/java/org/apache/arrow/vector/VariableWidthViewVectorInlineValueBenchmarks.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector; + +import java.util.concurrent.TimeUnit; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.holders.NullableViewVarCharHolder; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +/** Benchmarks for {@link BaseVariableWidthVector}. */ +@State(Scope.Benchmark) +public class VariableWidthViewVectorInlineValueBenchmarks { + // checkstyle:off: MissingJavadocMethod + + private static final int VECTOR_CAPACITY = 16 * 1024; + + private static final int VECTOR_LENGTH = 1024; + + private static final int ALLOCATOR_CAPACITY = 1024 * 1024; + + private static byte[] bytes = VariableWidthVectorBenchmarkConstants.SHORT_VALUE.getBytes(); + private ArrowBuf arrowBuff; + + private BufferAllocator allocator; + + private ViewVarCharVector vector; + + @Param({"1", "2", "10", "40"}) + private int step; + + /** Setup benchmarks. */ + @Setup(Level.Invocation) + public void prepare() { + allocator = new RootAllocator(); + vector = new ViewVarCharVector("vector", allocator); + vector.allocateNew(VECTOR_CAPACITY, VECTOR_LENGTH); + vector.zeroVector(); + arrowBuff = allocator.buffer(VECTOR_LENGTH); + arrowBuff.setBytes(0, bytes, 0, bytes.length); + } + + /** Tear down benchmarks. */ + @TearDown(Level.Invocation) + public void tearDown() { + arrowBuff.close(); + vector.close(); + allocator.close(); + } + + /** + * Test {@link BaseVariableWidthVector#getValueCapacity()}. + * + * @return useless. To avoid DCE by JIT. + */ + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.NANOSECONDS) + public int getValueCapacity() { + return vector.getValueCapacity(); + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public int setSafeFromArray() { + for (int i = 0; i < 500; ++i) { + vector.setSafe(i * step, bytes); + } + return vector.getBufferSize(); + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public int setSafeFromNullableVarcharHolder() { + NullableViewVarCharHolder nvch = new NullableViewVarCharHolder(); + nvch.buffer = arrowBuff; + nvch.start = 0; + nvch.end = bytes.length; + for (int i = 0; i < 50; ++i) { + nvch.isSet = 0; + for (int j = 0; j < 9; ++j) { + int idx = 10 * i + j; + vector.setSafe(idx, nvch); + } + nvch.isSet = 1; + vector.setSafe(10 * (i + 1), nvch); + } + return vector.getBufferSize(); + } + + public static void main(String[] args) throws RunnerException { + Options opt = + new OptionsBuilder() + .include(VariableWidthViewVectorInlineValueBenchmarks.class.getSimpleName()) + .forks(1) + .build(); + + new Runner(opt).run(); + } + // checkstyle:on: MissingJavadocMethod +} From 5559930eabc0bb064f7a1f79eb98ea0d99562b1e Mon Sep 17 00:00:00 2001 From: David Li Date: Thu, 23 Jan 2025 23:45:20 -0500 Subject: [PATCH 054/408] GH-546: Migrate Sphinx documentation (#553) Fixes #546. --------- Co-authored-by: Sutou Kouhei --- .github/workflows/rc.yml | 47 +- .gitignore | 1 + dev/release/rat_exclude_files.txt | 1 + docs/Makefile | 20 + docs/README.md | 28 + docs/requirements.txt | 28 + docs/source/_static/.gitignore | 0 docs/source/algorithm.rst | 92 +++ docs/source/cdata.rst | 468 +++++++++++++ docs/source/conf.py | 51 ++ docs/source/dataset.rst | 309 +++++++++ docs/source/developers/building.rst | 624 ++++++++++++++++++ docs/source/developers/development.rst | 197 ++++++ .../developers/img/conbench_benchmark.png | Bin 0 -> 141075 bytes docs/source/developers/img/conbench_runs.png | Bin 0 -> 62681 bytes docs/source/developers/img/conbench_ui.png | Bin 0 -> 70336 bytes docs/source/developers/index.rst | 28 + docs/source/flight.rst | 239 +++++++ docs/source/flight_sql.rst | 32 + docs/source/flight_sql_jdbc_driver.rst | 175 +++++ docs/source/index.rst | 48 ++ docs/source/install.rst | 230 +++++++ docs/source/ipc.rst | 202 ++++++ docs/source/jdbc.rst | 278 ++++++++ docs/source/memory.rst | 499 ++++++++++++++ docs/source/overview.rst | 90 +++ docs/source/quickstartguide.rst | 314 +++++++++ docs/source/reference/index.rst | 21 + docs/source/substrait.rst | 201 ++++++ docs/source/table.rst | 378 +++++++++++ docs/source/vector.rst | 366 ++++++++++ docs/source/vector_schema_root.rst | 163 +++++ 32 files changed, 5126 insertions(+), 4 deletions(-) create mode 100644 docs/Makefile create mode 100644 docs/README.md create mode 100644 docs/requirements.txt create mode 100644 docs/source/_static/.gitignore create mode 100644 docs/source/algorithm.rst create mode 100644 docs/source/cdata.rst create mode 100644 docs/source/conf.py create mode 100644 docs/source/dataset.rst create mode 100644 docs/source/developers/building.rst create mode 100644 docs/source/developers/development.rst create mode 100644 docs/source/developers/img/conbench_benchmark.png create mode 100644 docs/source/developers/img/conbench_runs.png create mode 100644 docs/source/developers/img/conbench_ui.png create mode 100644 docs/source/developers/index.rst create mode 100644 docs/source/flight.rst create mode 100644 docs/source/flight_sql.rst create mode 100644 docs/source/flight_sql_jdbc_driver.rst create mode 100644 docs/source/index.rst create mode 100644 docs/source/install.rst create mode 100644 docs/source/ipc.rst create mode 100644 docs/source/jdbc.rst create mode 100644 docs/source/memory.rst create mode 100644 docs/source/overview.rst create mode 100644 docs/source/quickstartguide.rst create mode 100644 docs/source/reference/index.rst create mode 100644 docs/source/substrait.rst create mode 100644 docs/source/table.rst create mode 100644 docs/source/vector.rst create mode 100644 docs/source/vector_schema_root.rst diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 61281964e6..9dc04fee05 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -421,8 +421,8 @@ jobs: - name: Prepare docs run: | mkdir -p docs - cp -a target/site/apidocs docs/reference - tar -cvzf docs.tar.gz docs + cp -a target/site/apidocs reference + tar -cvzf reference.tar.gz reference - name: Upload binaries uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 with: @@ -431,8 +431,46 @@ jobs: - name: Upload docs uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 with: - name: release-docs - path: docs.tar.gz + name: reference + path: reference.tar.gz + docs: + name: Docs + needs: + - binaries + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + cache: 'pip' + - name: Download source archive + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: release-source + - name: Download Javadocs + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: reference + - name: Extract source archive + run: | + tar -xf apache-arrow-java-*.tar.gz --strip-components=1 + - name: Build + run: | + cd docs + python -m venv venv + source venv/bin/activate + pip install -r requirements.txt + make html + tar -xf ../reference.tar.gz -C build/html + - name: Compress into single artifact to keep directory structure + run: tar -cvzf html.tar.gz -C docs/build html + - name: Upload artifacts + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: release-html + path: html.tar.gz verify: name: Verify needs: @@ -473,6 +511,7 @@ jobs: name: Upload if: github.ref_type == 'tag' needs: + - docs - verify runs-on: ubuntu-latest permissions: diff --git a/.gitignore b/.gitignore index 205be77eaf..b57597af47 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ /dev/release/apache-rat-0.16.1.jar /dev/release/filtered_rat.txt /dev/release/rat.xml +/docs/build/ CMakeCache.txt CMakeFiles/ Makefile diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 8efd379a73..8324d32cf6 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -17,3 +17,4 @@ .gitmodules dataset/src/test/resources/data/student.csv +docs/Makefile diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000000..a4de0bff18 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= -W +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000000..70c2ef2a37 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,28 @@ + + +# Documentation + +Build with Sphinx. + +```bash +cd docs +pip install -r requirements.txt +make html +``` diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000000..fa2d0bbe99 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +furo==2024.8.6 +myst-parser==4.0.0 +Sphinx==8.1.3 +sphinx-autobuild==2024.10.3 +sphinx-basic-ng==1.0.0b2 +sphinxcontrib-applehelp==2.0.0 +sphinxcontrib-devhelp==2.0.0 +sphinxcontrib-htmlhelp==2.1.0 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==2.0.0 +sphinxcontrib-serializinghtml==2.0.0 diff --git a/docs/source/_static/.gitignore b/docs/source/_static/.gitignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/source/algorithm.rst b/docs/source/algorithm.rst new file mode 100644 index 0000000000..d4838967d6 --- /dev/null +++ b/docs/source/algorithm.rst @@ -0,0 +1,92 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Java Algorithms +=============== + +Arrow's Java library provides algorithms for some commonly-used +functionalities. The algorithms are provided in the ``org.apache.arrow.algorithm`` +package of the ``algorithm`` module. + +Comparing Vector Elements +------------------------- + +Comparing vector elements is the basic for many algorithms. Vector +elements can be compared in one of the two ways: + +1. **Equality comparison**: there are two possible results for this type of comparisons: ``equal`` and ``unequal``. +Currently, this type of comparison is supported through the ``org.apache.arrow.vector.compare.VectorValueEqualizer`` +interface. + +2. **Ordering comparison**: there are three possible results for this type of comparisons: ``less than``, ``equal to`` +and ``greater than``. This comparison is supported by the abstract class ``org.apache.arrow.algorithm.sort.VectorValueComparator``. + +We provide default implementations to compare vector elements. However, users can also define ways +for customized comparisons. + +Vector Element Search +--------------------- + +A search algorithm tries to find a particular value in a vector. When successful, a vector index is +returned; otherwise, a ``-1`` is returned. The following search algorithms are provided: + +1. **Linear search**: this algorithm simply traverses the vector from the beginning, until a match is +found, or the end of the vector is reached. So it takes ``O(n)`` time, where ``n`` is the number of elements +in the vector. This algorithm is implemented in ``org.apache.arrow.algorithm.search.VectorSearcher#linearSearch``. + +2. **Binary search**: this represents a more efficient search algorithm, as it runs in ``O(log(n))`` time. +However, it is only applicable to sorted vectors. To get a sorted vector, +one can use one of our sorting algorithms, which will be discussed in the next section. This algorithm +is implemented in ``org.apache.arrow.algorithm.search.VectorSearcher#binarySearch``. + +3. **Parallel search**: when the vector is large, it takes a long time to traverse the elements to search +for a value. To make this process faster, one can split the vector into multiple partitions, and perform the +search for each partition in parallel. This is supported by ``org.apache.arrow.algorithm.search.ParallelSearcher``. + +4. **Range search**: for many scenarios, there can be multiple matching values in the vector. +If the vector is sorted, the matching values reside in a contiguous region in the vector. The +range search algorithm tries to find the upper/lower bound of the region in ``O(log(n))`` time. +An implementation is provided in ``org.apache.arrow.algorithm.search.VectorRangeSearcher``. + +Vector Sorting +-------------- + +Given a vector, a sorting algorithm turns it into a sorted one. The sorting criteria must +be specified by some ordering comparison operation. The sorting algorithms can be +classified into the following categories: + +1. **In-place sorter**: an in-place sorter performs the sorting by manipulating the original +vector, without creating any new vector. So it just returns the original vector after the sorting operations. +Currently, we have ``org.apache.arrow.algorithm.sort.FixedWidthInPlaceVectorSorter`` for in-place +sorting in ``O(nlog(n))`` time. As the name suggests, it only supports fixed width vectors. + +2. **Out-of-place sorter**: an out-of-place sorter does not mutate the original vector. Instead, +it copies vector elements to a new vector in sorted order, and returns the new vector. +We have ``org.apache.arrow.algorithm.sort.FixedWidthInPlaceVectorSorter.FixedWidthOutOfPlaceVectorSorter`` +and ``org.apache.arrow.algorithm.sort.FixedWidthInPlaceVectorSorter.VariableWidthOutOfPlaceVectorSorter`` +for fixed width and variable width vectors, respectively. Both algorithms run in ``O(nlog(n))`` time. + +3. **Index sorter**: this sorter does not actually sort the vector. Instead, it returns an integer +vector, which correspond to indices of vector elements in sorted order. With the index vector, one can +easily construct a sorted vector. In addition, some other tasks can be easily achieved, like finding the ``k`` th +smallest value in the vector. Index sorting is supported by ``org.apache.arrow.algorithm.sort.IndexSorter``, +which runs in ``O(nlog(n))`` time. It is applicable to vectors of any type. + +Other Algorithms +---------------- + +Other algorithms include vector deduplication, dictionary encoding, etc., in the ``algorithm`` module. diff --git a/docs/source/cdata.rst b/docs/source/cdata.rst new file mode 100644 index 0000000000..9643d88df3 --- /dev/null +++ b/docs/source/cdata.rst @@ -0,0 +1,468 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +================ +C Data Interface +================ + +Arrow supports exchanging data without copying or serialization within the same process +through :external+arrow:ref:`c-data-interface`, even between different language runtimes. + +Java to Python +-------------- + +See :external+arrow:doc:`python/integration/python_java` to implement Java to +Python communication using the C Data Interface. + +Java to C++ +----------- + +See :external+arrow:doc:`developers/cpp/building` to build the Arrow C++ libraries: + +.. code-block:: shell + + $ git clone https://github.com/apache/arrow.git + $ cd arrow/cpp + $ mkdir build # from inside the `cpp` subdirectory + $ cd build + $ cmake .. --preset ninja-debug-minimal + $ cmake --build . + $ tree debug/ + debug/ + ├── libarrow.800.0.0.dylib + ├── libarrow.800.dylib -> libarrow.800.0.0.dylib + └── libarrow.dylib -> libarrow.800.dylib + +Share an Int64 array from C++ to Java +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**C++ Side** + +Implement a function in CDataCppBridge.h that exports an array via the C Data Interface: + +.. code-block:: cpp + + #include + #include + #include + + void FillInt64Array(const uintptr_t c_schema_ptr, const uintptr_t c_array_ptr) { + arrow::Int64Builder builder; + builder.Append(1); + builder.Append(2); + builder.Append(3); + builder.AppendNull(); + builder.Append(5); + builder.Append(6); + builder.Append(7); + builder.Append(8); + builder.Append(9); + builder.Append(10); + std::shared_ptr array = *builder.Finish(); + + struct ArrowSchema* c_schema = reinterpret_cast(c_schema_ptr); + auto c_schema_status = arrow::ExportType(*array->type(), c_schema); + if (!c_schema_status.ok()) c_schema_status.Abort(); + + struct ArrowArray* c_array = reinterpret_cast(c_array_ptr); + auto c_array_status = arrow::ExportArray(*array, c_array); + if (!c_array_status.ok()) c_array_status.Abort(); + } + +**Java Side** + +For this example, we will use `JavaCPP`_ to call our C++ function from Java, +without writing JNI bindings ourselves. + +.. code-block:: xml + + + + 4.0.0 + + org.example + java-cdata-example + 1.0-SNAPSHOT + + + 8 + 8 + 9.0.0 + + + + org.bytedeco + javacpp + 1.5.7 + + + org.apache.arrow + arrow-c-data + ${arrow.version} + + + org.apache.arrow + arrow-vector + ${arrow.version} + + + org.apache.arrow + arrow-memory-core + ${arrow.version} + + + org.apache.arrow + arrow-memory-netty + ${arrow.version} + + + org.apache.arrow + arrow-format + ${arrow.version} + + + + +.. code-block:: java + + import org.bytedeco.javacpp.annotation.Platform; + import org.bytedeco.javacpp.annotation.Properties; + import org.bytedeco.javacpp.tools.InfoMap; + import org.bytedeco.javacpp.tools.InfoMapper; + + @Properties( + target = "CDataJavaToCppExample", + value = @Platform( + include = { + "CDataCppBridge.h" + }, + compiler = {"cpp17"}, + linkpath = {"/arrow/cpp/build/debug/"}, + link = {"arrow"} + ) + ) + public class CDataJavaConfig implements InfoMapper { + + @Override + public void map(InfoMap infoMap) { + } + } + +.. code-block:: shell + + # Compile our Java code + $ javac -cp javacpp-1.5.7.jar CDataJavaConfig.java + + # Generate CDataInterfaceLibrary + $ java -jar javacpp-1.5.7.jar CDataJavaConfig.java + + # Generate libjniCDataInterfaceLibrary.dylib + $ java -jar javacpp-1.5.7.jar CDataJavaToCppExample.java + + # Validate libjniCDataInterfaceLibrary.dylib created + $ otool -L macosx-x86_64/libjniCDataJavaToCppExample.dylib + macosx-x86_64/libjniCDataJavaToCppExample.dylib: + libjniCDataJavaToCppExample.dylib (compatibility version 0.0.0, current version 0.0.0) + @rpath/libarrow.800.dylib (compatibility version 800.0.0, current version 800.0.0) + /usr/lib/libc++.1.dylib (compatibility version 1.0.0, current version 1200.3.0) + /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1311.0.0) + +**Java Test** + +Let's create a Java class to test our bridge: + +.. code-block:: java + + import org.apache.arrow.c.ArrowArray; + import org.apache.arrow.c.ArrowSchema; + import org.apache.arrow.c.Data; + import org.apache.arrow.memory.BufferAllocator; + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.BigIntVector; + + public class TestCDataInterface { + public static void main(String[] args) { + try( + BufferAllocator allocator = new RootAllocator(); + ArrowSchema arrowSchema = ArrowSchema.allocateNew(allocator); + ArrowArray arrowArray = ArrowArray.allocateNew(allocator) + ){ + CDataJavaToCppExample.FillInt64Array( + arrowSchema.memoryAddress(), arrowArray.memoryAddress()); + try( + BigIntVector bigIntVector = (BigIntVector) Data.importVector( + allocator, arrowArray, arrowSchema, null) + ){ + System.out.println("C++-allocated array: " + bigIntVector); + } + } + } + } + +.. code-block:: shell + + C++-allocated array: [1, 2, 3, null, 5, 6, 7, 8, 9, 10] + +Share an Int32 array from Java to C++ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Java Side** + +For this example, we will build a JAR with all dependencies bundled. + +.. code-block:: xml + + + + 4.0.0 + org.example + cpptojava + 1.0-SNAPSHOT + + 8 + 8 + 9.0.0 + + + + org.apache.arrow + arrow-c-data + ${arrow.version} + + + org.apache.arrow + arrow-memory-netty + ${arrow.version} + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + package + + single + + + + jar-with-dependencies + + + + + + + + + +.. code-block:: java + + import org.apache.arrow.c.ArrowArray; + import org.apache.arrow.c.ArrowSchema; + import org.apache.arrow.c.Data; + import org.apache.arrow.memory.BufferAllocator; + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.FieldVector; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VectorSchemaRoot; + + import java.util.Arrays; + + public class ToBeCalledByCpp { + final static BufferAllocator allocator = new RootAllocator(); + + /** + * Create a {@link FieldVector} and export it via the C Data Interface + * @param schemaAddress Schema memory address to wrap + * @param arrayAddress Array memory address to wrap + */ + public static void fillVector(long schemaAddress, long arrayAddress){ + try (ArrowArray arrow_array = ArrowArray.wrap(arrayAddress); + ArrowSchema arrow_schema = ArrowSchema.wrap(schemaAddress) ) { + Data.exportVector(allocator, populateFieldVectorToExport(), null, arrow_array, arrow_schema); + } + } + + /** + * Create a {@link VectorSchemaRoot} and export it via the C Data Interface + * @param schemaAddress Schema memory address to wrap + * @param arrayAddress Array memory address to wrap + */ + public static void fillVectorSchemaRoot(long schemaAddress, long arrayAddress){ + try (ArrowArray arrow_array = ArrowArray.wrap(arrayAddress); + ArrowSchema arrow_schema = ArrowSchema.wrap(schemaAddress) ) { + Data.exportVectorSchemaRoot(allocator, populateVectorSchemaRootToExport(), null, arrow_array, arrow_schema); + } + } + + private static FieldVector populateFieldVectorToExport(){ + IntVector intVector = new IntVector("int-to-export", allocator); + intVector.allocateNew(3); + intVector.setSafe(0, 1); + intVector.setSafe(1, 2); + intVector.setSafe(2, 3); + intVector.setValueCount(3); + System.out.println("[Java] FieldVector: \n" + intVector); + return intVector; + } + + private static VectorSchemaRoot populateVectorSchemaRootToExport(){ + IntVector intVector = new IntVector("age-to-export", allocator); + intVector.setSafe(0, 10); + intVector.setSafe(1, 20); + intVector.setSafe(2, 30); + VectorSchemaRoot root = new VectorSchemaRoot(Arrays.asList(intVector)); + root.setRowCount(3); + System.out.println("[Java] VectorSchemaRoot: \n" + root.contentToTSVString()); + return root; + } + } + +Build the JAR and copy it to the C++ project. + +.. code-block:: shell + + $ mvn clean install + $ cp target/cpptojava-1.0-SNAPSHOT-jar-with-dependencies.jar /cpptojava.jar + +**C++ Side** + +This application uses JNI to call Java code, but transfers data (zero-copy) via the C Data Interface instead. + +.. code-block:: cpp + + #include + #include + + #include + #include + + JNIEnv *CreateVM(JavaVM **jvm) { + JNIEnv *env; + JavaVMInitArgs vm_args; + JavaVMOption options[2]; + options[0].optionString = "-Djava.class.path=cpptojava.jar"; + options[1].optionString = "-DXcheck:jni:pedantic"; + vm_args.version = JNI_VERSION_10; + vm_args.nOptions = 2; + vm_args.options = options; + int status = JNI_CreateJavaVM(jvm, (void **) &env, &vm_args); + if (status < 0) { + std::cerr << "\n<<<<< Unable to Launch JVM >>>>>\n" << std::endl; + return nullptr; + } + return env; + } + + int main() { + JNIEnv *env; + JavaVM *jvm; + env = CreateVM(&jvm); + if (env == nullptr) return EXIT_FAILURE; + jclass javaClassToBeCalledByCpp = env->FindClass("ToBeCalledByCpp"); + if (javaClassToBeCalledByCpp != nullptr) { + jmethodID fillVector = env->GetStaticMethodID(javaClassToBeCalledByCpp, + "fillVector", + "(JJ)V"); + if (fillVector != nullptr) { + struct ArrowSchema arrowSchema; + struct ArrowArray arrowArray; + std::cout << "\n<<<<< C++ to Java for Arrays >>>>>\n" << std::endl; + env->CallStaticVoidMethod(javaClassToBeCalledByCpp, fillVector, + static_cast(reinterpret_cast(&arrowSchema)), + static_cast(reinterpret_cast(&arrowArray))); + auto resultImportArray = arrow::ImportArray(&arrowArray, &arrowSchema); + std::shared_ptr array = resultImportArray.ValueOrDie(); + std::cout << "[C++] Array: " << array->ToString() << std::endl; + } else { + std::cerr << "Could not find fillVector method\n" << std::endl; + return EXIT_FAILURE; + } + jmethodID fillVectorSchemaRoot = env->GetStaticMethodID(javaClassToBeCalledByCpp, + "fillVectorSchemaRoot", + "(JJ)V"); + if (fillVectorSchemaRoot != nullptr) { + struct ArrowSchema arrowSchema; + struct ArrowArray arrowArray; + std::cout << "\n<<<<< C++ to Java for RecordBatch >>>>>\n" << std::endl; + env->CallStaticVoidMethod(javaClassToBeCalledByCpp, fillVectorSchemaRoot, + static_cast(reinterpret_cast(&arrowSchema)), + static_cast(reinterpret_cast(&arrowArray))); + auto resultImportVectorSchemaRoot = arrow::ImportRecordBatch(&arrowArray, &arrowSchema); + std::shared_ptr recordBatch = resultImportVectorSchemaRoot.ValueOrDie(); + std::cout << "[C++] RecordBatch: " << recordBatch->ToString() << std::endl; + } else { + std::cerr << "Could not find fillVectorSchemaRoot method\n" << std::endl; + return EXIT_FAILURE; + } + } else { + std::cout << "Could not find ToBeCalledByCpp class\n" << std::endl; + return EXIT_FAILURE; + } + jvm->DestroyJavaVM(); + return EXIT_SUCCESS; + } + +CMakeLists.txt definition file: + +.. code-block:: cmake + + cmake_minimum_required(VERSION 3.19) + project(cdatacpptojava) + find_package(JNI REQUIRED) + find_package(Arrow REQUIRED) + message(STATUS "Arrow version: ${ARROW_VERSION}") + include_directories(${JNI_INCLUDE_DIRS}) + set(CMAKE_CXX_STANDARD 17) + add_executable(${PROJECT_NAME} main.cpp) + target_link_libraries(cdatacpptojava PRIVATE Arrow::arrow_shared) + target_link_libraries(cdatacpptojava PRIVATE ${JNI_LIBRARIES}) + +**Result** + +.. code-block:: text + + <<<<< C++ to Java for Arrays >>>>> + [Java] FieldVector: + [1, 2, 3] + [C++] Array: [ + 1, + 2, + 3 + ] + + <<<<< C++ to Java for RecordBatch >>>>> + [Java] VectorSchemaRoot: + age-to-export + 10 + 20 + 30 + + [C++] RecordBatch: age-to-export: [ + 10, + 20, + 30 + ] + +.. _`JavaCPP`: https://github.com/bytedeco/javacpp diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000000..166a3bc4ea --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = 'arrow-java' +copyright = '2025, Apache Arrow Developers' +author = 'Apache Arrow Developers' +release = '18.1.0' + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = ["sphinx.ext.intersphinx"] + +templates_path = ['_templates'] +exclude_patterns = [] + +# -- Intersphinx ------------------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html + +intersphinx_mapping = { + 'arrow': ('https://arrow.apache.org/docs/', None), + 'cookbook': ('https://arrow.apache.org/cookbook/java/', None), +} + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'furo' +html_static_path = ['_static'] diff --git a/docs/source/dataset.rst b/docs/source/dataset.rst new file mode 100644 index 0000000000..deaa009576 --- /dev/null +++ b/docs/source/dataset.rst @@ -0,0 +1,309 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +======= +Dataset +======= + +.. warning:: + + Experimental: The Java module ``dataset`` is currently under early + development. API might be changed in each release of Apache Arrow until it + gets mature. + +Dataset is an universal layer in Apache Arrow for querying data in different +formats or in different partitioning strategies. Usually the data to be queried +is supposed to be located from a traditional file system, however Arrow Dataset +is not designed only for querying files but can be extended to serve all +possible data sources such as from inter-process communication or from other +network locations, etc. + +Getting Started +=============== + +Currently supported file formats are: + +- Apache Arrow (``.arrow``) +- Apache ORC (``.orc``) +- Apache Parquet (``.parquet``) +- Comma-Separated Values (``.csv``) +- Line-delimited JSON Values (``.json``) + +Below shows a simplest example of using Dataset to query a Parquet file in Java: + +.. code-block:: Java + + // read data from file /opt/example.parquet + String uri = "file:/opt/example.parquet"; + ScanOptions options = new ScanOptions(/*batchSize*/ 32768); + try ( + BufferAllocator allocator = new RootAllocator(); + DatasetFactory datasetFactory = new FileSystemDatasetFactory( + allocator, NativeMemoryPool.getDefault(), + FileFormat.PARQUET, uri); + Dataset dataset = datasetFactory.finish(); + Scanner scanner = dataset.newScan(options); + ArrowReader reader = scanner.scanBatches() + ) { + List batches = new ArrayList<>(); + while (reader.loadNextBatch()) { + try (VectorSchemaRoot root = reader.getVectorSchemaRoot()) { + final VectorUnloader unloader = new VectorUnloader(root); + batches.add(unloader.getRecordBatch()); + } + } + + // do something with read record batches, for example: + analyzeArrowData(batches); + + // finished the analysis of the data, close all resources: + AutoCloseables.close(batches); + } catch (Exception e) { + e.printStackTrace(); + } + +.. note:: + ``ArrowRecordBatch`` is a low-level composite Arrow data exchange format + that doesn't provide API to read typed data from it directly. + It's recommended to use utilities ``VectorLoader`` to load it into a schema + aware container ``VectorSchemaRoot`` by which user could be able to access + decoded data conveniently in Java. + + The ``ScanOptions batchSize`` argument takes effect only if it is set to a value + smaller than the number of rows in the recordbatch. + +.. seealso:: + Load record batches with :doc:`VectorSchemaRoot `. + +Schema +====== + +Schema of the data to be queried can be inspected via method +``DatasetFactory#inspect()`` before actually reading it. For example: + +.. code-block:: Java + + // read data from local file /opt/example.parquet + String uri = "file:/opt/example.parquet"; + BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + DatasetFactory factory = new FileSystemDatasetFactory(allocator, + NativeMemoryPool.getDefault(), FileFormat.PARQUET, uri); + + // inspect schema + Schema schema = factory.inspect(); + +For some of the data format that is compatible with a user-defined schema, user +can use method ``DatasetFactory#inspect(Schema schema)`` to create the dataset: + +.. code-block:: Java + + Schema schema = createUserSchema() + Dataset dataset = factory.finish(schema); + +Otherwise when the non-parameter method ``DatasetFactory#inspect()`` is called, +schema will be inferred automatically from data source. The same as the result +of ``DatasetFactory#inspect()``. + +Also, if projector is specified during scanning (see next section +:ref:`java-dataset-projection`), the actual schema of output data can be got +within method ``Scanner::schema()``: + +.. code-block:: Java + + Scanner scanner = dataset.newScan( + new ScanOptions(32768, Optional.of(new String[] {"id", "name"}))); + Schema projectedSchema = scanner.schema(); + +.. _java-dataset-projection: + +Projection (Subset of Columns) +============================== + +User can specify projections in ScanOptions. For example: + +.. code-block:: Java + + String[] projection = new String[] {"id", "name"}; + ScanOptions options = new ScanOptions(32768, Optional.of(projection)); + +If no projection is needed, leave the optional projection argument absent in +ScanOptions: + +.. code-block:: Java + + ScanOptions options = new ScanOptions(32768, Optional.empty()); + +Or use shortcut constructor: + +.. code-block:: Java + + ScanOptions options = new ScanOptions(32768); + +Then all columns will be emitted during scanning. + +Projection (Produce New Columns) and Filters +============================================ + +User can specify projections (new columns) or filters in ScanOptions using Substrait. For example: + +.. code-block:: Java + + ByteBuffer substraitExpressionFilter = getSubstraitExpressionFilter(); + ByteBuffer substraitExpressionProject = getSubstraitExpressionProjection(); + // Use Substrait APIs to create an Expression and serialize to a ByteBuffer + ScanOptions options = new ScanOptions.Builder(/*batchSize*/ 32768) + .columns(Optional.empty()) + .substraitExpressionFilter(substraitExpressionFilter) + .substraitExpressionProjection(getSubstraitExpressionProjection()) + .build(); + +.. seealso:: + + :doc:`Executing Projections and Filters Using Extended Expressions ` + Projections and Filters using Substrait. + +Read Data from HDFS +=================== + +``FileSystemDataset`` supports reading data from non-local file systems. HDFS +support is included in the official Apache Arrow Java package releases and +can be used directly without re-building the source code. + +To access HDFS data using Dataset API, pass a general HDFS URI to +``FilesSystemDatasetFactory``: + +.. code-block:: Java + + String uri = "hdfs://{hdfs_host}:{port}/data/example.parquet"; + BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + DatasetFactory factory = new FileSystemDatasetFactory(allocator, + NativeMemoryPool.getDefault(), FileFormat.PARQUET, uri); + +Native Memory Management +======================== + +To gain better performance and reduce code complexity, Java +``FileSystemDataset`` internally relies on C++ +``arrow::dataset::FileSystemDataset`` via JNI. +As a result, all Arrow data read from ``FileSystemDataset`` is supposed to be +allocated off the JVM heap. To manage this part of memory, an utility class +``NativeMemoryPool`` is provided to users. + +As a basic example, by using a listenable ``NativeMemoryPool``, user can pass +a listener hooking on C++ buffer allocation/deallocation: + +.. code-block:: Java + + AtomicLong reserved = new AtomicLong(0L); + ReservationListener listener = new ReservationListener() { + @Override + public void reserve(long size) { + reserved.getAndAdd(size); + } + + @Override + public void unreserve(long size) { + reserved.getAndAdd(-size); + } + }; + NativeMemoryPool pool = NativeMemoryPool.createListenable(listener); + FileSystemDatasetFactory factory = new FileSystemDatasetFactory(allocator, + pool, FileFormat.PARQUET, uri); + + +Also, it's a very common case to reserve the same amount of JVM direct memory +for the data read from datasets. For this use a built-in utility +class ``DirectReservationListener`` is provided: + +.. code-block:: Java + + NativeMemoryPool pool = NativeMemoryPool.createListenable( + DirectReservationListener.instance()); + +This way, once the allocated byte count of Arrow buffers reaches the limit of +JVM direct memory, ``OutOfMemoryError: Direct buffer memory`` will +be thrown during scanning. + +.. note:: + The default instance ``NativeMemoryPool.getDefaultMemoryPool()`` does + nothing on buffer allocation/deallocation. It's OK to use it in + the case of POC or testing, but for production use in complex environment, + it's recommended to manage memory by using a listenable memory pool. + +.. note:: + The ``BufferAllocator`` instance passed to ``FileSystemDatasetFactory``'s + constructor is also aware of the overall memory usage of the produced + dataset instances. Once the Java buffers are created the passed allocator + will become their parent allocator. + +Usage Notes +=========== + +Native Object Resource Management +--------------------------------- + +As another result of relying on JNI, all components related to +``FileSystemDataset`` should be closed manually or use try-with-resources to +release the corresponding native objects after using. For example: + +.. code-block:: Java + + String uri = "file:/opt/example.parquet"; + ScanOptions options = new ScanOptions(/*batchSize*/ 32768); + try ( + BufferAllocator allocator = new RootAllocator(); + DatasetFactory factory = new FileSystemDatasetFactory( + allocator, NativeMemoryPool.getDefault(), + FileFormat.PARQUET, uri); + Dataset dataset = factory.finish(); + Scanner scanner = dataset.newScan(options) + ) { + + // do something + + } catch (Exception e) { + e.printStackTrace(); + } + +If user forgets to close them then native object leakage might be caused. + +BatchSize +--------- + +The ``batchSize`` argument of ``ScanOptions`` is a limit on the size of an individual batch. + +For example, let's try to read a Parquet file with gzip compression and 3 row groups: + +.. code-block:: + + # Let configure ScanOptions as: + ScanOptions options = new ScanOptions(/*batchSize*/ 32768); + + $ parquet-tools meta data4_3rg_gzip.parquet + file schema: schema + age: OPTIONAL INT64 R:0 D:1 + name: OPTIONAL BINARY L:STRING R:0 D:1 + row group 1: RC:4 TS:182 OFFSET:4 + row group 2: RC:4 TS:190 OFFSET:420 + row group 3: RC:3 TS:179 OFFSET:838 + +Here, we set the batchSize in ScanOptions to 32768. Because that's greater +than the number of rows in the next batch, which is 4 rows because the first +row group has only 4 rows, then the program gets only 4 rows. The scanner +will not combine smaller batches to reach the limit, but it will split +large batches to stay under the limit. So in the case the row group had more +than 32768 rows, it would get split into blocks of 32768 rows or less. diff --git a/docs/source/developers/building.rst b/docs/source/developers/building.rst new file mode 100644 index 0000000000..f9ef7daea8 --- /dev/null +++ b/docs/source/developers/building.rst @@ -0,0 +1,624 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +.. highlight:: console + +.. _building-arrow-java: + +=================== +Building Arrow Java +=================== + +.. contents:: + +System Setup +============ + +Arrow Java uses the `Maven `_ build system. + +Building requires: + +* JDK 11+ +* Maven 3+ + +.. note:: + CI will test all supported JDK LTS versions, plus the latest non-LTS version. + +Building +======== + +All the instructions below assume that you have cloned the Arrow git +repository: + +.. code-block:: + + $ git clone https://github.com/apache/arrow.git + $ cd arrow + $ git submodule update --init --recursive + +These are the options available to compile Arrow Java modules with: + +* Maven build tool. +* Docker Compose. +* Archery. + +Building Java Modules +--------------------- + +To build the default modules, go to the project root and execute: + +Maven +~~~~~ + +.. code-block:: + + $ cd arrow/java + $ export JAVA_HOME= + $ java --version + $ mvn clean install + +Docker compose +~~~~~~~~~~~~~~ + +.. code-block:: + + $ cd arrow/java + $ export JAVA_HOME= + $ java --version + $ docker compose run java + +Archery +~~~~~~~ + +.. code-block:: + + $ cd arrow/java + $ export JAVA_HOME= + $ java --version + $ archery docker run java + +Building JNI Libraries (\*.dylib / \*.so / \*.dll) +-------------------------------------------------- + +First, we need to build the `C++ shared libraries`_ that the JNI bindings will use. +We can build these manually or we can use `Archery`_ to build them using a Docker container +(This will require installing Docker, Docker Compose, and Archery). + +.. note:: + If you are building on Apple Silicon, be sure to use a JDK version that was compiled + for that architecture. See, for example, the `Azul JDK `_. + + If you are building on Windows OS, see :ref:`Developing on Windows `. + +Maven +~~~~~ + +- To build only the JNI C Data Interface library (macOS / Linux): + + .. code-block:: text + + $ cd arrow/java + $ export JAVA_HOME= + $ java --version + $ mvn generate-resources -Pgenerate-libs-cdata-all-os -N + $ ls -latr ../java-dist/lib + |__ arrow_cdata_jni/ + +- To build only the JNI C Data Interface library (Windows): + + .. code-block:: + + $ cd arrow/java + $ mvn generate-resources -Pgenerate-libs-cdata-all-os -N + $ dir "../java-dist/bin" + |__ arrow_cdata_jni/ + +- To build all JNI libraries (macOS / Linux) except the JNI C Data Interface library: + + .. code-block:: text + + $ cd arrow/java + $ export JAVA_HOME= + $ java --version + $ mvn generate-resources -Pgenerate-libs-jni-macos-linux -N + $ ls -latr java-dist/lib + |__ arrow_dataset_jni/ + |__ arrow_orc_jni/ + |__ gandiva_jni/ + +- To build all JNI libraries (Windows) except the JNI C Data Interface library: + + .. code-block:: + + $ cd arrow/java + $ mvn generate-resources -Pgenerate-libs-jni-windows -N + $ dir "../java-dist/bin" + |__ arrow_dataset_jni/ + +CMake +~~~~~ + +- To build only the JNI C Data Interface library (macOS / Linux): + + .. code-block:: text + + $ cd arrow + $ mkdir -p java-dist java-cdata + $ cmake \ + -S java \ + -B java-cdata \ + -DARROW_JAVA_JNI_ENABLE_C=ON \ + -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF \ + -DBUILD_TESTING=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=java-dist + $ cmake --build java-cdata --target install --config Release + $ ls -latr java-dist/lib + |__ arrow_cdata_jni/ + +- To build only the JNI C Data Interface library (Windows): + + .. code-block:: text + + $ cd arrow + $ mkdir java-dist, java-cdata + $ cmake ^ + -S java ^ + -B java-cdata ^ + -DARROW_JAVA_JNI_ENABLE_C=ON ^ + -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF ^ + -DBUILD_TESTING=OFF ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DCMAKE_INSTALL_PREFIX=java-dist + $ cmake --build java-cdata --target install --config Release + $ dir "java-dist/bin" + |__ arrow_cdata_jni/ + +- To build all JNI libraries (macOS / Linux) except the JNI C Data Interface library: + + .. code-block:: text + + $ cd arrow + $ brew bundle --file=cpp/Brewfile + # Homebrew Bundle complete! 25 Brewfile dependencies now installed. + $ brew uninstall aws-sdk-cpp + # (We can't use aws-sdk-cpp installed by Homebrew because it has + # an issue: https://github.com/aws/aws-sdk-cpp/issues/1809 ) + $ export JAVA_HOME= + $ mkdir -p java-dist cpp-jni + $ cmake \ + -S cpp \ + -B cpp-jni \ + -DARROW_BUILD_SHARED=OFF \ + -DARROW_CSV=ON \ + -DARROW_DATASET=ON \ + -DARROW_DEPENDENCY_SOURCE=BUNDLED \ + -DARROW_DEPENDENCY_USE_SHARED=OFF \ + -DARROW_FILESYSTEM=ON \ + -DARROW_GANDIVA=ON \ + -DARROW_GANDIVA_STATIC_LIBSTDCPP=ON \ + -DARROW_JSON=ON \ + -DARROW_ORC=ON \ + -DARROW_PARQUET=ON \ + -DARROW_S3=ON \ + -DARROW_SUBSTRAIT=ON \ + -DARROW_USE_CCACHE=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=java-dist \ + -DCMAKE_UNITY_BUILD=ON + $ cmake --build cpp-jni --target install --config Release + $ cmake \ + -S java \ + -B java-jni \ + -DARROW_JAVA_JNI_ENABLE_C=OFF \ + -DARROW_JAVA_JNI_ENABLE_DEFAULT=ON \ + -DBUILD_TESTING=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=java-dist \ + -DCMAKE_PREFIX_PATH=$PWD/java-dist \ + -DProtobuf_ROOT=$PWD/../cpp-jni/protobuf_ep-install \ + -DProtobuf_USE_STATIC_LIBS=ON + $ cmake --build java-jni --target install --config Release + $ ls -latr java-dist/lib/ + |__ arrow_dataset_jni/ + |__ arrow_orc_jni/ + |__ gandiva_jni/ + +- To build all JNI libraries (Windows) except the JNI C Data Interface library: + + .. code-block:: + + $ cd arrow + $ mkdir java-dist, cpp-jni + $ cmake ^ + -S cpp ^ + -B cpp-jni ^ + -DARROW_BUILD_SHARED=OFF ^ + -DARROW_CSV=ON ^ + -DARROW_DATASET=ON ^ + -DARROW_DEPENDENCY_USE_SHARED=OFF ^ + -DARROW_FILESYSTEM=ON ^ + -DARROW_GANDIVA=OFF ^ + -DARROW_JSON=ON ^ + -DARROW_ORC=ON ^ + -DARROW_PARQUET=ON ^ + -DARROW_S3=ON ^ + -DARROW_SUBSTRAIT=ON ^ + -DARROW_USE_CCACHE=ON ^ + -DARROW_WITH_BROTLI=ON ^ + -DARROW_WITH_LZ4=ON ^ + -DARROW_WITH_SNAPPY=ON ^ + -DARROW_WITH_ZLIB=ON ^ + -DARROW_WITH_ZSTD=ON ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DCMAKE_INSTALL_PREFIX=java-dist ^ + -DCMAKE_UNITY_BUILD=ON ^ + -GNinja + $ cd cpp-jni + $ ninja install + $ cd ../ + $ cmake ^ + -S java ^ + -B java-jni ^ + -DARROW_JAVA_JNI_ENABLE_C=OFF ^ + -DARROW_JAVA_JNI_ENABLE_DATASET=ON ^ + -DARROW_JAVA_JNI_ENABLE_DEFAULT=ON ^ + -DARROW_JAVA_JNI_ENABLE_GANDIVA=OFF ^ + -DARROW_JAVA_JNI_ENABLE_ORC=ON ^ + -DBUILD_TESTING=OFF ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DCMAKE_INSTALL_PREFIX=java-dist ^ + -DCMAKE_PREFIX_PATH=$PWD/java-dist + $ cmake --build java-jni --target install --config Release + $ dir "java-dist/bin" + |__ arrow_orc_jni/ + |__ arrow_dataset_jni/ + +Archery +~~~~~~~ + +.. code-block:: text + + $ cd arrow + $ archery docker run java-jni-manylinux-2014 + $ ls -latr java-dist + |__ arrow_cdata_jni/ + |__ arrow_dataset_jni/ + |__ arrow_orc_jni/ + |__ gandiva_jni/ + +Building Java JNI Modules +------------------------- + +- To compile the JNI bindings, use the ``arrow-c-data`` Maven profile: + + .. code-block:: + + $ cd arrow/java + $ mvn -Darrow.c.jni.dist.dir=/java-dist/lib -Parrow-c-data clean install + +- To compile the JNI bindings for ORC / Gandiva / Dataset, use the ``arrow-jni`` Maven profile: + + .. code-block:: + + $ cd arrow/java + $ mvn \ + -Darrow.cpp.build.dir=/java-dist/lib/ \ + -Darrow.c.jni.dist.dir=/java-dist/lib/ \ + -Parrow-jni clean install + +Testing +======= + +By default, Maven uses the same Java version to both build the code and run the tests. + +It is also possible to use a different JDK version for the tests. This requires Maven +toolchains to be configured beforehand, and then a specific test property needs to be set. + +Configuring Maven toolchains +---------------------------- + +To be able to use a JDK version for testing, it needs to be registered first in Maven ``toolchains.xml`` +configuration file usually located under ``${HOME}/.m2`` with the following snippet added to it: + + .. code-block:: + + + + + [...] + + + jdk + + 21 + temurin + + + path/to/jdk/home + + + + [...] + + + +Testing with a specific JDK +--------------------------- + +To run Arrow tests with a specific JDK version, use the ``arrow.test.jdk-version`` property. + +For example, to run Arrow tests with JDK 17, use the following snippet: + + .. code-block:: + + $ cd arrow/java + $ mvn -Darrow.test.jdk-version=17 clean verify + +IDE Configuration +================= + +IntelliJ +-------- + +To start working on Arrow in IntelliJ: build the project once from the command +line using ``mvn clean install``. Then open the ``java/`` subdirectory of the +Arrow repository, and update the following settings: + +* In the Files tool window, find the path ``vector/target/generated-sources``, + right click the directory, and select Mark Directory as > Generated Sources + Root. There is no need to mark other generated sources directories, as only + the ``vector`` module generates sources. +* For JDK 11, due to an `IntelliJ bug + `__, you must go into + Settings > Build, Execution, Deployment > Compiler > Java Compiler and disable + "Use '--release' option for cross-compilation (Java 9 and later)". Otherwise + you will get an error like "package sun.misc does not exist". +* You may want to disable error-prone entirely if it gives spurious + warnings (disable both error-prone profiles in the Maven tool window + and "Reload All Maven Projects"). +* If using IntelliJ's Maven integration to build, you may need to change + ```` to ``false`` in the pom.xml files due to an `IntelliJ bug + `__. +* To enable debugging JNI-based modules like ``dataset``, + activate specific profiles in the Maven tab under "Profiles". + Ensure the profiles ``arrow-c-data``, ``arrow-jni``, ``generate-libs-cdata-all-os``, + ``generate-libs-jni-macos-linux``, and ``jdk11+`` are enabled, so that the + IDE can build them and enable debugging. + +You may not need to update all of these settings if you build/test with the +IntelliJ Maven integration instead of with IntelliJ directly. + +Common Errors +============= + +* When working with the JNI code: if the C++ build cannot find dependencies, with errors like these: + + .. code-block:: + + Could NOT find Boost (missing: Boost_INCLUDE_DIR system filesystem) + Could NOT find Lz4 (missing: LZ4_LIB) + Could NOT find zstd (missing: ZSTD_LIB) + + Specify that the dependencies should be downloaded at build time (more details at `Dependency Resolution`_): + + .. code-block:: + + -Dre2_SOURCE=BUNDLED \ + -DBoost_SOURCE=BUNDLED \ + -Dutf8proc_SOURCE=BUNDLED \ + -DSnappy_SOURCE=BUNDLED \ + -DORC_SOURCE=BUNDLED \ + -DZLIB_SOURCE=BUNDLED + +.. _Archery: https://github.com/apache/arrow/blob/main/dev/archery/README.md +.. _Dependency Resolution: https://arrow.apache.org/docs/developers/cpp/building.html#individual-dependency-resolution +.. _C++ shared libraries: https://arrow.apache.org/docs/cpp/build_system.html + + +Installing Nightly Packages +=========================== + +.. warning:: + These packages are not official releases. Use them at your own risk. + +Arrow nightly builds are posted on the mailing list at `builds@arrow.apache.org`_. +The artifacts are uploaded to GitHub. For example, for 2022/07/30, they can be found at `GitHub Nightly`_. + + +Installing from Apache Nightlies +-------------------------------- +1. Look up the nightly version number for the Arrow libraries used. + + For example, for ``arrow-memory``, visit https://nightlies.apache.org/arrow/java/org/apache/arrow/arrow-memory/ and see what versions are available (e.g. 9.0.0.dev501). +2. Add Apache Nightlies Repository to the Maven/Gradle project. + + .. code-block:: xml + + + 9.0.0.dev501 + + ... + + + arrow-apache-nightlies + https://nightlies.apache.org/arrow/java + + + ... + + + org.apache.arrow + arrow-vector + ${arrow.version} + + + ... + +Installing Manually +------------------- + +1. Decide nightly packages repository to use, for example: https://github.com/ursacomputing/crossbow/releases/tag/nightly-packaging-2022-07-30-0-github-java-jars +2. Add packages to your pom.xml, for example: flight-core (it depends on: arrow-format, arrow-vector, arrow-memory-core and arrow-memory-netty). + + .. code-block:: xml + + + 8 + 8 + 9.0.0.dev501 + + + + + org.apache.arrow + flight-core + ${arrow.version} + + + +3. Download the necessary pom and jar files to a temporary directory: + + .. code-block:: shell + + $ mkdir nightly-packaging-2022-07-30-0-github-java-jars + $ cd nightly-packaging-2022-07-30-0-github-java-jars + $ wget https://github.com/ursacomputing/crossbow/releases/download/nightly-packaging-2022-07-30-0-github-java-jars/arrow-java-root-9.0.0.dev501.pom + $ wget https://github.com/ursacomputing/crossbow/releases/download/nightly-packaging-2022-07-30-0-github-java-jars/arrow-format-9.0.0.dev501.pom + $ wget https://github.com/ursacomputing/crossbow/releases/download/nightly-packaging-2022-07-30-0-github-java-jars/arrow-format-9.0.0.dev501.jar + $ wget https://github.com/ursacomputing/crossbow/releases/download/nightly-packaging-2022-07-30-0-github-java-jars/arrow-vector-9.0.0.dev501.pom + $ wget https://github.com/ursacomputing/crossbow/releases/download/nightly-packaging-2022-07-30-0-github-java-jars/arrow-vector-9.0.0.dev501.jar + $ wget https://github.com/ursacomputing/crossbow/releases/download/nightly-packaging-2022-07-30-0-github-java-jars/arrow-memory-9.0.0.dev501.pom + $ wget https://github.com/ursacomputing/crossbow/releases/download/nightly-packaging-2022-07-30-0-github-java-jars/arrow-memory-core-9.0.0.dev501.pom + $ wget https://github.com/ursacomputing/crossbow/releases/download/nightly-packaging-2022-07-30-0-github-java-jars/arrow-memory-netty-9.0.0.dev501.pom + $ wget https://github.com/ursacomputing/crossbow/releases/download/nightly-packaging-2022-07-30-0-github-java-jars/arrow-memory-core-9.0.0.dev501.jar + $ wget https://github.com/ursacomputing/crossbow/releases/download/nightly-packaging-2022-07-30-0-github-java-jars/arrow-memory-netty-9.0.0.dev501.jar + $ wget https://github.com/ursacomputing/crossbow/releases/download/nightly-packaging-2022-07-30-0-github-java-jars/arrow-flight-9.0.0.dev501.pom + $ wget https://github.com/ursacomputing/crossbow/releases/download/nightly-packaging-2022-07-30-0-github-java-jars/flight-core-9.0.0.dev501.pom + $ wget https://github.com/ursacomputing/crossbow/releases/download/nightly-packaging-2022-07-30-0-github-java-jars/flight-core-9.0.0.dev501.jar + $ tree + . + ├── arrow-flight-9.0.0.dev501.pom + ├── arrow-format-9.0.0.dev501.jar + ├── arrow-format-9.0.0.dev501.pom + ├── arrow-java-root-9.0.0.dev501.pom + ├── arrow-memory-9.0.0.dev501.pom + ├── arrow-memory-core-9.0.0.dev501.jar + ├── arrow-memory-core-9.0.0.dev501.pom + ├── arrow-memory-netty-9.0.0.dev501.jar + ├── arrow-memory-netty-9.0.0.dev501.pom + ├── arrow-vector-9.0.0.dev501.jar + ├── arrow-vector-9.0.0.dev501.pom + ├── flight-core-9.0.0.dev501.jar + └── flight-core-9.0.0.dev501.pom + +4. Install the artifacts to the local Maven repository with ``mvn install:install-file``: + + .. code-block:: shell + + $ mvn install:install-file -Dfile="$(pwd)/arrow-java-root-9.0.0.dev501.pom" -DgroupId=org.apache.arrow -DartifactId=arrow-java-root -Dversion=9.0.0.dev501 -Dpackaging=pom + $ mvn install:install-file -Dfile="$(pwd)/arrow-format-9.0.0.dev501.pom" -DgroupId=org.apache.arrow -DartifactId=arrow-format -Dversion=9.0.0.dev501 -Dpackaging=pom + $ mvn install:install-file -Dfile="$(pwd)/arrow-format-9.0.0.dev501.jar" -DgroupId=org.apache.arrow -DartifactId=arrow-format -Dversion=9.0.0.dev501 -Dpackaging=jar + $ mvn install:install-file -Dfile="$(pwd)/arrow-vector-9.0.0.dev501.pom" -DgroupId=org.apache.arrow -DartifactId=arrow-vector -Dversion=9.0.0.dev501 -Dpackaging=pom + $ mvn install:install-file -Dfile="$(pwd)/arrow-vector-9.0.0.dev501.jar" -DgroupId=org.apache.arrow -DartifactId=arrow-vector -Dversion=9.0.0.dev501 -Dpackaging=jar + $ mvn install:install-file -Dfile="$(pwd)/arrow-memory-9.0.0.dev501.pom" -DgroupId=org.apache.arrow -DartifactId=arrow-memory -Dversion=9.0.0.dev501 -Dpackaging=pom + $ mvn install:install-file -Dfile="$(pwd)/arrow-memory-core-9.0.0.dev501.pom" -DgroupId=org.apache.arrow -DartifactId=arrow-memory-core -Dversion=9.0.0.dev501 -Dpackaging=pom + $ mvn install:install-file -Dfile="$(pwd)/arrow-memory-netty-9.0.0.dev501.pom" -DgroupId=org.apache.arrow -DartifactId=arrow-memory-netty -Dversion=9.0.0.dev501 -Dpackaging=pom + $ mvn install:install-file -Dfile="$(pwd)/arrow-memory-core-9.0.0.dev501.jar" -DgroupId=org.apache.arrow -DartifactId=arrow-memory-core -Dversion=9.0.0.dev501 -Dpackaging=jar + $ mvn install:install-file -Dfile="$(pwd)/arrow-memory-netty-9.0.0.dev501.jar" -DgroupId=org.apache.arrow -DartifactId=arrow-memory-netty -Dversion=9.0.0.dev501 -Dpackaging=jar + $ mvn install:install-file -Dfile="$(pwd)/arrow-flight-9.0.0.dev501.pom" -DgroupId=org.apache.arrow -DartifactId=arrow-flight -Dversion=9.0.0.dev501 -Dpackaging=pom + $ mvn install:install-file -Dfile="$(pwd)/flight-core-9.0.0.dev501.pom" -DgroupId=org.apache.arrow -DartifactId=flight-core -Dversion=9.0.0.dev501 -Dpackaging=pom + $ mvn install:install-file -Dfile="$(pwd)/flight-core-9.0.0.dev501.jar" -DgroupId=org.apache.arrow -DartifactId=flight-core -Dversion=9.0.0.dev501 -Dpackaging=jar + +5. Validate that the packages were installed: + + .. code-block:: shell + + $ tree ~/.m2/repository/org/apache/arrow + . + ├── arrow-flight + │   ├── 9.0.0.dev501 + │   │   └── arrow-flight-9.0.0.dev501.pom + ├── arrow-format + │   ├── 9.0.0.dev501 + │   │   ├── arrow-format-9.0.0.dev501.jar + │   │   └── arrow-format-9.0.0.dev501.pom + ├── arrow-java-root + │   ├── 9.0.0.dev501 + │   │   └── arrow-java-root-9.0.0.dev501.pom + ├── arrow-memory + │   ├── 9.0.0.dev501 + │   │   └── arrow-memory-9.0.0.dev501.pom + ├── arrow-memory-core + │   ├── 9.0.0.dev501 + │   │   ├── arrow-memory-core-9.0.0.dev501.jar + │   │   └── arrow-memory-core-9.0.0.dev501.pom + ├── arrow-memory-netty + │   ├── 9.0.0.dev501 + │   │   ├── arrow-memory-netty-9.0.0.dev501.jar + │   │   └── arrow-memory-netty-9.0.0.dev501.pom + ├── arrow-vector + │   ├── 9.0.0.dev501 + │   │   ├── _remote.repositories + │   │   ├── arrow-vector-9.0.0.dev501.jar + │   │   └── arrow-vector-9.0.0.dev501.pom + └── flight-core + ├── 9.0.0.dev501 + │   ├── flight-core-9.0.0.dev501.jar + │   └── flight-core-9.0.0.dev501.pom + +6. Compile your project like usual with ``mvn clean install``. + +.. _builds@arrow.apache.org: https://lists.apache.org/list.html?builds@arrow.apache.org +.. _GitHub Nightly: https://github.com/ursacomputing/crossbow/releases/tag/nightly-packaging-2022-07-30-0-github-java-jars + +Installing Staging Packages +=========================== + +.. warning:: + These packages are not official releases. Use them at your own risk. + +Arrow staging builds are created when a Release Candidate (RC) is being prepared. This allows users to test the RC in their applications before voting on the release. + + +Installing from Apache Staging +-------------------------------- +1. Look up the next version number for the Arrow libraries used. + +2. Add Apache Staging Repository to the Maven/Gradle project. + + .. code-block:: xml + + + 9.0.0 + + ... + + + arrow-apache-staging + https://repository.apache.org/content/repositories/staging + + + ... + + + org.apache.arrow + arrow-vector + ${arrow.version} + + + ... diff --git a/docs/source/developers/development.rst b/docs/source/developers/development.rst new file mode 100644 index 0000000000..dd1839257a --- /dev/null +++ b/docs/source/developers/development.rst @@ -0,0 +1,197 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +.. highlight:: console + +====================== +Development Guidelines +====================== + +.. contents:: + +Logger Abstraction +================== + +Apache Arrow Java uses the SLF4J API, so please configure SLF4J to see logs (e.g. via Logback/Apache Log4j): + +1. If no jar dependencies are added by the user via Logback or Apache Log4j then SLF4J will default + to no-operation (NOP) logging. + +2. If a user adds any dependencies via Logback or Apache Log4j but does not configure/add/define + logback.xml/log4j2.xml, then logs will default to DEBUG mode. + +3. To disable debug logs, the user must define their own rules within their logback.xml/log4j2.xml + and define their own loggers. + +Unit Testing +============ +Unit tests are run by Maven during the build. + +To speed up the build, you can skip them by passing -DskipTests. + +.. code-block:: + + $ cd arrow/java + $ mvn \ + -Darrow.cpp.build.dir=../java-dist/lib -Parrow-jni \ + -Darrow.c.jni.dist.dir=../java-dist/lib -Parrow-c-data \ + clean install + +Performance Testing +=================== + +The ``arrow-performance`` module contains benchmarks. + +Let's configure our environment to run performance tests: + +- Install `benchmark`_ +- Install `archery`_ + +In case you need to see your performance tests on the UI, then, configure (optional): + +- Install `conbench`_ + +Lets execute benchmark tests: + +.. code-block:: + + $ cd benchmarks + $ conbench java-micro --help + $ conbench java-micro + --iterations=1 + --commit=e90472e35b40f58b17d408438bb8de1641bfe6ef + --java-home= + --src= + --benchmark-filter=org.apache.arrow.adapter.AvroAdapterBenchmarks.testAvroToArrow + Benchmark Mode Cnt Score Error Units + AvroAdapterBenchmarks.testAvroToArrow avgt 725545.783 ns/op + Time to POST http://localhost:5000/api/login/ 0.14911699295043945 + Time to POST http://localhost:5000/api/benchmarks/ 0.06116318702697754 + +Then go to: http://127.0.0.1:5000/ to see reports: + +UI Home: + +.. image:: img/conbench_ui.png + +UI Runs: + +.. image:: img/conbench_runs.png + +UI Benchmark: + +.. image:: img/conbench_benchmark.png + +Integration Testing +=================== + +Integration tests can be run :ref:`via Archery `. +For example, assuming you only built Arrow Java and want to run the IPC +integration tests, you would do: + +.. code-block:: console + + $ archery integration --run-ipc --with-java 1 + +Code Style +========== + +The current Java code follows the `Google Java Style`_ with Apache license headers. + +Java code style is checked by `Spotless`_ during the build, and the continuous integration build will verify +that changes adhere to the style guide. + +Automatically fixing code style issues +-------------------------------------- + +- You can check the style without building the project with ``mvn spotless:check``. +- You can autoformat the source with ``mvn spotless:apply``. + +Example: + +.. code-block:: bash + + The following files had format violations: + src/main/java/org/apache/arrow/algorithm/rank/VectorRank.java + @@ -15,7 +15,6 @@ + ·*·limitations·under·the·License. + ·*/ + + - + package·org.apache.arrow.algorithm.rank; + + import·java.util.stream.IntStream; + Run 'mvn spotless:apply' to fix these violations. + +Code Formatter for Intellij IDEA and Eclipse +-------------------------------------------- + +Follow the instructions to set up google-java-format for: + +- `Eclipse`_ +- `IntelliJ`_ + + +Checkstyle +---------- + +Checkstyle is also used for general linting. The configuration is located at `checkstyle`_. +You can also just check the style without building the project. +This checks the code style of all source code under the current directory or from within an individual module. + +.. code-block:: + + $ mvn checkstyle:check + +Maven ``pom.xml`` style is enforced with Spotless using `Apache Maven pom.xml guidelines`_ +You can also just check the style without building the project. +This checks the style of all pom.xml files under the current directory or from within an individual module. + +.. code-block:: + + $ mvn spotless:check + +This applies the style to all pom.xml files under the current directory or from within an individual module. + +.. code-block:: + + $ mvn spotless:apply + +.. _benchmark: https://github.com/ursacomputing/benchmarks +.. _archery: https://github.com/apache/arrow/blob/main/dev/conbench_envs/README.md#L188 +.. _conbench: https://github.com/conbench/conbench +.. _checkstyle: https://github.com/apache/arrow/blob/main/java/dev/checkstyle/checkstyle.xml +.. _Apache Maven pom.xml guidelines: https://maven.apache.org/developers/conventions/code.html#pom-code-convention +.. _Spotless: https://github.com/diffplug/spotless +.. _Google Java Style: https://google.github.io/styleguide/javaguide.html +.. _Eclipse: https://github.com/google/google-java-format?tab=readme-ov-file#eclipse +.. _IntelliJ: https://github.com/google/google-java-format?tab=readme-ov-file#intellij-android-studio-and-other-jetbrains-ides + +Build Caching +============= + +Build caching is done through Develocity (formerly Maven Enterprise). To force +a build without the cache, run:: + + mvn clean install -Ddevelocity.cache.local.enabled=false -Ddevelocity.cache.remote.enabled=false + +This can be useful to make sure you see all warnings from ErrorProne, for example. + +ErrorProne +========== + +ErrorProne should be disabled for generated code. diff --git a/docs/source/developers/img/conbench_benchmark.png b/docs/source/developers/img/conbench_benchmark.png new file mode 100644 index 0000000000000000000000000000000000000000..3adf3e8cd4fde3f16a81b4619de5d6937f9a6b2e GIT binary patch literal 141075 zcmeFZS3px+_b;mA1`9<*C3FSFf|SrpfXyZpl_H{obO=?N5Re)WQHlgYq=hamh&1Uv z^w2>F(ghKc(2<0e5a4Ei+r9U9&da?I_nfEydLUUdYnCSk7^uJ8|L! z%RO!IqZ227L!LNs`tlh@dQ1EKo5m9-UY)oHR(tGgvDUQL@MIiTTBtgI^kS$W zXLe$WH%DyC$7XrXUih#_25nY6gT|;})1r2g=VZqG?%mFde{}!vKc8~Qy-fdg|LA_f z!#H8maE=x(t34bS5*F%o9KztVi8&Pf=OVFZqay7d-B(HcjP zc5iKyhK;Rl@4nWc$aE=K4nieA%)~vDMek+64)j0v@5K>qaWnE?GzaDER^*a|`7eNj zv(gA&1}+aMGs}yiG3s>!Th!3_XHov-`{ugEsh(Un-zS#$b!9zO>h()4%)jCEngqyn zxm8nFZu%ZQ%X@jKrgCV<(Mc(#Fu$(av!p=sN1&nIC8y`gUvq98I=qR@~GX7x#0eAiaf@x9cw0c97DPnKL0QZtpX1pt_;L~stj5{o7qDd+b|3)P6vbL zPLuBvoN$T73i=AV$=GVS#b(_E0b%WAn+$m@;CE4}REW~cb-lu|B3&Li_X4Mn$grST zd-9lT7nf}AT)nH7zw<^T&B|(6>A}d48xIg7M6ch|O{?8XtMToQ3=cQo+x6jIYedZ% z@p!6MH;uCqUNz$!2Jb`XGJ?&hd+Ml}qG(;y8h*JwGC^Eh6`JweAkK*EIH5fW` zmK$r4XDwpKAM+yWpM@@pmzI%CV*K@`b?Zb&b!e(tl#P-xv})sRXIWl}u~Mh>mg3w4 zS-xQLNMKfsu}D>~<3`ZqH_)Qt8lreVvJt>RG$JOj4hWv z2a8^=nEgcKi`%HLF;bbGPjSZm$m@%ddY1B|rB!|Mi|aQujM^z4oX*l9@mxYe9J5H; zj&44(rxv;gN|^^~6uK+q|?v#e3P`ynYX3K{=^oMy91pIC;7;!0LDFc}No)hw-iL z1znlGl8Bp58?_Y<8RI5<(IyYED^;r_LI{;%fR&(cw^}}LtPB3 zkAcNoSIov8*XF~)hEIF%kHnkbj~l?xhOLre7Tw-oW~beng$9!MURLoFLc@jS z<~&WGPh-lMu?aHO&hG0~dKp3DO=(@IvJcAr=h<#M_QmNd#}Qvsp{jQ_x!}--T|TI4 zM!^jE8CkR`@mA=8Ws-Ag-=VFZaRc5jm-RXY9h^^5EXpYm4K{!;aewj+WZ$!uhR>bjD*Zj!~Y zux_%7RPcAAI;pi?i(4o{M8M^yZhD4T<96?88M(@7JuKRrCDU^fr87qx^BL($shA0I<=KOR?Yg00VES?4+#=fuYQSgatJJrrlrOUj zF-qj{HNEr(^zkO!(XG_$fV$55?$936x6LCMb#U)UtV!~F6)of_nAX#U3uROFH1q$E z=`L=J!t8gJ=4R4Xhemip!)$SHK#Qf9kvf@S%sj}WJ-Oy|mEA3U^bln^;idP1i`%hk;jL0cM3G{QqFZADG zZlAOC^>7o{)Zj18>%-9n*OtSDH0ng>rp#{i^nPUAC)b6|RR$(I9J^4sN7KRV0cqj| z>+=B-{J5fk!AssKvVl<(O{4uh2c@^rB!5}TJ}*o?+?jwNLAjV>1}qD8@z8)>0q3#! z=?5mvf$(xvgG6oiOgCpd5wP%{GB3jh4B9)08tJ%a_-Q$%cW%^gD$HpL?Ro$z<+!PT z?~vM@M&uV#5L^O&q+-5oqJVc}dNP_0Dnk5v@{;o97}@R#X-4Z6o7Cv~e1qwK1;K-~ zg2p;ly=pOU`43RBclfl=hg&$koWzcdIrdiv?W(4;qj9MIoeL>vx0- zG{z&>4C}s)lB+Q2uIh%ps|U(9%y~M?K44qFsXk-u-4@Yo9ubV8w)Qpzn+;uUqv?_zRuN zC~G7Hr6QpUKZ!qWw)-(yUX`>V$e0InMc8;KBijI>^Yz=PLe2S5Hs$vN*Wg0faBXEBPWZM&e`HL~ zkt4@?T_I6ffr^p_%R*wTX07+F6-6zpR^)TS{DH-q(OjudOHAU14EYIPD|=I}sTzDy z4@YRb4F{LBDhydthXS|! z+d6vc_<%Lm+%}%&SY!9eGoN1hz2&O&+)0_Spu+D(408xe+Eus#C$zi#t>BS&J6@$I z9276Wjrx82?_e{|=n>v++3-O#wnw!dD-t}xHdy(uO8TY}hrsQLb5~1#<9}y$I*cnk zD-wCXnJcbr0V`0Ea1twmy=Bk4$^^(fFWz{4FI?r(Tv&7}u6B!q43cy}p{)g>M;ief z5A?T#?$lgk)*AcHz^0s5%11cG5Y5ClsuqxZQ%9OVn^)0a8X@Aep znE^j*@EOhq`yjrUlMa3?#W>2x*7(kulZ=oh#WF$@m=w*};Bs9(QEL6_uNcDbKmtB{ zIP+~7!Ef4mve?v0>QyE{iYRmDz6KVnqm&7bCNl2@Z|-U0%!>ux$y}n0UtLCB352!I zzSe?ksMhB~XW!wgt*#fgDiGFZU6rAt%GrEyHtbSbojVaYAHr8kfAIJu*>?!GXS8{v zT{EWNH7g@f{JEnxLU~rHs&r;Z)^#@JV2B}%^FVGlQ`#)9$Y~1v4O}UuIdk4Oy|1`P ziT{>_aAnd~UXfgvZZ|PvvC{<$wxcaGK?hS7spmP8%jQjT1OTKm3#TDFBeUevp4wnA zrHFkcdNqyoED3x_QdB)t3x0k*5SJ0CmWv^?ia+e%Z|71GG?|f85$B)hF z^PG9k4c_`>PO(8@PZbMFK-aguJXt=)JeEN3x}Vk!%*6dyDlQ!#U8EanE6&SOnTLV0 z>ibrD*Tsaj12vM_CIJJ-^&jB`R50sf|}`_bfU5R*fcP0q+8ji&3$b z;INCFXUAS(4@W%)Sff6=++-KN?Zn8&l_+#Asj1-xsdrQ*9g#w~({5G)2ss5fA45@$vE!q@S~y8a}q4_{O; zk7DXNC~u=?dniCviI|CVuW7H;G$_bS1vHE6ROJvOtDW^boQjTZ|HoAKCefOx?)#)> z>eQ-&xcwdb3t>)yd}*>O2Z5Psi4ZV$1Tj&V=xDwoRL&+9PL;4JU=u3zUVXV-zuI6& zPI1&?e$;~enu7Ho2%h8a4AFra>QO1d9OoWd1t#q zfn_@9i{6yZky60yVFt3%SwCpsKt(@T9^r)Dxe$;yqB%ya8lQ_T<%=P2uDUHHuI4fm z-Y}ghdP0coHp;#Qv}*sJleryv#$x1!z=zCXR%(YcRzk%0EE>WVION-fvpBc!w zNK`x{1rO}!Ac(rUOX?z~3&*E$L!hMT{uCRrx()GW(x*?6a9|I}bKN=xVpXB5Is7I^ z1+GtkjG00`lZ(Fko@ucp_)ZU|*XhFn>AgWv-b?WpnSy?C{{{=B}Yk4ufZ z_u74GcC8Mr>d97T+A}a#OPUA!ef9Rmol`gNr`18$>cmb6B|tlsy>s}?94ND8Hg2cXP~q;tY`9<&btzh*wNApRjvnvW>@BPtX)YDg?OqCc%q+D z6@%Jx8+VOWi~Kg?W`yY5jgTmzC>MK{3f@QUUB;&R1K9Yj-cW|BI=SCTiedK4w^v?` z7ao{*dAG^v=29!Uq{vfeI26XXzG#lw1igCvEHz$Vl?x|Pyv(x~Jff2^B43Ky2iqEI zZCRuKvY6e=xzFf6>D?2bhx_!WrjFSW}2TFtdqSbCb3Uivc7IHh<4{ z5tHMTxh#+dyi9G{zW)_WXosS%AsIOctHG4<1h(yB5j_8aS_;j@d5a<*SWutuStr(e zMjg<8HHKaJ@MbX8!O03A>OTxd&qk-#5I)*47j?(!W`~512pn!@63#s9{WzYlBEDUi z(rx|8dWrqA4mbP{c=p69s{39)cmLgf?h&g9?Dzl|M~hC2*ELe~?$0U@<^sp_3*lU9 z&jJ(p)#D^3M1@#_BX>I*+Z+cOCmU;UHgoT8*);?dKnqg%A zHW8jQ4z_$r4~k^d^U#(=M)1?HYh<%bOdV(nS1ae!FuIQ3 zn1nM6aT+ET8#)SMFRf>Z5l~EwoE9^H%X*b}kd-LMes6HHjhjb?_ETnci*R;*uUn;* z`^v~NxU#UEJrAQ%S{JWKW|vhqtZcU8?9SMXSI>t%#HWNMree&r#x;pp z?0BZDnTv_pQAd}MCUfZ^7AG*MUep-@?`#xiIUsrtnOFI{Vg@B_Aji&AtxaJsKR2;y zieMc_b*FUfc&Dq^V~3=~=$3_m-H;)AqOYgHmeJTds>`5Ze{l zb1nCOBpL3Rx)pN0<;Bz5_kv?{kWs>t)0}~%AywE6f7hh#)aSBC#2U|d$S_zqdzsYHR z2C9s_(|VWa@Mi3}&@(xy+k8sqaYCE#gz3?oRJpa5?KN-qN!|kwPyV=b6=BAcZd)(d8~qtognL^=C$h36V$Mi+U9T> z*S6Z2C5VnE37STVc+vT#8<9StXEX_uifjwa~ z)IgWdduXq=>A4p+jf0XBoL2Zgyv#cKEm!|b?(Im(XcR?bQQGdpSwylW?OKz>)I@ti zAS!^CssiSswLrE`Y5}a{b+6wcMm&vi<|f0L7|V=+t4Qz25M5J%(NwgbkuGj;LOOm( z{XD^UwVKZl?#O|40h6(cEm!Rip##7<6s%@uL}NiJ{OrSG&j=%7DW7U7Fu8mmb3VpBKTbF3;TowlO1O9Ll2)H4| zXqGzKbwvpL!{&1r+n`693~Jf~3Q_X4(g-fd{Af}l7sIY5pxV)891i53_V0kbhRihk zz6hZR?9Bp%dIePtYRhgaK!0xt!8~MwMIM&lPCWuLmeG+`ZhY6-1q;p{PF>rP*?{0u zNcH?lLWH8PZ(FUz)6oa<(vU&93lkf!4@8FNz#nk@gs5WR1WaG0w)i`^J~V2ybeXJIMhLkU z?OFT+$vr5IW6*7yS&#$Zj>d4C-`R z9`E`r_gPZ>OA2XfDUGPE=z}YAL9j+MmDtU>AgaK!We~~zE8Ethof2aZ9MRlaPC0=% zxtG3qWPW7uQ6j%b{V^f)a5A4Xn{B85VWOlwk{2JQR!&FAT;w0Jt+NWT3WW<)wtxKI zg_6(`ynkswMaI${dm_)A{g4MXM3#csFBnx2Wj7WguXW3wAJ&>u&jdb-c$e1%zG{QE zjr;WIia`^=9~tgRNs&R;pla9fl!GX-LHlcXhOnFVU+H))DWe&n1I%B%`Gf8tRd2R< z^{Dbc-G3+VdJVP}zC+)vk0>DE$s~AVCXd_u0k@zRkN`RV$|vz|U)V)WHvGlp$bzU= z%>arVk**A|FFaUmEsU+%z$!y!AF{26E&z(Ra?C1K>3%2X#|+1l9)Q(EXHZpTWcjVQ1ppfK z@hbQGVDhUE#*Bw|HzNDX_8(X}F|Au$S~LG119Sis6?srK_?fU5E`1jl-p)rE+_QB;(aH#vzNyZlA=4RVU(umw zYkBb2Ztu%be&;wDKD+p7wM98dTRXo+h56}mjxgtKhtA%wUtxX=S&ZW_j|Y4kH;NE0 zmpDt#Z|L%hr}SN5b2r`tZ_IgU4F&3!h56oU0YQ%igFTeDBqC&H;v;nwEu_SdG1B1I z(Dcs;1!#xN04(l+-Hsq|zQ~LTB$1`pa>-tISQF#=smQYwbwjFuD7du&A^q*YPj$c=X9H4yd1nRaKfSYB9$9z2 z;%T4zKHsJbsq?a_vND25N^C&DP=H}7>o{-4fCh9_64`D%9|~fW6lOp86xMTjXY0xC z5bWrIf@Cr8TO9?|Z@EPAuy5m-qv*z}Q^Z@FS^H&}x6zvJ3PZm&H{UvJ-#fnDjMiOj zZP~|K9_xzT>`(1BTr->mvlap`5FCWE3N2>@Yz%pK2IYb&VG3`0S`iu6 z@DT@Qx!votwN4KPhpNv8!gvgSlS-=Ik~xUDgLaFtj9)vN&%dmuYr`?w9v!jW7W4DpJKku_vm|LzSIgnam^ zZij46q=x|=3L~RZ$b$0_cJeufSSqkr;tH^w&h*F#4WV#HFf#})F+!^HA>1LPdb9%C zy9GzyM;H?#Fji^o(Y#P2X{6u4_&5-K??GxLXmQF~jH^Aut=Kzb7pFpaU3WuJc~>E3 z^(Gzs=(D*_YDcIfl7FT+295y;oJ2<>R1s=}GF4X=Fl$(2j8#W~Ob)j+t=1s=P^_@( z=>oHH`0aHYT4cN^cYF~KHe$)H7wfAcJP@)s0y^r5qojPl#!4PUJ!Bj|q52+x?B;C; z+l6b`D@lc-{1N*&Vgok|KUo%QOA5LyqM~N*6ORCCZiCQAza^U^P^F*uA9# zfAZqGEk7Hdib)ga3)LLBn0aa60td3Y@~up7`}=xf(r&sRh?3RlyQHSF;!wy^E!Fzb zp?LVFj#KXk(AYWH5AGiTCF*|kU-sA_!eVlvP^+RV>x6~vF1MBSuz!{7&MMc3$l z_V<&P{mQVeeupAJ11aFrj%1==YLfb9^bM!U=ck2~_(w01-~A;NX<~E*8nU!Nv|P#z z+FG5<>%rMS&H?wS7Ike!8luseMk71xdyfW<#0vTC=+ZM3t$Jom6h<=493+gsdqR6P z@!nLh)X zt*^n6tTMb2bZ9xKX(JmV>(+}FwnTL8w_I;UbuMwsHg0t#?N2EXIM|C9^UY}wdAqvx zxiR?6Jx?$Ok6|;qQ30snGM~GLtuM0r{e>kNS^XkxJBzSu_RP1=<4y}PssFT1>2@A> zb)=~kW5m(LN-u38Y&l|6hC^WC`|MJ3S`{(830H%*`~d%x-Tkp%U$U5e15x?}( zai1F6RY`gBLL%W(NLtcVy$N!j@rUY7eTRUt_&y!!Dmk5 zd(1jLypfHS9s(BF@Lfq^_{r5*+K&XaxRN9a>sa^uGTPxHZ09HRUisf^2YjC8GNN`| zVJza*@m)z)SS{wwi#gj6ghK~LUo;=!wwmS{`CQ_8$vLUSNev$qd3%6&E6Kz9TLobf z9YV=!0t*_7IRi`dsqH&z{)D7kQ@#=%p(jS5q7nj?kncR9P^BN?Dv&6i62C#ImoAye zX3>Yw3h59#Txk3ZUQz%s3|9j!3Qi?SgYVM=nudPY8N?6n)(!{rB8v*Knz(STBBL=; zXYuOSADVKy8k5KA2(?)s*j~?>``E|ev=mUjU3WKzyoLGe} zSVq-AC_6Ut;1Bo>m|daJM!WTwq&X#xztUWKOrrFp|A&T`?by*Qa@F#WzLNVK`Fec< zZj`HfkF8`MiIm>d$w&zb?zvoGIKEmj?<;6vh~gCA4!8~bK^Hm!>TS;U)ntOjovA3XMUWAbYY z%+(*-@w;|Nu?!R*A97|E>*r)nmQap#v)k0wv4l%7|AwizfR9~gVPpyuK{Fo5CTnK}C{vul98OAPjmVK3LB9?VTJBx)OGBd4oYQ3Jx+ zpMhtE{A2yH@9IcUU)U9nO#@ zal$jE8nU+a^Yt0u-dgI);3RL%`d#fVP(>IQ!;xeXAfW%`0wcIe@^~?a|0tSrW$*lm zRa+wN=Lg|2NeM&|!x--&S~WLE44VveaAu70(EW0S5GkXw>&9QZ`(z8K$+c%S$u94h3@)-g?->Ygr-39?V81ITQADo6|^! zyc<ANbn7x~??i^jeAxGht#{>GO zzgP=>+7rTtkmge4O6x6vi4?+F)l2s$x8s=N8CQsvTCPM& zPWZo-{Bgv2p>u#ijLH!p$N(eE^C364f-dg4@H1AreG@Z((fYP4`Uab;b~}jOday}^ z^tV}aqo=OgXAcklEWUWuH8@U)SXA1?sIu??72TbHUi;l%T0;TQ(~^uNr91(Q^VvXS zY>UwAD7K}*h_2HB;By^(``(00#_3T`f$g5qlaE#lF(?**COzjl!1$b3=o?C|%RzlU znMlpWi}2LE|CkiMO-nsvEf@f77;if8{%OO%D-aF(s#5H-X1E6F_3KxgGMBk`2NIRl zd(;A^*_=a`ObAf#=ZX-xJ20i47!s#?qU8RRv_TDC8g&nhv-Y<(VZRKpKeNy}b`E)y zs&QS=o{1fan(Tj2W=4X<;PkUyl8QkuEzJdY+Af{axs~nJenC()j1?c3NflIG(43M` z3aN`J79{x&TE>p6r#ZMp@-lN|c!9aF-1G7T*~6ReV(xAWPfxWdOaUOhmQx+Z9nQVB z2d^SjENVO;{6<(ha;qo-kFf1{aku1hZ_KV%>OFl`ilP!8e9309z$XI=c-pTt9J{p; zK=IClZGDANlZ7-z7Ib(fE0Tbsr9f)r)*+QAloW~}fus<_`pnPakH?_Hxw*kb)?-(P z%Zkcu6lkes9j9~ou~FraM(4b&7@HITM|Ol?gxc}PaBY0;nD!6=cLi>^!B_One@~e7 zD4Q54K83!-He?GDUer6t75j-Il6mtNoqk~ zY4Yezr9n{TTYqDyAFIfEk3AlQP#zw*H|@MQoSrw}5fb zx?MmR$sbbU6kHi9(+Z@vDW|NKH)f!u#sm+qr6?Q&h%~Ho%|u3{GPX^l>zk#M()8+Q zJcDyXwfh32K_;i=eOE=VUe$~)w$-g9<=BJW0EObx!0$oL z0xu~ox|L4?e<$vQghs2rdi+MaMxvP;*SJ^D?0Q=>_#_?*4}9q6-UKp`3+~(b+RK1f zQ3LlTkU4Xvo+n@Gl*Y?)_ObSQpTH-so`lN@&Q6@mo?u%!+57y{jK6CX_1$5CcaiZ? zBtg6;G8oc7{=kwq)Mk!EQI5tZbg5#q%d+iYN>dGA5W>b{rMx@n>WdM{_yhx(?8~3rFq6~sd z3#rWb#=mJ6Mb_I6y}b^In%lEpLHvnVgh1{zl=i@>f9s{bTe;qjJFR^?#PfjxNuVLBdwuK2qhok-x~nur=F?Y z{A;nEvKKO)C=HOV7-|3QC21(3f5pK=mHe(TSQ?2JI_Z9?(dn*vy)+At>|N!(8EAoi zrixQX1iMqu`x}$mDo@;nM;kmAu+nzD@pOY%FEyx*R$d)1)i=!7(@m9iRkT~U`fHv~ z-z(NLlmkf3!_)4~=_QS%UavmUn%t&y1g3x@hx2amFx|cEFnpDA~YpXzGJye#3)ZMT`{k)6rBzcG>s1F5(igh~nnmN1x}%J09FHCkfcK%*S*j9cng z*bm&_`{rZ0%P8@_&n}I{>NR~5YEK zdUr-LZksEy4qD1K{2Jt1)EkqPuQqmEit?T zetBn}XOv7B=tF)D?4x?Q4EPCl)O>d6M!o( z?(#An4U{>THp%8ctMjzm(d=i5rE63(-|U_;5=WJUX-=c-boCRQrh*2CU)!6r6la4fBSVcu`^BUgMYxtm>$RnGbR<6R~}wNu1uXFIUHk>!LmGQZETh254lN z9R4bwRxq5eSmn?MAz2o7*Qbd?6gHK8tNMeDhKf4ozzv7Odn)-!1FG}*KQN%8Wkm6jf>LD9bCLR5scasWOE9)v( zQw*Z*2O5gx&!go7_kVgFAMWK-$V4Oeci=m!hub)XM|Gw(ZWN~*mLaVZ(Wz^u?oA2) zyKBkr@oN=<3(<0fN;*61dy|hOkbXZDUVLxaZaXh-tFAN>%_SvVBjna^uxF^RoPaYA zKHRBu%SPw87CtTRU8iea+oL;!19Ed|xSge8p}iZJ6b`$Qy@1Uoxf(WKp>!z+y{~Vu zU_x|6Pkr7GoJH%V(nJ?Kqd18}<)`ho0<>N1B3o%w z&ZT&W!!*%MrC~wJ>W#Qr?Pqi(?;(ts?o&RmqxgEDZ66Oh(3Fx*>k6=%d0J#V>=H!# z?e)(y7!mHiz9jJ>_U_ZY^ZH)PBR6z2KnhIg1Qr*ChGB3b^orZlnKg-!(uR%tK*Pd4 z;Z_!4#^)_}4EO8EcV+L8@3Nu{9!LWZ?pR-kuvRGl#^_BPOeY%-&Nc-&;8Jd;fnW=S zcM{0kQ83i}e7JxfLiNaZFtiS2)$;07Rv!zh-p`ZTEreK`?ULzSW$GgC*Zx;eu?4oI+x*L-{)5@~-cHd{~&7Sau+*WnH^q1D8 zNX$!t(8NCR=Csf(iFZIGFq++|jmb)NaewB)i7UcRO+$5^+B1UtCw2ng8tOuYCD*6Q zAX-QLh|L!`T{(R7c51g3wYMg2iwnRI$W(i{I}PRTg*4+mgHBESG(6M0U{-3;^a(8) zAkDEY>-gn^w9CZjPvb3fbQ>oyuK72~K7ektzrv%#wx(wcB?dV<`*bfueY<0?)$zI| z&gcN0f$z1VxfAAMBYt+CCJ^p>Ni4tOxwRZ2$;y%J1t4jH4YXg>lS)IY_`^@^bqNY!Oa-|K81Pa^tz55}Dq<|h=HCl@f zq~Mc?LkIYwI%{~j`DxX`F8&UU{j@oKoQcFL40xvUGE_sV}VwOYT6gi1} zc(6BQaSGP{B17K8Mhx*?0efC5gXRn9q|x>CApc27L;xI|I$ckZ^dRMFe=a$ZYcXkM}Pb+^e0@98SSZr|jLDorcd zMP04gkqP~BrqSu9C8xFN`4}8)2IkSrt=ld$OOa0r(Fxgr{%id(6wbq|zcN;0o|-97(rgXt z)@V=Sxjx(tUiE|!82b#BbjBuBUVG8-1w3{g(b7T2rM$y^&3@%tCcF3TQ_sy_m^c@j z3X+?91sUy3<&q25v31+9rfLTJL2Ub2+*ac6O~#wjQdnX01M$>rLBon|nZUC6YGF$u*?srX70N9JDQdf< zR;na&i($g^lt9SU@@&;sxx}+2Idl8J1ws@D7yQ1%F8GFL{vHGh_YA^@oN>YI{Fym9 z_$mJL>5A)RK?iX<1@zkEwA>tLY$?NyCUOD&!$y`c8FT;TJO#ldM+n?#Ht>0Jw+l;a zs@5=AETxoI>>voO5KAeu7KDt_jfmX%Kq^PTMfT2qBZE^G5CX}*b&Rv|?R8T=RI|!iUATYD6L9vuR9$VYnQjeTB1>M zzq!3!n-4|y=}M14IXYYLEN@B!Uf^t%q7qZOT|`P@q{Ad|gz{dPfk+FlZp!V4V_F2`Zmx~ zAJA9f0K>i!Rr#sD)E6a=LFK%goaS=3-v2p2Y1IY5npzb`(f1cg^F5W;^$;8PK_!qG z6gu%kDm!EHM@50C27V1pcaEIejPX4Hw3z;4wd-`EJ7(s0plhT&))DW_wKy-_HEtn~ z#YqP(C#2r6WSJij4sv9DOPRXJk<26F?zpDT8^E@u*}=mrx8-a2V<09~-e9 z<;%+JU>2c6*!vKnNdN_%ZME;Dfp5nN=${ZjWtcqY+2-(_H$&4`;51=ajKpzz&zfh` z)ALU&%RhT-qW5*FomHA==mNSMvEYGQ1Tc!KKJVgTSoXushxPF2(UA z14r}RB8W^|#qWW!f%tKY$*@z_yd076NO(b?Zlh7D1;ri#gx}-AO4^H~<+>){h1UD& zesgJ1Yg+Sw*O}6?FcQc*Yr~fy8NB^qDzls)wO==*^rw|9G*qjxZUjNf=mw=Xu9f$B za{Vj)?GE4f%@A8X4$rGgzZ<(4cnAa9Hp3@u;+DQk20Ev-$WaI65DHjgfX>bY_B{=? zywB%*axn`;V}}y~4gM4-1|pZyDg4j#66v=&RBvR*OtL9VL9UbP=tdy$ddmqs8fzrC z&wZZpj%sX|k?elnFrU)P$L{Z1n(O9wHs@4Tk?q>T^+_y*x~wp@2OU7{UAeLDAKdzI;|SRI_B>U~2E#jBaO3}gm<8MIN2h(Ui_J6z)c|K(aJq+0d;rqZ5- z$Si)tZV@HX)#ud4`F$=hHTkNHj!kvN)erD>;7_r4vwsxR)K0f(q+6?W`B`vI3lUjU zx=F}uZeAb_2+#TIi0}3hJ*KM!Y*_>*`ldUc$1(Rd5#br7zxqSQ?x%1ylsLE&U1RZ$ zCUJE+d)>gNjwXngjB-D*9a`V#XFA39*J_XzZ*5+D)#6jxtVm!w=@f|Lmimw1$}_ZK zc~?}89>;mUfGiT8$?B|n+?n+F=;~QRqaNhl}gv$ah8dFaH-KQbEE0^8v8upDy(O62=cYJ34Cr3Z?%1sFD}|`#|Ym zxPo_XWYK@0D*aUQ{|Y&bfS>+zDwRIS6Qh*w|Gv*#xzF+JzbAQY?j$JkzZ_*1(Y5{W z`$yh+|Fei++wnh?_{EL?!xMjt$p0rwP_>un2+48_{RsH+(LYDPJsCqBLVa=u&71@a zp(a`g{sFmcdf4(Sf&EIn;j{|(a7sfxcBGdBmbyXg@7+$6)8GT4>~#hgZHH= zI|^nd|7VDQVZT9sY2=X3f0p^TQ5F<*U`y$Cpktt46-57b>e!JLOuCCwhAG`EX^H

2lZ#Jif!8FQ!O-OB1OoYXE# zD3i%-Qaj6m>8Q_&ZBceiTq|9{kI=n;gp!3l9qRK-lXaPR__mRbF!U1a6%GqtQRyz` zZWfn9s+{{E#9MMDUtHDnD%-zra8b1GRuD$Kj44!0xo-r7y+S!t95r* z5Yw&S$e-fo=koZvUZG+{t6kCk`2Z?sPF5=^7wso2DgiPd>;2kAiZ6*z1h7EpD1n95wSo@JLkW%lEBCeh-Kvm`OefCvS@yx|0{>o zFlH`N$cjT+jmzZwE;dXsqbjGCI}mJ{>O9JdFxplZdH;>mJtamK?sz)# zjqD5D9@fIWzwzJ`;p)Qf40H%z-A+Gr7Q4y?8FqDx_9}r_E z3z79BEvAqk3S*;p-zSw@2CP?sb}N;p^kBe^q7c9yVBF>#$4$K=?reDHXX|#7vB%M$ z;8_%>cu95Y4z4-)cqPRuDAhzg-e*X8?{Vpkn60E;&uFQ9pg?>X9Z6oPeZ=GqblV{` z8FSjP^2&c!bf9ZAJv!c8GFj~pR@hEGU|Y}?K=u7FIR|j)B`UuS#@VkBOcaLe|C7Fa7CMiQq7`_Mec&yAg^6ulngApHp_rs!=A*9uuaPA>Fho|GOG2-;B!$7iBK6lmB_(SDllp;J-P67C>;6t zZmMFGx1nY+sn`4RbZ>Jz2<{0S0KRPlql@H?Hq@21qqgLIHfHR*_&%Xz~x+;Qw4_kgmKhM00s}t%AoI79$i;C2>Hq)hH)>gD{WS+8F9fnYGa;d znTuWD8)>9I@%u}M>$BJ zcl*wRCP3S$248$T>=m?sA=Mp|jF8$eBf^yq(I7%b_{n}YXs=?HdC`!VdHo{LdIn`l z_*jdHa(?~MjQ()PUiD~e`J#~dfdloAjXcN#L|BKZ* z|2Ch+CIaQA=depP?mP)WI}?ld?=e-}KDVr`Uhn*KdRbhN?S9{wyRNg`rgq$wlD;rW zO-c5esHg7tq>|ja1u(?oi8Mm#RN7P5nGCA_vka=PMdDMpiI;!aKSwQ;|h)Oun{?{ zqX@5)Fyc~T?vGfyi!c=h_`q8e!PiGr`em5gnD~9K?uFCf$_skAcVH%TlMM`0VB{2* zfeEVmIZ7S2EM^B0E=aUV)C($d8zu$CB?5E)ANIa8s>!YERu9JtiU>mJ5+sNUB=n9- z1e7MA(n|6^$zan#57C<`dYfs0z^-P(%6Kt(Vnp-pxLECG5>*QItHrqphqR z6jJR8+PBH@fs_dMDHEU^u!lJ&X}pR<7d&uM4<(4ofynW25=3zp0JbCPwjP4M7hTCt zs;*$=Z71&Kt)vwYhD?1OCA;+jUgk?hj+7yooJC0Bi~aWBS$nchy{tVR6S4zH`sf09yh8$a=TBk-=c zYEVho#aw%{l=RMl&M{9b;-;`OM^dNdx_PrFSyJ|J?2y^R$oReFMWnPr^DPwG#yMY{A_I)zX}N;N6L{3%;Rvt;|K4Yl0?~ zUtAAvKM!c^YhP>tI4{nRNB2uF(&%qEy^Nps!Tkm zZPl`;0jAP?Z%$b%CVHtYG&<7 zwAV~aPg8JqASlW2ar*jbxm`T37_1Z)$6(DwVvEZ`D|bdCP_p?qJerv=%!=JpRhoe= zq6(#X zG@)Yo#sgOt&*mh|KWQf-{_p_>w*Pf&^aa3SfRjVp!d~7tg2~#M!Ar4EPT^^=Ttuyj zqMP7kJK8ycpo+LKg|!ziwl2A~?@bFs#1=1ugh#`DG*e-<#0=$RhVLCdbV}K|NXNc2 zeH(1H!v082m^qn!DI9}nv~blG!b&*CI=X>w_LX7V)R>@1?Vs$uAB$kvAdJgI^OpuW z;Pwb}WFL;^bf0Mc0JocZeYJ9xCd}36)8#{JV*{2_E~K%6B3^hY`;eo+f5wW_e9tXS zqNRfCwlARTWRW4_gkp4jQB}d)n6~w;4lO8x?g)pU#ygQj>5!w}zNj367IGVct?S&h zt||@*%=SVStW4sCNN>v>B3@Rjv6cc3itK!yQS*Xgc|laqScO5_v{utcgKMu@K1M-X zkzsUPQh=Xpy65e5rNpmRnC5>M2l}Vrz?JRZWCg~zq<7=|1V4V0e&;894nqy(t~(oj zy-gR$$w2y|=PWe{$LzHjka0T_DXmNUUFw0u+gk%RjeQ&+M`oORFp zV;tUlkrieAi<;O+5|*XC1By4M7-;)rOuqKx^Vs_fWa%|0&fwb|Z&a3XpXvkD>e85s*sPE{p)Ris?#bYpP?C5Kb*2~wL6pm@w!44wl2`!K)kUkIKLH99=5T!r zIz|swxPXVn&)27EfL=MxSz_cS=qm_?=Y=pXHSMzHr z>eo~^9Hc)a>4WO=tcS6f#LArTQe3!I1~kclR<0b-BJ$ddmO;yoc6ndf$C_bNY)$)g zA@rP3X|gC*dQk#wDUiAt=4Q-~I@ep}VsfQZyS>reCK^X`#+(C+t5$C9_P~m_gDhr5 z>IZOJz5vK&_PGIaoOhnv>Lk7Aw49H{PkrYBR*1+P^e?j4Pwp;#hC}tkBE4e1c7A$f zcg-v&SfHz(ctF@$%@#e2BF4#pXYlWcr?h^|NX-cF-buAPp2XIL?yMk7X0^#lLnP8X$+gaE_e3li z&a||Dc-u@YE^udeW&qQT8d#>VlHU~M=R1$fiTJNdOIENF5wG5j`b{=`zZU4Xr6ZrYU7|`+kEP>>euKbFe0%hLs&x@!U`kqhm5=; zhx@3>qeXIY1C6Uw#vn})E}_HEoSD&t)tX_;0)+5HUP9otZ9phA9P|o}IAEtaA0(_U zPRB#Xx4xe889D_00gZ+#wS@_w&V@|DW!qv);a{P*+B&2jD$Z)3CcelT>|1=PP&x>T zB?Yv8I*WV}aqsIx2ktw*0<`g$bKW9RH?_;rn=NZ|H@7*W)@?;kb8NhN8Jl%KlTzdF z5Oq7VNvwI-;uZ`6Y;R)OXFrM4D2pSNjvJq~z%7koe7vvHJumgf@Ti#Xh1o?ms2T|b zU4KkxOuCw_i9GC4!5i&0`;a)@IF&Ayy}>-B4@4fDsoz<5%cxH)NH3Dy!;M-ML5+M) zyt0Va)pXGiA)!?TT)K55>@!RGQ1zyCCX*=M>Y+zoM+ zF&&>`rHQ&rMd8KK(;f+b?2`M{a1#{oe9ACpa4^)w$mn})5d+}Esd>17GhsCq$6t`M zlG17>oZLFZq(1W@BR%Pen00RCcC>uFz)sRhLCLrX^P9^!-)yGJGIafOHHrF>npbsk zBlVPilhXyB{?@s?KxQ?GWGPtL1|6t+haEw^N{%&!Y0+D$Izqb_ycxb!LF0U*!!wxXyH9&MXAs@Hn*1% z93&&-xq;Z;$Sk!n*d|o22@X*t#w6o7M3qbLoi`sWgFyLU(h;V4$0&9p-Vb41b`d(~ z{fVLXTY3BubIwzkCXl?qieIiHWf`~Y`8XX5w9rwdR)3Zh@~X@Am3lUwXugzRGLt2b zS6obSY538+nnypw@BH(Orf-gOf*Rr78QwRTQWmIvQrOudXDmqcGE!{x?pT!q^ahz- zS;QaVuOmddG4_n9Bs%=3^+l*m4##LH?7@Vw{Tk98b&}l1;@0-%1)?)36YbXb`V1on zO`3%3gxg_R7y4L#X9f{t$3A-r)-7Wx)EXqv;c-P-2Kr<(YO60T0)HTCy7n!qV)b>9 zi_KCqy*zWtnZc_YtyR4WDBEHlI)8P?1#7=#7CP(o#1g*IZ}#iG-0LoBuCNj$?X^J> z+>1cBTYF9R*fILeE3UWbjC6#;&)G4h840-iPd~Sex?+2^)z0OZm7P_-#tbs zz^SosGjB08-^0i=H|WBC8SG#T2Pp{j_tKgH5*_68OCHMZ#(wjP+A(8{lpbGSaDR9G zf&$oNb0(qO{5z@EMpTib%5JP{d__e?@WJsuiNS7Tdq;hyVDj2!%ERZ5*vL)XchwoJ zDlz0k<;+?`8sswN3?Cy!QR^EVtJ-`3uz^Q8?lH2uor;e{BJvRn2#mQ7qV^Ic{N*dV zP`e27D@qt8h!V-8FA)ce*gY?E+7NvYBs35<(wz3C8IG;8?}C%2PR?2=v|xJ5Pw}2D zW2U)8!$=cx6vhK5KuuwTV$rIWcsz(0E`)nF_Z)Eqz0NzDT3%{$&YIKy8va37 zSr_yGv=Sa5q}`p$w0)FlSs-|5IQimclkm;~^peibm-=1f4*L^q=Xw zd>A#QVbB|PJuJ+SA*xt*&MVu)8QP4JxUtsC*76S?9oLChhtyFD#Rzb*mc1n zfGaDJJBhaJ8@Cm{ITSd>=t7n(QS-7C>F&eetruit zWBQ^~gMTja;61onTRgeSg^P^t5W`HTwGFc_F~dKq@B-}(zTZ5q;Ch|GUVqJTtW@>_8UyyPC4Spnp;zDi2H^8> zJ>O7~wx3;7k{*i1nPrxWq_#MiH6(F3d{`rlVSe>lAh(R)9)0SxC+7)t^iRtB_LuXq z_enZ)B_A#1GuVB{I@1^#?FzVuz%j#yzL{YNr)`~DVhd|&&8v70zglAy@$$vW{= zPrs{-$rv2{0s>uLz5-Y$V@$XViVek_eS!D$<@n%t>UiVqD}^*}4E>D6xXmpfMZTu4 zuzj=T;WcNpHr)Bmk!5l#$N>)S<5s^)cMT;l8X<6h6rfjPIUD_&p5@i&3db5Sj7Cbg z1Vtqsyhg-jM*upO2BBGi7651k@u;m%P@kSD(zYez8%or!`r--Rw+hVw zX_E|c$DboMr$LKzc|_j2{E!P3z}lm0i_Q)Hp)9*(sp zy^gvQXzm|`a{29jh#~~G7_l~e@?5?T3IKS|&$AExxzQLzQ!L%rx4k}GW=f`UNW_rk39OcopWn_BPHxy z`IUZj4M+)`o2=U&S%FicTWE047fy(7YggjKLMucO>7TgfWrqe6^-K*W0Kp=k5jY97 zlo!xGLm1Gqlz*n-8vE^`b-IXCnoUfVUXKLcjBEFkX<@1~KnXF)wGmVqlfu$`g1+;) z#JKpm9v){PW)3@m=ALj93DBp{33dm`!&ec10WwHk z$lHjthr%U5QfL*&-4CJDUbj$KBYLBC@mH`+R7r@?hb~%{s2*tCeRea^ScQiyB=AP8 z^u0`rjx#86lq`_J&JU|dsn6`54}jmdc!j7y{9LP%bGoNF1FZQEC6~U5<(uxkZ!s3H%COTM`5x$T^s6F1b=h@Yp-@YB zPoK&lm|}Y?f$&TSwWFW&yb+H?c zk(km!XjofpvcCOKm@KbFW(CVWb}xEK)K+mrHN8B2qx?cp{1;Zm8>;no+*{DhnN!6K z*mUB6+J1yG2~ef+##KE$GXw>M3+)*F1uD{IpsBDbO2_jT*Hgo8q0Y@B!*SB>->PCg z9!tdCXMu>|+G4fYw^Ue-n@UOgx_B#xJYCdw{$Z&mZ=l?zlasWz&L~zS+^3X_#uVU0 z-5c9wk6UgLG?(g965zr-G;_h&sf{1E57qo_r||#SX*NWkfsR1=AW?A7zLMm4>pgA< zQL)^vT~kWGp2YdJ5A9>$_o?zp!DIHy0*^!@?Vv-O-P>Y95_8@D%5~&qd#AL?n!X_P zMg*N)oa(sYLwfB-(IhAJeo^+m^7FNj6tq9;zLMg|z7KxPzVG$OhyH1CBex@QQHkUq~iNZHYGrp+_6J3yRu$qvi$+~NRW?KdCnim((*U?Qu$xL zIe;%O!vob{mxDauLky0F!Ryy02TujU@}Li(_Tfw@?!TH6vQg6+)gTp)n@6ggB`|AP z?S1x8+Ko^H4#xtn>})bUg-Fnru(LC(rU&H@gh=0)2kF2uu4#g&pyh4szT>AEIzh9# zT-IW}Z#AOl_BnT6crq=T%h{e$X2n|nJZzUf(&2VfB0x79E%Q+82fYF61H(sPY*;C- zD8IXx@GiB~0uQ>mAjD>Ikm{0W0`lHW_90IFVb!Cs{w(7FaVcJM0l{S_8|vY}e#t*4 z9le%Ta9C>Nu_ouAy|kG(xCiF8k6*8*9}wLHm8@6w)9-0g%FCoA61WMje>u-Ye@P4*kNF6p^6x6nF7{Gi^>Xpx4l?CT2mUN^$fLFk0ZZ9<8c5ooH?NRxwAaVpF|{SikHqD8LZ-jkPWMoE#UrhS zGbI$sKnJC*g7x)%@oU?ivHgkfvnp0)qICm|kVtflooXI#HrDx_eDwG9inoI%qoI9E z(VT>bWAW>bU&prx2C)@$vBKYHv2~@TcJG|uvirJMSC}~Yk}Leas*MfyY-8IsS0r;o z75YQlHOKm0$Sa}S9dA>G6%~sqOBWJTOTK0ybqVaFrZr{(zA_T^D~U0we$ zylug1_u|dq|Al`QPCsC*MpJ*C^DoSo$A;?vMZeVrepYa*|HZIf(e;+rkrRu{%a-<9 zHhx!JT)Y#M`(M;tBwzOHHP>9~$F?-Zb_08$V)6f_?VZ0a$6m{RVP+a&!yQYcDF!MW zf&P2z!s)Z!%8>wzhZ_F={t=7(pMU+&Nc@i#{O|qYz{tgv>O${PH*QcoZ%~196RP*u z0JM4JoMq<-)O<95b!ToagxfQdIp8mWN4>NSF;RJDxA>u^&iT z#T9A%bC2tXpK#@?vA~rtMJ2*?ZMsMWw{$L1{l2hRn%xld`TD_-Ja6N^VGJF&t@tm{ z>wAaiffC$4-4ncx&KYS}E^^2wX4Z^DJ-Lf7Dz1q0*7B$F*3~DvU8BBZqMo;UOP3B{qF`Rm9A4gKs>LfF7W0+y=MQ)7NuMYz?)-K67SQ$7Gl9 z5Pl@~H$?bG_HS_Uf*NCW)qfX(|2Hn^M1TW-Olp^+a_KGiR(zeuv!iO9Ed6~FzXnik z@$%#Q5Rr}GoeB^`OXG^t$Hf(0q246=a<<3sB!11QiyfE+;7%9XrZmBi?(7s!vDDcN zkZtO$gNF%Cddwk%C4%PyeXKQin`w~5dpie5l7{#kg?=aFUAu1ble*3Oa-2&kS{Vg)otrKJW36rWUV%2S0si>Pi@f( z$n?@3P;?>}e-dL4(z2$#%UnzGr<#z*R?UcpC7mUSZ$hOgW+PkYiG$+hfPR}nkIse^ zrGex6?#>G~-RbmRTV^2MycJ)M``)&zpI74~ws!OY3wuP#i!e5(+OFM2MxPA~BjRuw znKgSe>Tc>#Qt)E1wyBu|-^~x0Y0xIngtK*JSbHI!qSJ```zd8k` zjZMsNT2lR~M&w`9Kl%9>j;?v`yc_7u?b&+`V*?+4BY69XgY0F6(R>N#b=)ZGoY$t& zrJ9RIS4J_VVtyOFZBzaR6WsNI>Q#%gAyYQ;HqEx^lRmFf91yB(D3wVAhMi;Gis+W7D7JPuW#GI0z(UX7_m-)$C0xsTK%d+5s--;2 z)MBjqc}G*T%STCqXA6qP83&hSm^Cb!?FnQNJ~=6GgIy$HLF6!pzLdD9 z&6`kJ3Xd9c|1mNlMxVVD|6VCl9eo8CXO6+e^mpfYS(Z@R)tR=k5-7vkn}a^{-Wq4; zl*Ez)$C1x>-M_jZ2yyWULa%OC4M|yYQxR=O+=_B|PWK2EON9mo62c&{?GNV!uWmX}Z(#qYo*Y(PA z?C5}-w{gXrKNb1<{o)*8KtAU-u92?t3!!iWiN4bWyW+<2nGH^CSmsvRtl@<%PYTT;iDmvM!orVKIVJ@P=O%m(^yVq3%&9qo$_;KyPZJ@8^4C zHAVhtJi6I|TM7v@5x)O4TiA2VH0XtL)tIaJ^;}Ti2%hzw#&8FktE1Dt#GU&2mOJC2 zm(dM>K9Q7`q0PtpjqjIKEdJ~rJmhbTp?O>ySXo|MW&Q}Tn}P3SYl*nzz|9U458fk3 zvegQ4osattX3T`U=oRtXHeBUthh_fNN>7b=_Zipl6Z*FeMbvGD%AF^D!Gl|C`}W)w zZ&xyZtBjJ<$1V)pIkcyd9!TqVi(l1d$bTh~J(FaEP#pCQ3xn!^K zmQ3wvb;eZYlW^2PsH0b$Isso+bkx1IsDf_2kXvbiSqJhTB?7INkqb;lUYBRfcm7~$ ztx@dt!s0Pd4llkxJsu|4MQ&)M&dH6Ke-gD$EAU$defq;p z+0f7!=aO)bgM|nK*n2DcieZX!!Tq%?Pq}x?%Wqk*XT~P>T5~_`s5U{?I zJIL@2#a%b!I-%5M>9lq&+viS1g! z*N-QP%cs)76nQ$YxSh!kYEz_hJR|rqb?$X4n#ml`?Og?B=^@}rR}_~O1SjKKf?ld= zSnAk|TY)Rt-3ue|CPsP|;BLb*;i3s2F|M5k-^Cpb>H}sIA+9deIwD_MeMVzHChYr& zMNkX9Pnh$Hq@odrApL%9dZfUu){?No^TO(*moQ@<4|d1gHX0j<;t0xM6L50WcAbir zge|!2FP;(+1{t@(rP#pZ>>X6qGmDoh8Pg%D7saC@8${2(Q1u!2u{o77)J3!BwidM}2}iGoK4p#RQ9BAAze2qiuglPY>_!D{9UX5z z7_d@Wn05G9-WtM7?(9kwC!~42>2X0<=wsQKP=XC@BlixP86sFx*AbD{qWDunFr`tl z5pEqQ6ceBS)497dn&mKBnYvU)^8L$QK&=-Vrb#Q^-BF6qX6^+5CCctId3Am8Q%{bf z^Y!fub4i`@VOpULZh-CkgyZXISa9=1rzX9YFydP7v*xB1!)Ly>6~hMQl!vtwWdm?5 z6XfHcz>e3CJ3ei2r{u)@BY6-J`admS;w&Q0+DPd+ePvF`^8Kepx zXqs7oH_Yq4$i{|fSsng$5ipP!91F3Pc@j`3s?|Ok8i3f_7>?wND(blPyM36;Qdt2@WZg>B-llb3uLVB)Y zx%p9rKWxP#l?_AiH%#}|7>z7nHRlP}A1=o;OQAb%(M9x4@dy*hGJJkth|QyE*kq`s z08x|$Y(Bg)Sj3ef6I0b0zXP@PIUeka7q;uK6ewYwjgF*5)gQIy)_gYT(Uzbsv|Rn% zJ;arT*M>!(ic_TK)S0$1Cy!!_04+|&$A=R|^xV8!KSGf8ahy9Jh2oKleZKH3r395S z<5J%i`J5H+J4G+7e{?g#$j@cf1G@t=_I-QhQhMJN4gW}@9E_C?Ai0}KX4*u zh)JEXjM2m7p-=Sm^-&x7o?qP9Ng6XEn3Th6bTHd9xj%2*HRrkeB`T~_)`rf+_E(GXS>GX<){oQE%KK`ERZ({0u>3=GZ z-d|MKM-0DNmygX-T3Y}Ks~c>86I8Eg|E9KX{r_-5iX}Mk`aTbJ#1n2rMp}U9#;8*( z@L2Po$|c~tjrHz|uS!U41#8Z){{jKvh}p0X8*pG&)9z|q(FKYpHHLezj`~8*MuoFJ^?%{=GtLUNxekZ_ILIEi?f&;p3vVaXfr|ORBL4YE-`)y zoW+54GG8=Rr%q6?$v78<*| z2&i@;0Osiwz`CGlNT$7i|9+VH=n60u_62FmPST5*-hqLEV45=*gQ?nPl7Il2>PbM@ znwC7(B_+xR>cjW?Acxwc#pCoL? z<@9x_Vs)1oi+eZYDgx*ot(F8H-mKl|lBH+>v+8=~?-`a%0r;7rET-OBPRpNtHb~Eg zOuRw>lBKdPl@lBJ&kO_kf@QW|xboy@D>Z#UuM8ksJ@>0;VnzUBXc1$i?o4i7g=>I` zGBOJqGHowUuy>#(91YR(zMgfh*4wGFUQXi}K%%O~vqZlXOr_<)UQr32Yx>(+&Jkbi zhp@=&ZX=(qN;v_xP*fD}!i=~Tx2$K;GKl$gVW)1s`Z<7NSTts8V#3)m@(H0N)?FCy zPYBpwdIoSA!+4ZJn-7PTgXs@DcBm&nvr(tSZLki|V}ROprAbL5y57|PNUsnjv^p1D z!1JndVvR#Zs=PJ#(ZFeh(xS?AD`SlrZNDrGG4n>|X1@9!VG$QE)#O}YIyd-X5k%IM=-aIxI6`=7$?2pIqISVw^C zVgvBO__GVuqz9tnqKCTz;60Z#XmZE?I$ECiiVGakWZ&OfWI5KDwSf1VGCwK7>^&WS z``M`6mf?5_khGF=nTG%L>PQkqW(w>BGjD$@%AGku9N>8ld$#k6B%kGuVFaxAW)9%9 zAM&afurpRp@g!C}ACS`nBFJew}Gn2fJ&f(dgE2 zWc5QK!2DY7{8L>ULqwu3qjbQ=r!R7VSgyv(t*eSzM%W@48n?F=hr$3EL+qi!_(RFD zk<5DoGi~w6W;~!Mb&0?W4pU`ge!vkVtro0N=`_)(c(fL~o6^dGH$_bX!iTtbgz{%# z5fC&`fWC=EDdiN$EYGNtCB28v#oiV;(fo4R^_*C_XQ4-*TC+7!?6hJr{5ofvl3lB> zB>0;GI$5g#A+$w?lFID^1h8t?l#bRq=9C;KFe;fBmWhTU?xMD>gw}TIkL8tV81Lgf z@lz3u{Go|A^+Ev_t8)AfpcjYoQiFw=3pKRt0FHH!C}}braJV%@JmpKefiHvRCObnj z5dp94@Z-7mL&YuuO|_?VeE}+Asm4!Si}z^D5kAqoQ$gS{>qdp8HxO@CVgLBM*xLw{ zs^m#{_az4Wt6)`gX80{rsC$GpA}e^U{g1Ovp5s+6E^A7UG+tdt;`N1jU$q|4?bb5L1T ziM zE(=mx@p!6HDqm#dI*A{TM0e+&+5T*98PhQFL_mYtTEt5pmN`8wzj1|h9Z)@t8mm@| z6sfvNHylPx{t7tQ&8`S^*|SP!s8#wzqU?ZIosvdgaHXskUxb5<~vyxbOx{b2=vdEmdY0G%He--k4H~{X0Jg9pA{ssm&&g%@E ze^D@}b;$?6T{-#uCWbUEF)L{ec9VP}5~n*!l^PWQRAzpqWho?RoI$YpyY&|`KTi6f zKY-pHeByIUx^BK;v36HXxYG6WZE$-fj4`>@s!#7yp;F9DIhyBg@|dInh#}e!s5mMk z@9IKCATZqVM&>a}8Tec8ZGn{P0XRhGx-+;x>8I?mIB36Jtcm8>coi`C+@@VJkp4pL>}GU6C&NQ^L-z7(470>hlOt>9DBmr z?7Dbdf4XLLaw{3(+XjcsviuS}9UuQEjY#Q$Yevy$hr&$zgVyx@)RRT{Fc5xjpY2w5 zRD*GfXf^N|*G*B@+b!R$xWR8>*3zfreE=eOrxUnf-gdGnS5xbT4MHW?75E@r+=t{- zwt#xyf|FIJVzkMx5HF1yx6hMafLunO8HF8##B3jev`^D=w_CcXM4q3-t!nu5 zG=_=9!=f8dC)4^2E={OQgC^VH*oT_r8{zr+(X}KgyoLI+BS;wHx&FndIvvL3Q*5(u z0xv=T>R0km?WD@+>M8$iVc$Dj>`9DxUGy>3#$qzQ-Ab08MstIm^Z?J-N8?isiYWx1 zV7{kwn&c)DH2VIi#Mo_16om z{AdVzTOM5x8~~h-IR29X^HYtB=>GM(;`= z&-13xSms!DHn2p?7Y*weDRI;9j#TzvF^i z-7jE4>(@JOY}I;d(u>qA+V1@C@BABsz!c}oRy|u}1ej2+z&}l)Uy2+$@OHbIq%x{t zLtOqpcJa%oRT%^d;*7d}k&aXD{(e}_FYVuZxxbRX|KksG_27_kzs><@Kp%USY3u4T zZm$ps@7^5)eRWFHLW9DYiyUWx1ftuyM&SDIcWv)%>$hnF8exsQR$rUimsA?tQpylR zSqo7z`kdQb(yW?R`z8Y44OltcP8F1Rh47`)hpef2g-9=Me ztOI%>SpVWg-p2(^4aCH|Mwb>-U=_FkN{;x@&{a-Cl5#@m5;78gCLS8jMn zZu;BNZOeP??Tyj?yA&v70u((i0Zim)_f9X)>I3qZ`WQc-qoq%P?(XVp6MYy!C9kz7 z{RGI)clWoKngGtfz80_`&}zx8!KM87+>bszhjIZV^$XNsCFMp^4{)3MXL{N@V}iZ| zyExboZueBSJ&j2aL}-i?(}TFX+up?3q$i_^Iu4*rotsJpTH8a`P7(QIv!E}wHoI-a z{-)A}j?DVW3wtP2_FrAEr-R!QVedCmbEJ>H#KbqQhHOPwy8z?kEH5MNFyaNSCr6US zpK9#xZ)~hPOSYCj_HAmO^<8&niIflfK=eYax3d$v;-Wl*yA`Dr#;LtT&$-?M(ue%T zPmiZmCJI6_cTyO(q9#Qfg+;$`Rb5M^A2@c>OP66()WR^Db&mr5Gg6qiMmZAq@_o;G z;=L*MkwD=}1rcevgTcsW^D@HM{|U9!iY-?MB;#&74-^qcOZL;%a~ZV4b(=azOHe=y zqX!TG>G^Ec0rlPfc|fwBYK~y+J)sP0|5bqy>H~+%?R53)1N`N@%+5JuDAu<9`5F!@ zDdP6)skBa_6dO|CKtzfW85R{Ij75kKFU zOsUA{hpZ?_d}&ybh{FXg%kl^8P`ECn)QKNeZhm7|6JjTiGyU6Qt4Da&L2g%wJ8>#p zzp9X^%Os9UK+F~Du`pnpo1{F$1qeCd(`6-r)IxwO$cOgklcKX!vE2FZAN1rT()lBf zfGl6735cIL4dVq`nKJP%fYL6^wR*t)+H6Z-o|@9*$B)H=eCmKK5b8WxYpc_jM9m`s zrhW*ZZ;acXL1a>k3Oa6ugEu=s)x=K1Co|@2WpeFj%mEB&*i#=o8kPmfA5DGNGy6EE z%w8z@h5vh}@1_T^NO@=v<6<~z0SS*tBF@+Dmgb2LZa0 zp#mSaL;Iz9UGDSF3XQu%mAfUZ!GT4cXsEZQKci1`ZQRxRk2~0b*@2<{|`#<&kgleLK}c;xslTAljJPR zQ5nYu=~Y}L+Kk0U5j{TsI|9sDpUH9l_TnULVVl|r02G&EPG3EJT^Tq=TKNV;opAku ztC%(H4lMXS!=2}6E7pMD<5t<^w}+7)q~(S0ACMGkB946uXkIU5zlwO^)SKPHr`-su z%FUDVUNxu6Z+UNz*dU*uTezmT2B>m?a)JRlORgQ?`K+)7*2c6PSX|fz_9r`B9*V2q zXkZ^tFXeDL*RCmxDKG*u>yg9;&?WiKBY{{^TKWaGr7R9Q3v8iTeZUk@PewTA2%{MG zdz44_26VMrL|`*y=cmFYD^@{41VsUA$tOekAM$=Lw*rzt&ba`36M=W6h`!H(PMd!h8o>*Vj!RajBa45-~+fI7t6|a=D$q2E#bn<>0Z2;p5YY4E9KB%3T#3-U{GD=GbP@jqMc6kpy4;l}&z%!}U-Wf*9)`Nv zP_g{MQS$WC79cA;&%VPEyYGKLd<7uJu{F;Mo7*M|1qL_f6JmVOd zpeL(&Ht?sB@0tGhoGML{-05j`lCXp2!jY-|C2XjooV+ZDoaYU>)x9^XZAB~GUjx;5 zH3q$~Gqw`efq_m=yM`5t#I@>eVk58`ilA94v&K!%FKV{pz|wcSy}nW_LF&VOd1lZ>nJp_P9m3%*C?SjvLEQjHIXqENa;x zMG!t8rDd#UC(+Y?1|{w(Qof;kcs?+0)RXWdyOYAc-E@k01siOz>f|xdL?72QKv(33 zs&4gkD^=6d8o#JoYN1RKN4z&!k=HU?u|Gck6y-f^2XH+p{*(R434dTu0$0?E$;rjo z!hoRO>Yvk;S*9S-^7&&}84-s~0AFYb38uH1Rj=)FekYz(5IhGNsDBq*sCk$d3Ul1M z{UPoxXkRbt->t?QC%dei>kmD}WFzik57>d+A8(R|?*}?r%1HD(B=|KTPUa8&C^ouTy9FciC@NSnarv?ayr5d0& zARMSi)EXZSm_I(Z`RYQlt*N?IJA_lk4nwSsKxJvjmq!O&KB=DUMez4RT|d1y33!6- zSsLvQ&^Nf8l1Q9gH$GVu=?e{l<*?GwA=)8tMM4jNzJNvGaF{DqyGmc!pH0@&;g;Lx ze6Ncf?90g8u;X5*?8K79{{-S0jTr`7Pq=OwuJt?Ya|K~QV=AK8zR9ab^IYKb=b9*R z3%+T1X|pS`X2ZH(fIa7B=zY~0fqWr)a%$;wGqH$>Yp3uH1bOM3r05np>}>}C0aJ5b zw+~|e@Us6Hd1-bk%3w?fzJccZnvL-PF<%Jv*Kn+M6HZVQTXfzZNTThu0I$e%O^#W> zcNfdtkTR!6rR*pfs|`SZe8|Qzy7x0EA2bf9SdcWk_d0eD*T{y9)kLF?9J`b~D6hRF z_8PHM>3BK72cd5B@;+YQle_c=TP}*XXwu~x+ebKZj%WHx8B-8<_ya5xUptxz$^>u` zEsmh7$~rN{6dasZca91~)UqOK5JEc%3qs4(Ra!XKPnFu-gz-rMI?K?X)t357Uc zp)c4!)GccY3VPd85A(QBzxGTbAI0Ji<~s!u^f#^C*Z;}LA{l;FI69z9>Fd!O|Kels zthMp)pyhoGQ5qqiP}%?wO1T!f@udV( zzka>_=9)N(|FrmN3C9)?a12?;fJSopN3jR!BXclg1KlP|6IzTp?;*rLUsK&9yOPXJ zR0mErN?^|Y`OOlO5k{Z%yYD}npAk{}%tnmTdidbljmDX#KYvPpmY6&0`xw?3w!(e| zz_dV=IqT!kHBk-)RYHM8SGkM@;QSO5$2yQqLS&BN*i9Bto!LB8;iuyB8|QQ0?v0$$ zO>Uheez(b%rVu_V-`j{M4z9>B$C#@6-=4XtY-Lw1Xi?2*Q9hro5hi30X8fLFV+n8^ zU3gKCYPn zvpShSuf?6obzw4#R%r9){*G^~#<@VdtM8NESz)VJc1Pb0+k6g;IO2PdT3esljRobT zxq|qys8$qURVCYotP<=(4z3bdT}qK!5B4O{|{Q} z(b|Ll*rI6Z)MH}t)CE(HRwbj|pDB@-OfgHUK>i%Arf!MpN4Lgas2{HXlc*f*K@QrK zup#iM&)iRGy%O;^-V_Yh5il>e03#3L2xwIVYKCbl$|Z`NyiUt#pjIwqcMWRY_A-sm zSbat(Uh>|)#rzw#Kh`7#IKv%I_nDa2zOg|wU53wEC|~~4aa4OoAhJ^I-l<$wa$new zSq;n41_gmYw`s~;COx$SeBcY5|FSjO4l8#A8p^%qI5_pNfQdEKsr@oeH1{Vhg*V>q z2?A^WdtbcQra@Oa!J)k|C5y{Wf2mDo~>#45Wsu&idibc-363IdcQh% zO|og+j;Sv@ezm$^U}J;Rj3M1b8P&)h5JM=vP)K6)1PwFx-K6hlBJ7p!VWF{(hWRu@ z@+dk&{m4jN!*zkgV{#^xTtV{~ahO5uUAxM)fwf!AUPe;9Gx4ht=;7C#-%iDzU2#NF-~GRCxvz461haOMh6j zF&KgSyW!)JLoGY90Kn336Vq!TYyCaC+c`7kimv%8>*bLB9-}XI!k0l!=y5-;+=#zw zFMI_?#0neSo%Dz~)AyO*a8gIIM*VQ+3T^K+{4D=&Ox!Drg^+TFnJZ4|Tl-@y3y7k@ zrKh`D{+sFHUfxc9p=h=l@2xxrZ0@;+tNfJ``HGo`vsK4ie6W-Z+f^)(GDm?nrSGsY zauw~&phub5@>VA(I@^TH`M!U+9nk|IN}-)XyKPP9!jGuU1@0`B0fncc^TZwfdU>@57Vv*7h`}=zS(;v0}Z=>47F^1 z4cby`KmS86Y9#VxYCFIHSg=rpur8Q3o@X>L@5Z^(y)y^}#rl4pML4_YL%0?WM40DX8G7XUufNnALh$ z$PiIr=BfeVO|cxGG(T~+r2O)03lzp8lgHBW0-D$r@+42moFM^u;!kk@{uZ1)azj&`L>FAB-|3Qw+2&K7AyqSK zN~|p7tbN)Fky12h_&iT~n0h0kl>`0Ep6bj3Yr-rvIgAT^1c#_Fx@$j(X&q~o!?Do-ipDFPY~n6v`hvcd$(~)&j7DyB7c+oh=n{Mp zfJ3n8{u!RE-!~!m@A3wXKunKcxX$YbVxExQ;Qo0sH91u+upeZ5a@?$A9po6Hsr;Of zP))UMSueVy8`T6k+3rai|A4n~wT18)&S4GN@?T3eeej7@k2qw~8LATr@zTD*8UCza`pxyi-zL=zAvs3Ge6DAZ z_Yok;$OqIT&g#$k2x}@q)NJ}nAD+Dw%;Q4)u+dIj>zu)qWBouCu^ewAjyAF-hN?)nC9p$DhA23>4<@ELrKDr^$dIuaBf)McRTOAC31`~l|=EQ z|9JvSO3%@K9EpPnO*W=r?eoehY?)zo;2W-k4Uaj{z8qg?hEMS*U#G5kNO=v4S#H$d0mY%cH za>%xxRQ)|w*L{+K~6iT#}Ram|W~xC2yVpu46%2m}afl$u=mc57ux9I%Ynnd=cjt zP54kqHTSpaoD9DE+LCR-pc_T-zs0la$)i(21`elQH(e z2JwEh)mjq6r<;(_7q~{QQbT7y0#pBWBep>dJBVIIX>kaqe9<|pv>INKOFWcuJl{l2 zTAurh`7Dklt8`&8Q^xutgYeqOs8zAuXc&)YldCF(bX~NMQ)Ze{PYkZVRe@^uPAiUv z%qsCHVuupD=dsPwRj;wPzBS=Yy`g|)bX6QLc(U`I&E&_f&ObuA^Urm9+?C3b$p!kJ z2-+7&Y8mS5YFxRp6Vu7ON_@c5>J}Fc^5)I;`FbPSQTol1BgpZq6p>Vaw5#CpA)n)u zi`4wXTq=fpZys!+>kZEvm%w;t>g!P>kmmnsbproe>WOipctKQ2OFK$ZhaA1jz z6)o8ZLzv0K1<_xTsUIw)CY7zhE;`X2aU2cp+%+l|yqks3XHk`z%F?mhQNv$8{ zrPz#f1Xe7;1p^&FH)l#c9NOKkLTB**SBGggIGw4}`0=4H_inAuSH?1#i9LQnC9~MR zuS`yI|4>B#f6=1A|GPx5|Mr83*aT#r0xZ<6#VYPe#9*4(3lV1EQT;O2;a^nUD+7a~ zFnu6#CO2y##0uE3m;i;%+jnaPk+E$9;n97T;1^K4?i!WbpZ!Q{Yo9{Et$feH|3$`u zV>Hr4$+G{pfaQJykJ!aakR$T)^8W2BL?feV|8;x_`>n4T&=gl#dH)5=nW+Dltonbn z-1y&q^Z(n<=+?gnU|aWr{-7j)uvzV=9(T1=Ht5bjj-MK9xE0hQs|%QTC)BIcrF|^m4%5HKu5J2@oq1bM`PCzz z>1ay?NB%=mA`B^#(Gt$9qyR8Q9z`eiW!_uo#x^GSo;b`|JS72_f zaa7W?_O?ki3r~IFQb4`>!EODQd(NAqdF_YERom%+PO}W4UwXWqsr$PE4)}&X+n8RS z+`6M~E98YJ7w%@S3PG>grrlfk8w>e%Kh++eB4I1+h*4&0*YqdCqb6d8Yhm`sZPSH? z4|ojtz~zaeTI9on|5T1;3tAifZuA=r$n4iXNl*vFRUo(B9ncA60pY7Xf?9BW&yk0hs9{k6!hXTlkUW`D(Taa9rSue@zd>CHlct0qjK+lk$Nc5yTo&M=x#JX#j0B z*75+5G-Zb}q|i-3Zz@6n<|`14r4}5lx9JC=f(WQqp4Z}|_ju z`9?1{I=SgcpeX_=F@0SVqUb-sc*-|LdNolp!MBY^9Gbwj=cj)HNrG>?9w=b+@82pe zw_PaP^EI>gQ(>?N?a^MU1b`tT*6$)^^;S9tWak41BdQYBc{;V>=ozSZgP7MlUwB1@ z`N6l{y$j?=;NhHV+t1AA_9|}ve30q$ULUxq#=fJ%t*0&wb>N%hN24y2KX#Wx*gl@o z5`VPNBMHd`HXvjKZXW@UAnEn_=8T&Z*kgLVaY1luGQJPSQ01=OaOZt?*2X*!^T~7f znsviEXPa6~hSwsvsjU0E)*I(}mkEPmcl#$_RDi1|wXn#7dDK6<+?R;-6La7;Y^b%r zn01?{H}pCB?M`e;Kbs{VUJCRBuJ!rT@?oLrVmmyKQ-QOK6I@F`L5cNYG7_SRf6i-$Jt6mclw#QmO80-X<6r>YS za^wGv1Aavao&oCVutrOvVY#_-Va=(ZhY5r6J>$vKU*_Bq#Bav4o*4&9ab)fv1f#0JnpUV|6G&VvLr012Q(T?~fMK#)jHWKfKjh*)%ffZg_)GB4^LifvVR`6VO59G%)-_f3zI{D4sW(dGABAtNt5k8$U{a`IyN;`4Txd!T?VNwI+ z)`OgeFdp2Jd`VB#@Z*J5+fuw&TcOKE`+iYKdx8&pTLv?`S&q%~fl_`h^m1VY6k|)2 zdh+{P#CTuLpbBWq3*W0T9T?0uSxND6!TqzhK5Fv(^>HiuD6%dcnB@Tfe2^8OhVKB5 z6XK;TH*LUwYnWUn)*udtPnMs+PCbU!zG3pi+4j2%=YW#_{UNn!QU&g5O{*^YB60c) zW%J$1-1;_^7b3ybI;n$cMZ%?_9gW%Z0lVqNFWOc+>({4Uk2lZw&5oLWIjnb0u26h` z%S@|$)3TRi^ae1Jf~wrD8skX6S4_zCOrNX*#@*|Q#E*a?QEyKuY6&QH5Kn09q~Hqj z=TJwv4g5(eI#j@t?m}($VX##uR;FUctwC#eiqGoeR=NW8ewCAN&2f9C#|S~`KEU>C z6p;tmoK@*ZE3O5k50iJ41BS?AJpq^XLRGR}8|{?n0rb~NSf5+Sj&m~k_1R*>Dg9UO z$IIQQgX)kEQ<&&M*7aTW@w`N~315;COq<)%t@Ep@J}QW5 z>LmI`dL;q&deo@jZk<-h>)qwHVApr`OaimS%};3YcU#vglsu^8iS0hOlp9~bu6t!4 zBOX{Lo!~Qf2^F#F%;5K4h;qOC*vCO5U78jd?-(!}^0WO!M0|2!fLdj`me7*Jc^Ma(pXKoQ%99Q5Ws2#u~&CdCauR8)V z#2yhGR`OGk#rW|Vx-SG-=j9zd!j#;i#YY0h{!R-5vjYK;h;Pb=5VFd&*jhAetJvzG z#7@M!&0AW#dDP>6=%mHg+#4YQ40|tTZ~>bN2C)!xxNFz0)Od4!k*Qd)KBtm?+%RUT$k?CVBn_fK{cdChmTX`0P;|FrH2A7BlxopMx0 z9tc^mO)B_Jkp$8Lp@?D-ppGc2FoACWNAU(S23GJujsgom*vZ;1KbKG5UH_x_h0(+J zDMG6^`ib`C=`f}vNHo}|b`jb3>1XaH(kreFWf%flyW;@ZjT)OFFsqgs(mbk$?OeVPv^8d$r%GHv-JWX`ryw zh$e`NYv<~W7MdhfoYm*<(GBEhH2xIeHPx}2InJ!(RG7NX2QoTF3%6=4FErINA@9(~ z1zabVTOh{B{&^W;iEUtS5%l4*8sxWmv1LoUb!9QWqan#JQb||KvGQ$8z}!NJQ*7nC z)c25hR8+@_9^spZO^3fe?$)zgb>izxyPU$*gld|{ZA`6`?QYCpZx2;SSNL(}AWwl3 zy9wY8HJSd=xWH_up!foqr)ELY@xL{imO5@N=~ou|>%D`=Oz~Jpv++K}hYL_2%b*e~ zA4Sl0m>i(QP)VqUqC2j=rLn#{6rS*Han-;mJ zSRrC&d(Tqz$PX}#ZNf9D0pjZIrbMQM%LG)tSdI4C#DriRoDgna%4G z7h?#d<^Q2PyPIgW^6{jGj@mG(l1Wm`)!;JX2cW9pN?2|#{1{zu(DAtOHu1gWzWX9! zC`G{UbG%xPK%Qi!kD1M>>+5Y`5gzJ#Yk8_VE;IeK*rH{L^%brUredXjYUMX;*$rs( z@a94%*&E4UDdWDA&u{&Dt8;=~p@hh&C&~ilP`-mbpKN61GIP8buv=j$SiW!Bab=XL zALjJH$Tl4SPvE`f>J1T)XH%Y9CH zO+Jk>NY}_%%**vI&Qna7ut5K2A{A%jp20x3WHh%w0v?Va)zpbo%K;HsuZZ^tCMlND znivhc*TaM5{_95#LvAlA2y<1dC* zBkY8-gl3>n^ps|cTWV-ndRmzlA(6bV;zKl;o#v;4HB6o~8KKFU)RL3c^2W0Ni%HsJ zd7OfLebynEVCAml~cG2l){RD^~qG7K4`}nNeL`;nwlN z&guK>j0yAcnKVoPpTl?XQ zp#VfHFz`zKF_16P)$P(1s$@PL zXcPVH93@?m1-V|UMrSb*(Yi8*UwK>EZz1r_5Lc-xXnC^f-wQ>{du{lZ5V8UN!&9YH zh)bp*?w(O*Ds3~Bw>GYoT^qMi6A(Tio_9Sc_qyZQc*WG%><$*$@NJ#Di*NQ3^8H}l z)t<}7=7mP)JN3+PiyL&%w0?3F6$FD0pa_snX|Mrhou7@ zKSb^OlfLk*Z%v+7c29m%u{+>1+Dg^&{z%#TbD%-RRvh#%hl50JaSIFE^G}@uD-byK zwS{^|QRSLCw_!8zi0^oP&@VXDA@oZBh6=1j3ceH{lT`0*>84d-nR zb>kfA>r*u39PyBFk_!@Z@Jqdyxh)0GuGy5PA~tjnvm19^UP*tS(Bxd_fyFQA;Lk(9 z&B?MvTV&tkHrnp*5suN(vJc44|9ZP>>S zxi9|c*5k{3vume)QweRFGa#oNKZVN3yNs761f?>md~5SD-h-)v-%0Sioc*Tdi) zhs?Bi@0Q&^sePtn76plIcGn=&7q7R8Qjh7P{rF}*MaTClG6dUhqZ!conIZj+0(J+T zfqedC;bp;`3U#9^l4@-CLv0YVJkuL(rxLaDHoV#<8{UPYv>$jYr_ zIWv?CDEvuCxdGMf!HqP%>`gA~h<_Rw@6v;RP#sWbCqP&fh3q`d#>e?NywoZ$RvM|O zG+|8#Vvc2h!ZzXZB+XR~ykDqZByuc8R~1B-=c)E?fBF5vQI&?E$qH0+zkhDWY98?& zSwcWtQD;56Sw;2YrLnBTjE`i3+S_mqt0N$o19F99!{Iw=rhA#XgKw4w*5lu#&JOG2 zQ`r7IksQvy=~VlJjbiJ+Rch?iXU(m6{3V=&b43ZpKtbJg z{BWz|9xX+(7=@jQNy_#ebb={w-3yX%D|@1*IlLpl3OR7U`KB;N=LOu?n75^~J5P0K zMCnb0=JB$3tcF_8S;zqqhJn{N?;#l|KgaS9u^EFo(lb5ZlGj5kKXJc6>x)(UC*_O^F`Ls6=4KqgNr@E7r7?dX6y zhD@pcvMQT^1}~01(}H{PR}q+vy7?dMq$p+W$MQ=nfl8_jkKNv3loz5khN5alV(T6! zcRLJDQL2?OzoHr)zg$=ogXoGmD&F8M00Ao{UPPcw*m54~S~u= zZwqqe6`sCD-rpO)lw1nAGC8qXxDCI#>=+A7{sa*3VKRH>*g#DZ%7VD+{sF zTQV!G+yV`_JT;f>6lWDAQPH?=oLhK2;a7okRI26?_8)bWrm>u;V`cAM0j%|~Dr+Vm z;-ey;{i)4_+=^W^Cry6lA@=I%RBcnq?{5Be+3DzE6;}o3^Omw$$SasY+zGuHHIaqW z=Dn3|K+C!xO^7(GNyJCReKH-Ln^xM=Fh|F)qu~v?7IrUrF1OAQ-#WRfG&}ES>7LDS z1jaoiX=q}JVp=xPa@65(4Dv{;A|0* zh1+kzoAQ3)L%xHmIm3+NwtbYvW_MuvVzgl5I^#J1A2EJ2jFwg7AzMY+eHyiZInDLEhN>YalIgJVL{MT`kT%-`>Vd zXjij|(iA`DM_%8OW~rHfxlsi_|QkY~h|AXHMoHSqFxF1^66q zB)28u8daB&n@OT{H|eh8t^m!aFHKti$QwYC+yT1(9=pN)4t&dJ9pErVyW$6V`KIbx zpg!Ao-7s~Tk1W*~koujLwlI{CjB?{qbY;Kt38Y#s+l zqrgUg{9kx@d?j9y$_l@}~pHY_iNl3r{tc$E(Yayb2g zV-~T_K2g8~DAXk2YIf#)R4jFo`HgxwW*0!tQRk}jS4MAhtRAE>ktP` zOW|Qe!6rbZzt84m)4`919ZHiZv1CFxc|x6Vn?jPh_4}2OgLV9_q|7m>s7NY{P+}xE z=H3Hif4({W`mbhm%676Wp3yNAO;6tvGoR?%MAd~|)@@az78fOWUP9s!1Bp0K+}=Y% zOic!5zlitrC}C?0#Ds-Dr&>ssgIGN$VK$xX^n$Td6v+2R9F|Ixk)56RA7BTwDmcmw z;V#m3cuJ;+XbqZ=kNcPPiRB7~INr+yqV@^mBdHVOQn6>^`*8-zm=(sh+Hus!&u1cE zJeOk=Sn9)8H5l~EU2zVe>wrB^%&1USd?q9i)8erU+T{}UW5d^CzLzX*X8c_uz#qwq+gO_2XPd9-RXB>%TmkfQ{2vCc>0b8q{#POP=1Z z{E1Q7>0~c^sC~zMARv^{zL+NOkZOjZ-Hv}#F*)tmX$0_o({PqoYE*j@K9lpfQnIVG=ajgX}<@@`-%Eo}qcXYXHkfyM=} z31|U_QfF)Az2XxoB;UEqFR40Qv7sDUrs#bigOrqRnGGWca5JSH`UC)=p?oQVF$e|z zkM+&S6 z!ZGk~_SoNI9)N2yK$uPFMTg3rhwA!L8`=zxQ99;-d&{4eP|DGYXIU|qRg0v{jW2|Bbax+I2H&4(RWfcOMr;iznz7tLS34O7>%l9b?&N|gO?vUDLeO((qqL)sBB$}n_3E8dJPi>atR z*v$LLyToyw)^MbY;GF+5bY{u%>Xe{~KL|qNQKlREhHZ#v`J=U|(}{21mLwmQdjTCA zk4F?l-m9r*)OVzW;|Kcr{M?GQhv6rMj;$ImZkP7``r-11O~UTRi{&VSM#H4ooKOJ# zS+(qK`^Z9%#dl>Uew4RK`tezYepWc-H$*>q@DeX;Ie&CQf^R<#JE^g?TfgaNQ-!)Y z6jzLF)a=10pELL?HNOifH)rQfND~uif9Z+{$Vk}AN+oM}YR+HqET0y&(&@F?*K?34 zg@dIa6!R1DelyIME%M%lpv$YU=F(_1(p8D4bsiO+=0*{}@qxprmwi?lV}p|7J8BrP z$hb*HKc~4rz931u5I#)%XW3zsag_7obn30f-L^7I`Y1k2VbUFsqh~e;_>Ca!7 zr!@v?*XS2HnfbMJ61Oy?GmrJ`X-R(X$nfnG3@`hK^%GFELy(tjRS!@Y(0ihRxtB7v zV9zUwVN7k9WYg|nayilyG|*Q8VczSqkq%hd68>b@lauGxa6^S2w@3kZPG-(${Pkhm z$VFi=Zr*w{?A09tJJmRuR58Lg^JB#TTX$} zCuHVKA4#WzkNuzWP7`TCNOOX$tz_5nr^yLbRSmlSTt*{$PpMxBNp6u-ZBe4V16pLC z=blQcj;w&av66DJ6Eves)g`r{Es>fHA4vq{!oJuSXDsAaY!Y45Djy*;7#SZP6b?xOi+<~!-TH^5qUf+x9-6U`UKjm%s9#&_fASgvguoA0r zZvQzf&Cnbit_rUKAd8%t4`{rUR#LaSXG}wC19p$2L(9ty}DZz=Y zovo|jbxJ|*j2hvmIfBDUAJ(VeF$~jIt4i^!%8jv-+B|=`(?f97ECu^iezhwA9l1xU zqwihbq^i1CO2X~5q?)TpUqSLT=Lx{(PM9`)q5L4=zGfu3W9NUaYZ>WO=@mYGJCj=#j-<*>U3Yjt8+Jd5~n z&~cp>H3%2AihIVwGrSh}$T-zIY9iHVK@>M8A#lsFXsPX7zXo>N1-F0~Ic?!gULsa& zJ4*zsk!LKj;=YkibR2wXR>o35PFu5C-$Zl%{wKw^cu#mIgvD==gRjqo2OxKg6{GNM zw}=Q8UgX@7vAcRhclp9J+(b+Un}K=HG($ zp@m7AX7%ZDqr@AaKF)ad0{+>5F1owB;qfSMgivUC4q-qpcp>X@x4`glk~FiZD{ zC$N%aA;=L-TbUQ*^_@khnp75_2_|h0{!W$L4Ib@=QObrcF*%*>?a@kxR9I8TCNdts zBmzsY*etRA_itVZdun*drb}Evu`S{gphTrigHTT3`3M|0h6ISLkY_9J1|Ix^ZeJ^W z5ps&<*24#eC74klW;9&YJjTWxOmO=;@_+RSz@JB>JkqUJZVS9kHq`r{Q;P}XzI`q9 zB_}4;dhW~pNC-Oey_EKV*iRf-o%fB!mHJ_q$&oE%YI zU9H^s_xrzmdHL#9JUHgXr`xL$_s=)i)6ntn%eefnRs8>oul)8F|Nr)$g1vbA?p>n~ zB`^Xa{uu$=kKVp5ncA!P?YeayIisUitM4|z`*K1=xB7V9#i)6E9WPb(m3FbNyqdxG z5(&|=_L#n%(AHQ`pX;|D)x(wErOkWd`i-v4BS&m?3@F8fg77??Wd^Qpy9b`qtz=M zh~ps|o$N{-cD&Nel$9x#*=TNCv90^<$P+!-)92t&u`?@auc0<+9ipKv(m(IKn>N8^ zSi?K)*hIlhoqaG|wqa+w(5f)1VVGa$`l9CC*C(}{U4*xuqt3o*$}VgO^_RCu^$Jf$ zYQvQ0&`i1nLMLHI+RiT`3f+9dUCMaKTdCYff=JFJz}*mx74ew82l_+j7KL`pLZ5zL zcxMZDp1a4MKEtNXiD+CLxK48CuYKsS!t z^CNoXLmb|nwx0)DZM9l6s?W~gBG+q5iuUbfKt9@C+&stc`1sxD44YR(LsqG+ zm+hD1Y=6()=(xDJNx(&73j`}%C*Sp&C^jh4E-@S+zMa)WB|f;?1Fl{Xz{wAsaJsk! z5Y6`E2*_)3&|e8?ovC+o&_g&`Y!^zH0yso9fCdE*jhO*~U1hBOz$Ci0oM==H@J+%g zzU-x$Kx#udU=-E;uAY8*?u$~d{|3dJBvYSRllW!xc$wP1A;cVN;@^B2tkSyZgRE0DRtZ`OUFy9Y>{t%*roKjw<;=lF z7;AKf_qk3o_K0`N;J=Rtid8m)pdzRr*3+MHF0x%PC|Y?ni6(~m3Pn@E(-}cj=bmdiLw-CPZgspFB$4UCM9F4RJLTDonJpUTc9jI~Flv z_o^}-CBN9UkaxF>$Krb906sHGSSpSCp2KS{LEvIYo72lqZfe$Qvvj_py0->i&vB@T zwBJp|W5mAb^K}bdFBcJ?Xs4nt^?N5VR8p^>2g7_qpUy0)5sl&w&0865*Z!K*b?F!y zU^W{^=;-^dl`9h8SS;yYJabM#U87Nx?yytDXU8E!m4Lo*6n}Elo&4m+*TfSZNf=T5s<_1F-Ns4e z`B`MxYRZym$boxhgN?J+yO_TpG#n}U3~i&O&+|!~?ouECaB&UKiZ&G ztA7F9aTa_Hy8#24GV=~?Bf-Unjnm(OkS;E_Jxhf9UREUFVJ4;%HGc73-Ok+iCOhUo zHW1O(yCCFTqt_My<5rt?K8Q{*slWVnaH?VqB$)Vqk72Z9U|xWb($H^n^XgUiKQc1J z-`iX2IHU1$kSmDf6t(ViO)sBLjb7$MdGs=@)^!{+G%qPEpSA_s2Lu_vR;|uF<|==7 zU{75o`6K#Fsd9bWy740W2+AP&j-Wc@y&3#0W0WW+0mJof*4hVZi9a~6Wtr(?(^SGZ z3E#|8lrWpN7xQj1IUjGwND3TWG$u@8rg=&SM;(&CB>6YfH}swlO_}LEAM~1?N4@Vq z5vEd|&V*e&i(Bt=m>0-`+eldatO|p5a5+aLG^UfS2b)zoP?>SkGSF0wXQWq(RmaIA zkFyf5x>t`F_GngR+@h~ee=iedD(CtyoQ|(Ezt(~Oj_jTq7Q5d+?LH&TNFTVbavHx|&$z-6<1KM0Ho0IX^7t3GcBMEQ)BBzd*@0{r}Gu}9WUrgZD@l<^% z+lb{bUDxQ?ZK6orB%r5s@9_C*ePVN=6%v&3DbRX>Pr(nahFeR$i*D*`^u5ZeyDD`s z$?jgLeo^BoP_h}m|CnMiB8G(j#_WyKsG@o!-$MJqdQ-TXL3~M)sXFA#K z9rPK*U#+325ljAliaRVmDsbRIyc*bdm${oKAm2ki%E{ondnXM60OmXQv;=8Qchv+% zb6`=n9+EmqBPp-3omzAO)BMz{v7ejJt#>VQNVMxH`1$kT8sx)uZV5zp&fDI@9VGlk zjXlcPj&1DEd%Fw5+nOmgX}2BA*SrRD0kIGO7#a!wOu`_!YDc=pHeRTM?t4T<ABh%l{(G{o9W9f zR89RbgK_30-nz1ngZ2ObsmFTax;jYIW6dIEN z=SQ2cteNyZ!?V}V79d%d4uF7)?G0%FQ#d{x)wuW?8Y~M8H|+MPZq=;U%8gIWmq#g} zo-FT&w_hB09us&otkLn;^8fa1s2_bnZ_H~(UP06kA(SPYY8e~}w|BiR9dhzxxoHk5 zDl2i-Uy8~4tzC@p2-b{Dj!18}is&>V*0&LcMpi+!<0rMVTzns?U6kY#8-=dRnR5g2 z02GlCnkT2VMB~>*=lGgmef>W5wwGpK!+L#7v_H^hqoQX7vW8w6mSvK>F@Zx9b%v~) zr)MJs25_(r^=1>MlT+EYs%W8`(e1DGP@H;_779BjZm2dpcA61+%$&Kg3U2>pn6U~5G_&g$k!#G}1uOKVrdWvKVNg_tu7)QNm%SU)Q(E^>gVo%RS< za0O~Z_9nr{O>lppNW}S>&n%^#%-ZOOWx&ds8`g29)BMQSu=d#<&{sO`!9EQ`1N-cR z8G(o?nGtPC^55}FW(n>3H$cagO%jXp*^7GavSt`EO}wf3+gQ!eS+?PPcro<22s3cV zjF)ZJIp9w^9MKbg7Tt5=Ifdu)s0x@pSjHi7*s~f! z<{_UM&QYPr;L%~(pEUfJ1Moo3$QTf&`XJ2s@u)gZg$R&fh7C;sM*4;wF)kM_e=sDN zgW--HMeut7<-c->(9Zb!tZEA66&p!QOv}Ht-x!?eDrk-kbD~MO>?bO z%zS+o=qOC~Znes+^#Ne7XjC{NHhwzFJ2GeT9xpKQibO0^2l&^;LPpuAbzmMa zGI(^j`n#r<15asNnfH^jeJpR1)awVt7xH+Kf#T&vD~$j{M`Y)x^3NuJlm;lBtq>}7 z4d`%Z?rF7R4_K#nzE>K&y#4|f?|vn+v=mEr&7Ct+T7<#9>_ zC4@{WY5Lz+;MG&D)@T%+5Z9RB#%0Q*37)Z<9y}Sl0|WV0je?=Q{HA_T^Z3+;B>Rah z3%7W~N=qs$1 z?LB;>EJL)UpyP`QATac#ni(_unWa2Vh+uDw0Ja&d-!UeUp#dcY%ZB9;lPZ7G={>|4 z&2Q#CaoG`sy9MeHfWly1zRSx&_HkXz@Lt8@grL-F-n5SU;cE6^-vi+;g8E z8e=lQr(*wLe!j~02Em|&ypzTr#dOhs2F{|RsM--NIf zdz}U+oZ^$mShSVvk!8o73=Iv~Ok(1_HcGUeDB-G*eTdfw3x0b!Y3m5fCW4%y$BW5_ z?1dE)=L{b=cA|%c!^(UFD9MH(i)`JA;@rEmcONBgFk9pmi<{o;U(#Lp95#A#isvQG zpU5Kx(x6(ol2SHiaxf9GimA-(Wk%j1V|Zn73`$Ek+xA?<#9t+clI%HQFm;>&zvUfu zqtI3_kdgq%XTOL2$}cN-ReKoP_Hbp-6t^j<$2RaGm1IWW<1}e1EQi6ZG%>ihzj|}H zG?k@rpz--zAi;%eh`VS4nI!ju$dN~mR2?H*R2i=L+MAE^?q;6zG|JNw+qVQu z_Ji7(OsV<5*;ru)QZX(_cYsh-Tp2#rlF7hcoOCP<*lBCHi-jw6A<+K+EOX-mz%oa| z=ktN{MV|N9u>LzC3d}-(Nj`5A%6k9z&{=63w<&wHDLG{=j?fL;UcxaRH=d1e_7XR4d>mY|p4?k{h3k}T680P$WzLfev3=90yIbUW8GGE;n|Qf{o|2j zcu!fov*aTlqz7x~y)`9iiu4L;sM^Y%zU7qeO%guTh3}3f$;wpnLH#-aaYpC<*MlycOtBr!xD6RQ6!X@BM=lW=eJIrj5AuWk;B3%DA$ELu)+N_I2Of#t z-pKmTkiq>%=;V zz6@vzmQ7Y#;ddD(nxZhEDFl7FU~nYP8f<2|d@c57_BtI-7O zYwvPL-b#BxSNHS4@K-MOwZRWb|I#&F&SSRW=BI#=1cp(5cP#9(Q(bz-Rdkcq#W&tQ z(`Aj)b(%Z4-WRA-mJhMpEDy9dj(vBsC9FuZFKb5mWrV5WJs%^eEr+Tlt&639pgE#U zg(kQh{yRgwW1uJBc3(I)*gszL-#wfta%!W=T!fM#8&o8i-bog$xdDCnk z_r<{M#sn_ZmOsbuVY-~w$1z1#hy4w{jQz!bEAWJ}fXQ_=m}j{CFi@67AVy;#xjPBz zNRj-#3b?fbjR?)F-O(BGbT-sxFcGsHT4Wp!$*L`i`ajrG-1Fpi&A(fyWUp z!k?kV2`JPLUf>q6CJj=E+A=3~&%{My4x7`%RK+CeANtAMG49Y3v9q4k28Ss(Kcio? zTiQ}Jw;wF$Mg1|rMSS!Zn||9sGVPG{ZHjSvGXeasB2-=#K*-UC$wakgGZA{8GA1e7 zv-s*;0Ln^AV5phu(g}3N^BJvXA^c@9v|lMp31i>%`y2Sj z7sYLeAS4Fz5zWVztrwybe!p>fS4=~V4BE(V2bt=|>IzZc9O++Wm`l+t7GWJ0S?Q@ z`-xWujVBR}``5hCx9InD?l!b*xaUK2TVxf{@)i|zmX871RB`Hz1_~DIa^$z_@#*tm zDzTTeyIb&(-z-djb_Hn-cjmQE-jPt@bDvduA@EkYOmV*pe)q<5_xXm@L7KYFIy zw11|=#c(O6gA^LW8SOLYygU9U0Iq8PR@J+C)q$#?$Yb2hPd6-;Ui^(g8#wZdqy?+ZSSHg3Mfbs5RfK8 zK%|4x2`yrwi6{t2?_GKgMWl&<2vS5q0V&dZPw1f+>7hiV6M6|Hgxoj!bMJk|Irqao z=kq)hfgakq;$0N9*L@JQ+OI+`?8a**E}Tmv&ORYntiS&4 zeV`^TF&2F~;atwiN_X{TK!L_B2|>$V=nyFOG+rZ>cwowk#+2t-;Pg1KdHhhbEJ~ae z_4IZF+ea5YS6yh$(Py<8zM<@>ViQ#sQSb{#DbvwyOu3UQ1Z7t>C$2y3Grk9r7k-!{ zNDGYmr6ti}b_ht^3S{Yti}mZr>MZr7(jFDmm{Zlor4Hqi_+OxM~-y-0t?+^`Z;#jqy zaTlwSCCvIDQ0!BuSw@`a=Je!th5{zUOj){5>oNcRe==@rc1fLgNlybi)XZlec7F*Y zwQ$*sVwU_tB5}GxLX+IATGK z!)^H44({*pF~{Mi%ZFn~sv~cnc!Dt6t1V5Y5Haw)Uicw}K6!VeMdImwC}hG>%55xD zu!~pK-^j2HF@Jh{aCeGkG?BSA>*gF1%&>vVoFBvX5v%(4 z`mKqTZkG&ja|vRZ#)or`|G?F&yZJF7>l$kKIo1TV|@B&XZ} zcj{NjA;*ual2&$dVGS>L$TsMGDx?<8#P*=K){;{$W(^N8^WE7i!%Gg1beT;%DMD1B$1?) zJXrs!?h`I_?1f-bi|%K)2pM9eBm#DUql6-B!{eXoA|7wH^?Ts@3cT(?q;G z?iF5G;+rzZOFC9mj>MTEU?u#~`d-%}94cx5Ixe1epkY#>hjRiZ487;%J~udgZl)q) zxMhhzFp2{U0TE$cv0&IrMzsuIM3&qeV^J*71U$uWaX_gYkFMFT{2I6LXTGL=006 z%+b3Nlvze!PN|Mn>#k&=B(;;KAY3&3efL_tZ{2(B6)^|6BX3TBj1)?S@=a_Z_~oA+ zqLhoPi#_$sQ4wQ^g=Guxps29^lY{$6x*~;l6JBq`W1AVK=a3e6|jH*2^ ze^HFw$gL@J;8^`yP#@WCws)`q${4+=^{BU$N_Gsj{NL_%XU9O|g@YfSD9J?k1b!M| zXl%L}2cyQZ5tl+<3#WgMZ<42TO|lUBz<5)X%Hm6RAU(sI-DeC$Zfk0(!ZC~whsRCj zd(FD%X6VRkgyZv42$+Z?FB%sRBAtpt2%Mt^`jC(~-JaM?{>^~n`*qQzeU%024k4}^ zNw>`%{Q&5aD&^h_0SLI<(CadPhj6yA#=skI8EC6d1$6q=w~ZDn3BRL~>d1KNFS1$u zQ8|<3PY|zPUvOellC>eTNd$1hlNBFTGx^pcj(V{~6I|}(c~U~55mj6Uw8jZqf@`#6 zS65}!KTFIRlJ3<6_28!^U`%Yv|7<6HGoL!iwQC{~^USqG{H%Ekj!YJ>Jfqp9h-C^| zt1_ z0NMQvOIC6USd5}gT#2QIAL_J7F=tAQa5fYOHDMyexL>FE z4JXajx_3-3%TT+gkgc+WUw3pgWUfl_WuBDFD3{dkTO6lA#O7{J&tF{c39<%bnaHOl za+GH7d@5#&ZRuSOSZe_Ubor@5)-{nY)B7Nuu2-o>QPY^N;*BZTaif&;*A9-V6vxHZ zq2InpVHprFk+AqeKda8Rg%--Sc-3W`WTAw=Jxl+~>Boy#!K}lHVEQXD3uP2g;jx%j zk^R;Qe?MQ7=s+rjXmLbD%|x$X4J zdx6&*Z;nq{x~~jkoVCO*7VOB0zX+zJ4q!L5=Gd#8iC3vt{dD;qssxM{60A^ftV&4?~48dTZ+%JwY3EhXTbIa8s2VoUFf7{yZs04 z^z9LFrU$<`c@XTaI$3)V`v=ja_dh~3Q6~I>Fa5|sBK6pAb6SA3JPZsB;{Ju30stoW z|9Qx5?mw6>(%`HtwS`;V{~)_u{sZ;)|4)JcG92AmZr=CrWo2a|^vsI*G+?%-1Uw2%(Sl-d9&q6$DC z0~y7{$e;_r_&R9~a<)pWxE#8HdzPdEPI)O`h1*{%@ZX>eF%S9x^^J{#aii*t3=qBn z5X|ufg1-(xNe4-lE5Q484g0T&`MUL_mP0I&%jfDTcp4FUO}24K}D z3)Bz}M;Sz7&d2OiI@)ruW%h)7*dB+i9qZG8?E(VMU>E?#%rD=FX^ctBD-+jAm>9f| zbwDCBJ??PdZ|~2fw|_ds&csv*UmdIsT@{2^_OY^6H)ELVtux8qI{3}8OM8`ps@{6_pGWMxYs_@vI?nJ*S)`w zM_Nm}R>$kwcz@h7T3pPpGDcqppipCpvTomh%o;-5(Dj!1bcuFR*0+XMjuYSt=Zq_ar03uspAyX)1;VO|5FclQDQpSW_^dH;BM|nTS zhn+o16fjty3uD8ippxJ~vB6y!8IKn9Z#*e8o>x|fZ+-7P6)*qSa zPPQS)F^(jfNtmdv7p(l1qP+Wc_#5g^F8hjsY9r$Tb)@<5IP(GS7`@_-nM_=o zil+;6=)%NdqH@*Y3{TAQqNat=Z}W=2#~TjXz6W(`(M#oOE<&yP`H$BFWbX|F^J_%T zj4OZ_8CE^cyfeVB@Spx_iBJ%ugd2LS4@i^B#(ZJCXB$s>i{UO+h+;fh@Kxe_&8 zAAVUW>1^^0huv78EEjWGcxaRsZr%M=Ls@;HyZdoWD24m2w@EowK#DEK7swwspxXPe zgX3nYN3d98Ajxac-S#Y-VtOK{xZ`d&kV|vk5^+*Vh5|?h7XD(&HIQtZc)dVrUjACt zAP{%9GumpH$h3>7ShVnhGnIAc(0rY8=%5hS2V~_)+zMwsJJgU+gdniwE1|24A!2UL z{^I`VBExEb0ETnubA6(HP0A?ILO0BT=KMQN8H+^|{OQ7*D9l97)vMV%9ROT0f9RCX z7px^AqXve-qv%zD8~uc-uT%nDk2UTR@l(!C}d8PFNmMfBT)2Bpv z0xOx$!E6sTLc7!&SIf^MIAj*@cMFlfS_?K|cI+>AaQsy+Xdh>>5?LDHW0Fp#I6Y;2 zmIm{hR5N-iy6@+d16}%doR)UsOvt_xUL(-oz>nBqO8RD%KV#1+LT~2LQJ0AsX;Z_5 zV94yZb|S@mhn=yO)toR~)5rYrnUapc%_EE$7We)n$2qWo8qQXSR2Bdm(|@;}8FUbL z&u^^#-lp}ep>(%ohRvaP99!>qtdoH^s)}Le=GZ;GL-oJdJpzJ(=qfR_$rpuL=?woz_k)}x8QQ4wT7JH6?f^l!4cxU zz^;5%ueH;Gr%+RjO9Yyt5KMf}KJ2tzHIovO9zq~cojlAU%8Yp(F%7s;p@$Me3|E1q zP3Qi{qZFo{^G~e?w-|*@-vnfD%3SKeC?N$;Oc{pRuSo*6m>T(%*sj*tXr#Y&_txW+ zuT2`~)0m7$NI3^5;W8c9$)<9cXEgx>^J{i#u{s%mlmY;PDa&~&o8Nft z1FvL%-7gpHZV%Z;Q^XZsq$|>YqWxovxle zfuJhJCS@hJWwc-^$s<2TpI#|PX=LYAATez{BmM8tb#f>9-EhpM42N(-6DnF_`By1++`P~YL>Tr(ovSxOByUH~XhW<<<@@pI&CNhY(pml2M#t+^114;3e4WDD4Q89FL=I}C&afXRl-m?*Y zC^z&uLWYo?dvz9L*mGm;e1*lHSYoY$nooq0vIPw_Im2jDEeq^>3$8w&?e!Psat{wviCQLf@L9Q<>2G5U2&&JM?M;_U;coo;%_0f zJC~=Q{`Ji&LL-=oF9QUc2Puj5LP;h+X<0|jw-uj}2Xhj06(GJukkS!zLAu`ezkX;z zZVBE~ezgmV%LYMyKB1tNBKNu-*3jc;>9467JAQsR`Cy()gj#~!|IguJHRs;D>4YCQ zEX+W|^z3T)g<>NFmR9rMJ0*1*oJasgGvB1t%Z!EuMVj>XIX%??`ejl0wQ$IzIl8o? zUtI!@C1HubSY*5gGow~Hc9I=Fgh@8sqRf<M9$!VMsWGGF(#an6wpOPqj4;3-`%4l`+=V01fYvJmY7~CKP^kmnNC{z=9~3V!mI4Z zu=4inM*NMMmOT!$Hn3E7X~bzgd%kR#$bM?=R1c`{Vr-X|cKk#RkYxdU=}8pAVLSKd zImraJ^~PSL#?0EPXy@F!S``EBb;%8$JFtnbr4g#LR^)reAf5Yoyvd3!Ru>P_h){4e zduq8Y(sY$%;6x;;8!u}gg&TG)clfn?(p2kwsXa|UnV!Rv&Q z19NGJACNy}0HXMpH9NL9>$mD+dvAt^(-Ct+W`zb#U$q313@09wzkYGORYC^YmD0-S zCM>tGF#k;u!cH`m*}I*~Ef{3t-D~3__M33_FXzY6e3*Vt*+(cgNP%sM7nkxL`I^>X zKcFU7Js=!Lt?@F%pDMP8p)Cj~F)0q=<~UK^si2;ATjmiSqZGWJz14p9sx{SSZsyvw zHFrjm&y(vz{XmUL|9QCvnPlJ~$I#nEW30;2et%OqW9xqEZK6LTNV&8^c|u#Ki=8pN z1VA$j(_ld_JEWrxOW@IlPC(aG;%t9ji2*OQJ2MRA2#zjGrj`9FnL$}Zq_o?U%pbyu ziz)~>dRuhS6q0Dsu^UWPsKU5F-SIO7fEyr1;~eixnt{ypVsxK6QJ9g(i6_(2EIe4k zJNLYG7$}Xz%)>%F_EqOAXdn+!%kDhebq!K>)i~3pE7;-i%Br2g$=ov)Koms?x}k|3 z{qhm80?abc=T=1`WT!8t`Tt|^EPpo1 z2h%U!JN!YbYQ`F}u+uyJkPc*#6gD?K?IHB;*knOi)o}MiJ(O6Vp@m_xfGNyfI6645 zx6VFg(S<}B*umaBWc$Y{N%6k|jeSKVLotOA%4VZ=EFr>izSDOc6iu~QA)^lZtim9_ z`-VPXSrHxve(K2h)o{msMX;27_4Y6C_m`<8To%GLV%Q>=Fi~f>i0|^;Q|yjjS_4Rm zzh)Pns--TAgPmb#)$nZ0r<^YY<`swQ1Ag$&;fI>dBDI)vvG9U`qB5An%?rm9x*hm~ zy-AAL{_uyh=a%?~dYwBpn4dno*#EL_-69dCTHB1NPMq#`U|xQvZtx>X&WIRZ|hBP;<3iryVq*Rmp1xRIl-X6clcM(ue&oO ztDCDXQ3e4NhIJ5oE!p?}x)~tlZWv$~{TX{P8DP=aEBd3-wl-`(^egze{de(~_*k+( zM8Ijg_umz&De%t!U6cQ^$k9J>y8mEx{lCt@{8vZ)Kctwto*oJ37k>OhLUnV-o!sN) zSL^6(hh(oaiJ$HKLoA)|{@A0f%|#;_1w1Et$@o!TYH@DW8n4bj1kTL2pY)~zN^Vip z9IsW{8G2Z#{YL=PHSzoUV654{n4Ei^E0rDr{W@Uf) zhtUF%cL`6-y8HN_Hcz#>{!Kt7G=6%6^G};wd3pby@W121JW(Ga_~ry{Yx2jX$&KW%Q^`}pt4_WQfPY5vpZ!>7mp9NGbv@b4%5xSibj zyUimt&VRGP{wF`nu{+7_+_}JWk01)q{%<}J@NIfePQ%*|R;%_hgLH;1^!4Kvj>+%3 zF8_WVx2-#5fTN!=02^^*N{`4M3OIf#imrIJMoIYR9|)jU=D#AB9c{Szl2PEKdw^M) z@R1dB48~z9pJSr>Be!g)*lrPf#P5s`q1UBl6Hb~U6c~v;Hp_m0>5P6|A^@$K0}ejs z0Ziqy{)0J#SnjJ>IvftzQZVDfXh6k9#M5_prpL$otL$CdRNRAN3y}|mUgnwqJHiCf zgWwEV3;GS@0h6}pflp(RLABkul_T`` zys8tAzeK}GEI2XV21#o;eLsNh`}us9&D8ruI=zACjDotG{W$~f#t9fAA}Yv-F9y6g zCuR3W|M}3XpMlRAC@+WBmVl?Sdo>Uz_tX8H(~1XBUn#F=m-dysTnYZF@x-HE-$Hym zJ&w>HtC@*^IV)pAu`%&FxgnK)Dq`do3Rv~NqO^cG4 z*BO{VLxP=J8Gf>�XUQyJCLAMvi zsoA54dCL4eRe%)4eopfW8%{tkI^4Xf>tsKLxPJxm=CWmyLV8#WkjYtEP7Mp}sw~ zTVQe&z%6X;sAy|yS&2JNhZ9TauiLTEA1~{sMAuUHVb~ZO`_Z$QK7LGC?pabN3u@9a zACp>&9>Qb>!1_@g1$%W1V}DHFJ})nqNQdAsY{;TC5}s4Qx8n_GZRn1gb|>k^?Gaew zzx<-`vbJ64w+;)UmQstqSAOCh_wy+}^q2&=Ij383^a4}0Z}r6AXBNdB_yr`oC&OBo z8yOBV&rnvDx&Va-y9^L?8&Dob&%(Mt}Hk4AOB0$g2LGwb-i z_rwl=)YkXx8xE_CsOr)(I_S~SRU1jKXBgcWXdRz^9`KZ48L}t^|6^Uzr+?o+Pf5)O zxu#*{JAz@YUx;}g9Zsw+V=3v@3}*W&1B7)S{PZif3SyDdF$2Qw7+MCQ((U(hid})u zM!F$tkyXKr(4`+1zMfE<;E3MH{lMFwj6vfMhI^|sWOLT7y+1_nlwi{P z!!+RQEWpRDs!i$fyAj*T&}yF^z7*i&QPuZCXKhd2sNt7}*Rj@1>H#=xGA$Cq65){@ zpgbaiNZIg~Ro6Z$b1JyK;{xDV(3S`D04^3EP0z&k_9SR3`m^V-?pEe|LeIn){x!UM zt&rJ(Ur+B+)h$u;Fw}fo90g*6pD2yPUmuSzFC#G=^fGl8D=j#&Ti0TqZ%X(y7eRWc zbO~|Clzu|)BIeNJKhSyP!5BjHewd}~%=NH_vB4*xmL=-MBCFy{fOg4V z{MImAXwgpR$|q}Ih9;xcnoj={+t`#AsMS{0WJ+}XqXuj2W$`E#bsZnJsFCaGwxZQA zfuUoDx?Gstc)+Hi`1@i*xCv^ju@@Usi%|SMna2L^!k++3_Q}< z@!sE40FLs)m&Db~Sx4jlcnF6@WyT;Sc&+a`-rFS`K25Q62uPfzBe3i%BtvZ9^BQj) z#C^A)?ALNUy`Ekm9;>r1>+m4+Y&cO`V4u5c$q~G1Q@`Rggvcl$sT+mAaB2b|OBp7! zqCwO+AQtJtqlfv%qmQ#d1TSsu?Hl!YrtX4Ai;0U}%Ii+Lp98?c9|@ZhtGMcR;GMeL zC9-249ACWjvfe-Qs~{rIdVg7rHr<^=|Bnj<-T%9F4(oC4c3aCUwmEy*kOzMzC>#LU zuoyRvlA?VJ#83iJQT-6pV>eZ2RK(6GQ_29v3q8`lX_jb>jUy#%pOuz2BC(%3a^i-`$ zX5cU(M2@V8 zPotAr({v+7W1z2;?SS9SEGeFWTX1AzT$&%VVoE;GTRC3PQ^A6Pr>9u})cFI4QMj8^ zTF$^1RHgOQWlHkfvic5t)d{2HylYc^Q%|q6r0B)GwKPgne@vbRkXW)kME^(ACMK=0 z>^XM8ixn=EN{?(TKM8-3FEdd;m=-~-ygeIgmi9fPRe$RAl5hjx62Mu3&SKFjQ2Bk- z(CLTatoGd9hYQf5z~zzl9?bMY9}r!RVofCyyhtONLAJ`5p+GAb!HzNp(1pCkmc zb6RfRIT|WuTF=X6yN|rw1j&|zNTMk}AK2s{jvfv@e#QR$lGD+w>uCld9)Kbi zTriUo^km<-6!Y?0Uhlb_nGpxpgV*eR@|Wgo8-Bf&m37Xop^0xByAZ>$2{sE$EiO2Sr`u0i(Hg?9%I2Rf`jILni=6Z$&G4tw z%y7*C%%emF*%gl1uwx*`b+&7^v~j2LCyoz%7*Q3XDeJ0~m(p3#y2DUrY@`&hX*-h6 zfn?mn4nMYFCv*&+>hJjikf~z?&Oq1dV}i8pZv@04Vw#OQHF|%(MzFE5R`&a2k#`0b z!-AN1kEu&fCnm09mgnsJQcQ$eL{RAGR9D@`CSy)$Pcl;}-SpRjd46a=30=O_po<|I zT6Q|jO;6X?`NudD{MHl25u>kXM(7cVKB9m)rzG!vvz28(jx}CDSE171_48uH+F}>~ ziCS-_5iLp#D!z7@XwBl-S4od#hF@dJ^> zcWZ5KhXn zQ+s!vfgE2oxGzz*c+z3kF?IU*w+Vr|{mXRS_Xh}lD@h}n_m9k%^YN~QwEs&kmmB{}1ik)e!%6>J zDVBk+0OVZ)bK2eI*lAH>?vo=tpHN!+wf`I^?}ASy4*gKd?U?N5~+w`Y;K z4xkF)8^S#yW*RdD!4sjV4K1CuR=o2)oux3&-vU5;&G!uT^wx0IQ=^3q4bzVwKR)## zgazC9SMmq9z7~%%?;EU$&N%vmS5!(JkZV&4_}3e#qO16lzhO$7t2Ev~Z89+jSv(r;k4@*kjI{)*q3{Jq#$tnb_t`XDr2r&P&+=EG;GTGe(aTz3>EJQQ9wG zP7A>$g){*Qt`$&Q@4&szbIXxer^r4j;AG#1Nzx^2!1s7D(E%vMd6O;g94a6mhz3MltBt=k8X;#rH?f)gZwwQ*oZEoWIRnk@`eh@X!NtvlG5 z8m}g+sATqyzF+M)^U^-iV!Tj4vHTmQ#&3nYWuo93Ks3f5Tq`-mTns;}lLcyA5L9CD zy^>zojp_|RT0l~z>T`@~3ZYvcToP{p%7`yP;|I3@-7a;G*2F6Q-W!A!vI@$;6 zCV)D}7=ZsVN(3VTy(AZU*B3>eTeH;;t#W+->0eOoOqj(?_9lzsM2VP*(JBdiwGI{I z53UAJai{z35~-a7g4e4#1qHpxYA2^SF10cM8dZD#^jD-H5>N{m7lH%5MgUcbSH6Aw zcKvucy&9NCC#UU^^w`}bTcBEG6~I(}F;(RN)g>m$d=P_CXa#1K0}2LpJTiT~R8L)J z(n<_Bse3i2SUfDf)i7`Iii1x19m8C@OBoB{k_^7HflC|*h1IhzqJ99$ z)i7N%Q~6k)2ZA=X7Cka$J4!K|&Q?naogcdPTl7$x4Jfm@vvmQ%Y6h3A5vXCOA~%H5 z6H4xti{H}gKH8awrT|FU{!~r^yo=(*b3b*PZ{?8hMadUUBU?bmXI4DRsp}NL=hUY+ z`LJCC^i#8Al&4gv&WrnbyYNz2431~SJd$sk`KwE{e6WV7lSPua-kBjuB+25qrKLqV*r?Zr_*QW}1Ay+3Y~9-jPm><#j>t_% znWH9&V?I6w?=h;*^z8t2$st;QJ7JmmScJ^^;75^N2tU;yUSXfDdjX(OeedcC-3|l+rDGaw?E z6rP?_N;v=)3Q~x_>C9EMT2kpm9Z;)tC@k}?pxYDM*OADEz?399$e*lo49Q&<2S^Z# zs9>8@4Y^AaY%ld++D%iLuud)Nk>TQiI-DmSlU(DS1i+2Bj8NQaPRL>n45*SmpoHfV zKOmi2{;WmP&xxWN<9waE$5i!*@umKLND_TCBO)g7!ihG-XNY9px9MzHkX+omMz^&| z`r^%34p0U!sURQcf(ZdckV6OD?CTbNv|P_{F;O%RBI0Bxl@cn~z3i)CFyVPpzJm^Q z=s@>)1DzbpH|LpB#T(ewefNa3CsFW_spM8k_x%3l*Lfv26QoSEn-7 zvSlw1_Ls~I&JwwQkfombAmt>&a- z6!?M%fhwV6%aja4O*tm=a)8Q>|Dgaw*EG)TORrni06yu`T-b+Bo~mZ~E5dRz{=Gi& zN~u8AyG!ENte@SEDdQp85g*Rw&@?6F%fsY?@uA4NSzUL=1ab;|I9 z!NNJ1&h@Rwg2yU%jmaZy$SQ4E(0R?iit`M?ry8lbL=LU**_9lbB6`JaG{XHNl-jt1 zZ(vd#LS90XS{Ry-n`+GJE1WAFcA#yy&6psvVJBs(Lc_vc*7U;MlN5;Cj+?>{GgI3G zBb8+HL~+3{wRY8Q2$TXq+MAp&;zo{B(}8k82z-^-M8K@(ha3nLVTRvsrNp|c?@KLn z&5~pUD6)mQK7>e2{UTOq2x!3srjW?6choNWD&XI_w%?07?tz(9?Iy#>&>)!BSB=iY zFexpglc(_t0pVwi;!MA$7JNHBLmhfRE~OCjsQFQH&diL-rF&605cv8s138BU^~#GM?Gu-eRg zvnCK1GoHA$5f<@`21q*&HE&5lDNn5S>#ePJIBMoJjKV!HZR=}bJIeN>j#TAB2iW-; z#Sy}RjC-ifjsWHpr3&GSnf=rk^m4am0{|=-cTkvucLho$w*0roD}a?=Lc}PLq*@d2 zxJ|4+6#Z%Zswsvk_J&*2YW&{(4PpVee&HlW+7NZHTpf7{qgyG3CwT-dJ6xEEpICW% zGSm4BIWhm;YY_FJIjg~Dzs+6YbhcMFsCLCf8BA~XTM?f(VLCn9B^4#QKr9X+AIcTV z{Xzi=(L{)&{D#q;UqY1GM`FvYWvrN(^9UlLjJ|0KZ>MBh1xzmmxhe+L4A!m6)%JOG!4U57< zQ(BKZF9TAB~^a^6Blpk%9J{HF^(^6%t!rOTbCJqA8uMaS zs%nT8ECxuku3wUbcsZEjLxF?2#^jH=;rl5jbc&mny7y=L)At;=FFJW9jdJUl$Er1?cUA#iS>a}8lS}DoTAA2?^zcSU*!gg;T(rjr~&+`#9`j*SVgn5 zB*|JagcO#}J+6n+z%Zf)tl)NMc^E0KU|A&kvu*KSwP66YBI(@VPa75@-}NutA0}Ty z?u8Az(mPNf*Qz5lf5~u2^ATu^3 zdXZJ1Z)nn=kEWB%g@NQgi6hR3HrctvD5LC)GL`(x!Dg~+ml~rLsC3R}N?oKlsM+2d zkV6%7yt1S!=pn6iCAZC=r*Z>!@)!J{ofHdUQ)WvaNA($-9I|&o)XBHoz*KYlSf%hX z$SpJ}MBX0sdUgB~=Np~Z@z5iSBY{)Vy#Je%tNua=bwZO7NoAvK1njX1)^D? zfIuwap&%D1A{0rfslgRxW<4-Z)%;=p=Jj26J^KfKT2&y}^?WK9@t!+tMNRQ9dJkXW zK`dJqjeGGQ<^3}w!Et| zZT6A6&vQPwcDaW#zqWS9(_PS-ZRk1nC#-a!`z1ZC69nSXP^o6IPveY(RkY)DG~y)j zXibny#G=r-UoWK)$kmreEYkRE44=b%T>5W@`Re+HV?ur!rJ$>y@9A8-Mt)%;ppVmV zqp-mKbpVjocQ(oYZnT7^^%M4|a891d(tHsnh|oSt_y1aKg`K!r<#9AJNmcb(pgDTz z`2;ukwseo!F42+NXu`GFu;6Wy$B>29Yt)jxpPq+zGOyqwIupjp-n;#HO|0&JN&do( z=13@Yu={BO)6m}0EfS-10zx0}b0ksk=eE`zNSMl=y#TpbD0V#=(iNp(eNOH5OzcTS z=UQT(3~em0Yp)bJgcqbQK*}R?!A$OTK5+2NZ>3?gzwp2>X-H*|QQb;Go`x?z&T;dF zSl~~YUim8Z%c}OeSVdV-iNza9Q-&#}0#hYh*2b2*6m9m&pY2U}JIXxUgeSAh^wwi? za~Xs8-WrYryte`>6=gJaat6=vR;Z8Jj(VB(bKV!>f zzH9PXRO3!<^g(~kGsjoM_ClsSjqwYqIpiakhzDsR8iPui?F@aybMSZ(k`*+MJZGQT^I<@%XGo zh9@wjO2M#nmcc`GYx9R8q}c(?wqGeieCe)VebEqIO^0dY&zqre1-} zB;fe1J=;(+szMh8(nt*N1kM_Ykc3Lm!|ia6DiDZ?k(QSF&7~V3M@=A4o^YHK2}q-u z#lO}wmA8m~#`G8>5Tqmj9CWW$;pRt$_-xxxz;deW;%_CX5vHzhYFuJ+e}Jz|@> zz$J_uD!)i!kf%_ouaSMax9arzm8p_1vN8p+?Br$!X1$KRmM|%!KvtSZmmkrFoPc?K zyDv@pVx#BePPvPb%$>$Z_hyCG*HvEEgYRefM263luwN`0{g8 zG)xU`U{Sc8>LXSq>dAY8&f4-ThVDf@4?x%UBFUd@2VG1j0f7>K>I+KkL)8Y_)16bV zR?)n_)VxWL<%l2idrt{IGB$FOeXfn&mo-1G$M_R{#0ko?zU|yB?lxZL7(32Qyjg#E zafTRr9XA%K4JqPj)1ywtzgRXp=LGHHq_tOua8*eYZyeO7nU`oRH#Mdon!KTEsac8Z z(-+LP;@Cg-YXw%1R(izz0s6#K?AcRA6DDVPGay=0d#AoBo4zb&5`=j)CMw-2c%UOa zp(qCB4UVCX%~63Y2-yL>kob`A%Mf)>#$3E>jk_akeqnrS(XI&8tbFMq#H9IQBVAgn z6H|2Kmr!F~s+$)?FTT$Iq%@Gpv$k6+VjA9gK~(vx^0`XQCRM-dr`DNG_dQaK<#G2= z!fW>h3&ZW%MBE=kswffc^5(P*RGDd5-$*-PQbn14zXZ)cOPa9QgeJ!<`juNG&?HCL zuU}xgN%J&aAbb$sbBBv{6tjN0w5!VgdHpo%le#hWL2d?OA)@_azRv|^p_cy3w4}o# zcPz@OViP4#Uth)Ex;qY@OrBGzUr%Q#An(N~&=z=`l1p1b8v^K?p&V4WD==$j;7>Qi-!LuXU1Jh~@=bN+cr@x%G zgQkhzXkyt5>Owa;GF&1$4gFv2kxm)P`elDwYPJG@u$u>0$N1Qke9@MUfR7CcjK>tu zmvc8AJ=tVSw|gr%H41hZsCLC~=bw&^bjj^X7p(H}jbl7+&?3y`kQ_9Ifm;)!^|LCs z=juqCQWLcyViV7_R7Q>urQOaBDBaIyPzNGSJ_yH{qx==SNyZlY%W7@> zudfUD))5~{OfY+)rn0M^7brdCR4r%UfYA+G@}qKUp9Hs zR3ep$D`;eG=udc!(MUVKEZU62a1VN6_`W9>1YUT0r4N^UyUM6|#Urnn2F564^^-kSZ& z*ADeh6%Wy{g7(aLs+JTVXIJX{QfmA%P=}QEv#}spX0iZC`YfzCJY5G0J4@jo4M;?_ zs(*wvk%$i3ve~D(AGtH_5KTBBgB&JBJ)9f4tB=gxBIY8~DdWRChdDtmN%!)pV!sa4 z^>*H!Ge?iu!V{d=c|{)unkKv7c|qMNvq_zL`uYJLqegD#8H#aW?7>{FDzDVQX0GpooOLi<`cj|gYpU|d z`wsd|rijPq<)d8#{ut%&DFcHitcxl&;%g06um_RYB+3!uPwpF2Xvt{}G>Wz;%5K^+ zdMf%T4;q;r{c8O*_aH$KnR8HCi(D?oRKQRfuD1BK9pAh~45wzZyLIGN!Lrws?<-9f zB~-Eh*2b|DHg*o2k7vN#i$pRhlA6eT$Fu&bucEXnKaxd9)Y!)?p7@vW|2qB3|NSYt zl*zsiDB_K_833iP%Rg~~EZF>UVIn$>?wi+py>!cTeG1HUQj__D$DK3eW@9X{rkjoH zM?U8*=upoS+<}pNCB&}oyXw+sGkqP!_=Gw5@9wS*hU zCnW5_bM5a11La5*CvKxBGld;A|2j|Moq#=1Oae5j2yF=n+>&?yzM~RwEp)$w)&)@@ z4KyP)Vf?!nfIwk{>$mTC7NT_tTR&Bc$lom-Fc5@mT1Kr-mhw%Y8T%Ux2qaB_I<~a{ z@XHC;JR;fer7dR&&nE2&50^0j?x#ybElIf6GFN5y?5!g_n<|0&AA`dMav@x2cP1%r zCb<)y&CbUG+I1uN`D%~_!nJCwaLBYUn(%DJ7s~&YA4nDJxq%!d0m_p9`DyX^V%8be zI-kSqDorMe00sNsKj9uAKFI;n09KI6*os#bK)57%{ICCW2Z&@csX??4OJx84of$p2 z8;G!mZmJ}mBXm}ts`+_6AOKGIbwBH-6^RAng4k~!-?IMv-sygb_|tW9lCgir>TxF> zAB%0r-{DH8`D3sDx9YzCBjyK1Xm|sV2>@^V0Fs~Bn*{VWL*#%QP^Ek=FQ-lpx2iFd zjz3*?+?s8i0fN!P(9<16>+|}VdQZsV%m!|ST=b`zm<9?$uPi{=jBR*f8&z~^?$_K-J| zN{e&S{p;$noAuZ|2W;QJ7grMfDIaX|0UNa98VWcT|6VSA%~Y`{A9Ud+P*!_gU3#Np zbH)?JB5-z1stI_}^7_^6loR!PghfQx*2H$U9X8_$+e_UHWpZ3`m=z2-?%4$LAZ$g* z8P@^X8A|`{U})QTsS`M-3d7d zWSO2nz^2ArkoZh8B~6lWjPK%3Ur%g2UP^&N-z<`_dVQ(iZXv6WQe>FGaD^2I zm%F%fVa^GP)@Ra_1G;eHcAI>Q&`|DCyP*Hj43Or);V*I~m&36$TXi(*5?|@P^{J1G z>NnTaWp>&&fv5T&cJmgy=@>!X0c+up-7qx1_2Ue9-bxRRhRhJ-glB zlFUx{?5N@{kBSsu;v>vc>;SQw-0?zOrN^B%6k*QdN`WHhnbmOQG;7Zx9FK!Fo?cvD ztw-S;J@@*=4-sGa0VVk9Po&%oAc6ALR}BzLUGWr)-$uK&xgEFeIaj?jO9_}^itEJV zoXp$z@c?1ma{!FB-xAyzx-NTMrTKTOD0mFH9@vv`&7Kffh7!lU%f1$3Vd`D+ysA_l z7yE#9mFTIiR=`m!o6pcOFr~HZ3AQcME%?|x|*F1Cagz=mV$x* z@#4&YKc?KW5D&`yMDJ^cO^5=xa@ zrDF#)=>dK63(mKqV*Yv7Rvm{Js|e~Xz0Z$ZQoK1=ku#BhO7@MN`W^|CyZ0tNl`NcW zIOEe@;;MMks#1+wShk!Sezx0*i_#oChRbMUHSDOk5~>7D&bPe-BrI4*!@*Vi*apaF z-yv~0k#^gaao(Li>e|+W4nNR0<~K!>&A!FDQICk1eXQiEFwLzKD1fva$M_#?q`?r6 zNB61bP6(Eu1ycm%knviFU{3#r!*m|sXUk+*KqKkHE}quDEMdn_hKD~_T^6_LE zg<~WX1A`x-Y@a}fv1JXxU<8&iNAA8H-ZW`%DpD%Y9^%`1e3mKVX_^kmcipY-1^VC} z5-gvP6l}cEcyM>l7hEwn8JU^iA1*UcIx8!FwJ`TzscF?`1r6B$-dUS+C3u-p+um(> z!D)OY!h({L&e9F`_GPOD_zh1t4La-&d|g?~V&JX%n?Re>!5H}U+l`C%H^B^R5#o;N zz^YQiHGgueac!iW(7eg};A9v1L*n-R0U&nrK zW`75uSH+@l;yIf00rj<(TelvSItQFFVPA zNIGd&{{3JSk^e)#jEtDQ%pKvLOo&F+xMD;@5sCN2<%`#Z7{>!8VZp)BcT7$@regw8 z+OWgCl|XNc+iY?^b#jeSfm)smhEB1z7a$Y9)lDAQ42I!MbzQC#i*7ENwma*3;;^Zp zVasqR`HFlF`FbDuhM1&LyG%TBTJY$h2Uu{NKTYD5!gzsb3bP?&w?!IJDgo8bcgYrL zpS&>cc!%hV zR$=XDUpKSeO#jwOH0vf>9A8L?UanoJt$;v`M?KS<%ynxXXP3xeFs#bgz*oFxRQ~Hg zs`QwG{Jf`-Jn z{`_i#I7}57U;APersXBVDZ!DzTeTDYZAC!4rZdp) zuDfyH$`i_3cyw!Jbd*qKA@%KDDaeGpN6_rGj7venpV9IX5Z`MZ~iA*FvUlm zqm^D+>}ZLjvk~%wkpIDb3gI6g=IuEj(W_jSJ11UWY4qZwDh0+tC^=dp_AQm1Y7B9d zF&GX1bC4Q8X9`J(DW@JWunQ;F2qCoQXy{vcJRu2DLpRv;V+?C_sZ8v1(++bAn2jzw z%8uXiN>mwDD-l*5!_X7oH4+2tVmh=VSQJa?y@RP#Jc@Ndow zeCJrb0_|HrFRqta*WAZTy2*V>wOTfGPk$8>ws;6oaRMA`;l<0)zO?YVb6rH%U)6T@ zx3n=zHf%4dplaYoYRkHL?`yLb*;&)CABmC;WhS@B5x={b0-pK`Tbszh+3e1JOt<*r zqv5;IxiR*=XI)8oOGU$O!RPFP40$7ToK_8MexHUEHd-JvING+aZnCUM?OJH`~d&C(239zYGq6p z`wk~lZc2z&xlNgq6=$UnLvU+5J8acvtB1Xq-U!zY^^?dBnZL!i!@f%&xg;da9!A3x zQ6qO#h9)c&J!`v5`y|FD!rZI#Nk!9xl?PE`{&wL~>{>B!T*aapL}|iWE)}#tsocTZ zKhy%(l&yu>M)zd$&L^)av$J|yY_@??{9QTMA%>#z-(8{Uai8z3FZb`C?$q!`oszjn zmz$E}YObZ)6M;>@(BXxE=0L73rwZdo@nae@ z%Mh#7J7@OOnp^G|JH;u*A7f0z*Xn?oT&@hFPol5bjqYgBx`VVvNPMPE`bRqi7Y{?= zlE7VNhy)uwL(y~X1;2oVM_&<;?%~d)eeIbC@p%>=W8CPD=B`6$!M+jX3Qpv9c< zfP0Y*HzHT6VljNjwWb})HRt1xFd;w2`EM0S^2fy64t!Ukh)$+9JM85$G#g|r4B$IP zDRR(A(%w+nEmft$zu(ejo8(_*#23Iv`jugp%ZmySTbR6c9RdZrbHck9<9vR)5r@v{ zjq7!Nn{^YzuM6F?BPrg?_M6086d9&;wa9v@2=Ohu4+ziRKd zLe%pR;z%{;5L(VV-zaf*eXM?aA}Bh)GA;61T@Ol}zo%l{r)l|V^siN&?kIdxKX#I1 zm9qU|Gk^5-lI3uL9D|DD@PdWYTEBO_J-TTz=n6%vR>|i#IG+cHHjUGo?H9Kd$tWS5 z2Gdqhks7-^70P}Def9r97#Ypm8qM3^?hx8BJPZ%4Yf|<<4Kwb9P4-sa7R0N_b9$wV zZJeIoy%8MjBARpA{Mloa_U{4i)VfO8)vEqdp(H0|5#6Eav$`Dq?ucW3cmt?Kyy(v& z%Sphw_8v4o_~+-V#N`aIhZQE%VTCNvQ41gX;BQ)WyX#FCIL+VSNElIa);;>&__O#* zVEW_|*hN*y0VO*je@?ZKB+${8{3Dwv#AuZBlcdfJF`o3tuM+?BjT*gIZi{Ye{K0+m z|M120f`;Xu?@t~fM!UQJ1cd*aY5p&UwqP4bW?e1-&`jc1FpWTiW8wM3f0dWN*H@9X z!AA1&zHaeOz~SD)B;ZZMP53GQNl=k0D%O8v>xgPS^G@xpU1{eJ{=Nq5I7EXIjM>{5 z=}lgG#8ME6i1#O175@Rs_#$_mU8lpy-*Xl}l2JG`Y!YGK7#XWhKTDzxc)fWk>2rVi z6x;Nc`tDhE+`_R9|6PG)UPb6zv;)8CMksw9_+{`)8T{XlKE)p9eEywoU8SAjRXXKeRp|(BKts%Y$eamd5(Ge08I3Rc31|rX_AF$^T|J;-Cw`1fK_SxUa(Hw_05h| z9LAL&!7fUzIs^VukPo1UKA;2GH#c_xytByEk1&OA9wV?h_g@(^8)8JNFS05hZp8a% zzq9fotw$+uLnBu9&K~^5?!OW>{UC@wcpe}!ON_+>3@HEjUD|Lt-SXjj@~+OWW5hWg z)OaDB_4@j!G*&xaxvDm+WBymCp8kMB2Y`BpuYr@PVRLg4Aum>HiO!XgL_j1ddSjfQ z{l_`><<~!05D>*Cw`%gnk3QCS_8CuuZJc~T;|db9KNF=y1bl9iEG4(<69k?|lPhuJ z=T?Fbx3ZyD^4OB91~M7tQKS)}Ivx5A5QA@w^`LyAHGfKH5MUG04S&iKXrc--d!eM9q++FvWR)2qpbR(OCt zJwe-8CwwdaVL4&d2i52z*MH;0pJ0W);I0xNQ3R~$E}V|DWm|wgBlvJHFq;j8Cf0+Q zJMG0XJs;qUEr~_=v(Y6`()?A_NkBrq#>^*cn|Dg5T+i=90|XA~{J&hCi#hOfSXRJh=V8oCaYS7nPXKdyx6J(7qWYflK~7aaw6PJG-~Oad(9fuYu2NF@IjaYm zbP`4vim>zPuB%w7;AKFfu*onE)BQ2jVGTYsh_PC?#L^#wh-vI{v`d z?)Jm3gFl4-JW8#_?7uxMU^KlMkO;qC9ZL(RxxVZmH~?v*kG#(OlLVT|*}>%-#fuY4 zd3yFxy#e(<;uQBOwAVH5-Y5+tHAOE~q^yY2*M{b_Ul|e0u_4hscMcZ3sh2J1oCLx& z`lFJ1&#$fdZTXtrA;U&i?aX}gCuz!&X1ud*zMIAR>r~6mtok2hdc}w}oEG=1fxaOa z-#ER~ZhP;O*AxCnvEKEgVfV)#P>`ojGe7;KgvW=4%ES-KwD|FEY1H@HZcDfy8GRB* zD;Sn)68a5QWl8rk!Majpcw30t-G?q)z zp3GqVQTa3JiyV7=m+8Xi50b)|MV7mQ92BEJjjXlgTvh&Jbyd;-yL6*Btb6?0v4!Qh=IQ1UL$->T0cuA%b4+wQjbfJ^$JAAzZ~W~_ zfuW!P{i;S+dhFL3%O{;8MBCR2UUB6VJ7n&oXwe;rYYN#0!`6>9-gzOc8cg3=((z?J z|MThZG(&{zPW7FaP_c21xk*Eij20AT=#p?fc3drQjXr8L;obLDL%H~0eE5&#y`lC3 zH&s4GoC;j&*w<${w-p%@N04&<58Qqzs*O7szkB7MQ&Qxz5Tz`f!fCe|Zzl!`iFpZ4 z=KpfFsQ-#iOwN9JD~qWjKFnOTEW|U_Z82#5Tm|t0T40=#Zw8hJ<9KqSorIeIAeEnjh6Z9 zq#vj8ja2)%-X|neE$-A6hRLAQXctsm`;+$8WUd34uKfmq*QnhcumN+r_Wbvl!(w

57>=slan@qCaRf@W<)t#j2es6gH{Lz~cb@U`$Pk znq~c6pf{qXw|*C_6cdbC07pavLPm8%pm$IP$YHB9{w>WXIIp_JbqUKwkL6AO%wBAd z61s?KWxe>yVMfQY$AW8_0t34@93rGR*mqTTcx1X#(7Cj^LDO^_tMLfgV`cl9yFC3s zG<<&FHq8CAs|^qHtIf~VrJ4JBnP_BU;oT$B?yRrD{d6=#>x}yZ`x(ctE$`DQvv0m4gQK^ki7ux)pPDTx>q~!A9i#n1=zGl;6Jx`5gz-d4QtiZt z%F7dushMrslqTuS!XcGr6vFpT(YjXhPB=G*?Kn9xzM*$H9iL`0|?+`O}J9J{C>g&e&rs(U`e zI}_UgQJ@)Ir8|aS8WzPzc-KcRy;{+{msDe+CW=}C;hEpb>Ci7wnvDQuN*Qazbm|VA z@x+&~#{@T0oh}1qo32+;II4rkQ}=iTLv>vGIqeW{Xa>&;&r_K4?6Bj+1mdMv<{f

h*tk8te%)=XM_O zs;Z>>zCFerP%R`|Sey9*3LeJ07l5zXd&`W?g;hg?`|a?X`-%D{9lPftsV782oHXrP zUjK@?B(!;NK(wu5B8(GT<8g(tjXD?!$uXXCn0=&7QlZ^;Fn;!`(b(X=&z|yplmvZV zUt^DGYR7?~svARQ<$E)&uy?RqoRcr2nN)r4K?ztNZN5&Cd1NT)x>gl&N3E#D@Z`fQ zVsaZs>r=D?1?~bftGXJa2404d6&|@PEfnq}KBHIQw;5W>b*W0UZSdLtgT4ZlJbn6^ zcc=mxru$6o<@|emPez!pcxPR(O&!mXOeC6yUwn?NGg#X87qH-f4C`!5seTHIVWsA5 zFNfsja3#P_ooIjL-#)^^b~59BHde0Z0Aze!6iiBcu0>jSoVckt&Zhf$IOG(;dVnQs z4uq39ykk1{Edu>|<6i*Xk>809yU2N^Js_r|l(h{HE>`ae=*a>}*-YkFRUEWIrq|Es zlkh*;(kG5LXxI_u1kL%CIO}3M&`1cTobLz<8c32A`_y_WH){r4A zOY5>}D1=ICZfL#2M<`W5liNWsM8Fy1L~|=?%#%t$We-tM#b#AQ*lGIJb}8PT<=8nJ ze)oF5?#s%~f@~a2qn_MJW1LI^RL2{dyM~PEUq$ILvkB~(BU-0HB-BFYGKD4mTZT#z zqa@%Nn>Wsf4FX0QfM8EBS0B@!mv*nyxINbC^)q=7>u1{D9k=TlcGzX%QdWj-*49Xp zFL{ak4IO65p2Wa;sqpLxypB4#v)9*o7~bQI7rvcI()5=ymtOI5Zlh{GPw5{l9Vi7! zwK6{@gGg>T;J&{wcxARxL2fK08oL|4)<(5ZPEEO^@RdE`>So82-k4>xQbK(1tdS+_@J6`enQRx$0|_dLl6N4Xa|j{|f3P89=yDc5c>?0>;Y zfl=;lrc-$E#hw!Ay`#0Uc>f84?D7qaD2GKj6)GoDj+8U8}W1^5f0_wK>u@9UL!M+qU4 zj28Dtx;N#-g^H`@6FmbW%qBV$2bcUfF4#{v@g%(AH#}+6_tFaE(`G(4eX)AXv|RGN zx9(%Qjg)KX=VN^C+~bOOy-LJ{PQM-CgL77osYI)?EEKIa7FAWgze8>DHS~E0^>HX- z@2$883XOi=5oAhD1$S>|SXhbO!7>bYHq;tL+xU*OehL=d{C0i;_ZD zkzH=wJJ(Vk?CVyHjR@?RB7xocj(UsF+7R3L<c>>7JGI8jbX^xBzV}u>Z=m_2p*r?Z;5HlL zJiXnG`sPC)UNf<=mL1`~Q7^%7`^#vCk2J%hwb@{b9b6jw0Q&D0V8@>mXitBS3iI7L z)i&VJBE5mRf;awTwq=~;s>SIOxAY|-E;+=?GD(pS#QKsr5@}Xf_46)Lrk``pj+p4G zlluJ0{gi}zynl(2qMb|QH1*2uCaVFZyG)43vN)e06%Fs{(QQEiK7WWs43`%lFf-R; zdV=mmz!thI43NmVV60+)-%Hrycge@TkgqflMd0g*0a1bvc|A0>TJ zbifk3*~pAk^{>x#{L9Jurgl#aOr!n6BndTUGQ-UxnIT#|MM#Eu^o#7?^5cR&RbXRE z$yX8o>Q2c|=@CUiZAVo$PhOga1JO_s2(r>(&w$zx!`LTzsa(LeC@^7GXgU1c?=%c6 zq@MN^+E+yI*!YzZ+8v`Z3wArjp__G%5?7zjLeIW1aQN8cfx(RWwb^Iy1t7~Ls?(l_ zf4b1!BwNm#yyirqDVG9|>GL9;{&>$d^G>^5M*g?8e2amxzQB6E2$~g7<|4^e|8`Xk zl}xR%xV0sp`7>|IOLY3c870udM$P^v$GtOA(>e@Ou>#}if3@zMr_8waAY2s@#|#o<@L)@P&!5NUV~H^f}|-{k# zi%S|245;@7H7dO^5(8f58DqGWm1jscY3wsqFH1I@{cje=jb=1-d$5bn)YB~KalW^N z2?8Ew(d~5ir?(#$KCZCm0^#G5i+fpZ`-5wT){T+ycE5$nf{y|hI`}S8=wY|KGGEWi zKdAOH(#@Q1hze9TYbV9Ex_0`Y;iubYPhgNE`^{&mSuuHRX)Ca&fstNgB=`qEb$V;< zB7S~u;;_(mMeM%RA)R7%){W0}MWKU&`*(5AG9SUU?nCb}=dc?0^NV-Uox*M0v)ogO zb$(RSs}PI!&iRj!%}2rcW%~xVX|uI_M{r=rmrGD@fS|*kH<0tNU{Cvh3V538w|Sr&)l|LLa6eTB}7{ zy>C{pN0URh<&dMr_Y4q3yrX$Ks^go;s0;+${-N)pR#U+wA~B<_xZ}3Mo0O!_w(X}= zeRl5&q#J$Vkz%8-1Fb5`IRD2)bD%%}xg$wb`vd3` z1HTNIdMoZ9Z}ja8^GDYdy_ZikNLy2Z&t3(Jb7~1fNe5d;$iI^_RgUUig@&CMLDBi& z?_&>o%7Av4=g4wuXGRBDi^=M6-<|76DsM-|KS~SaA0PF<-QoX5a&(nG#247Vld!U; zQ6gE8!mUm!ng!hafF$4EZ6q1BTIx@MaG49ZNv&n9DSvEe3oibEbyfUkx3rII)6S^d z{op_-aQ#++w8bxF5-z>7R9uE@=HiEGLjjOoO*FLl4O$glG^@5hlm6ppcf%^U$MXyr zJiyRC{DIqpZuk2EbVf_#Ypuk{Enb7KN~rUG34)_K=xkt9z6+VFMEta6jKPt&?NBVV zdD`{kU?#wylMG@R19UGH+7(yp@*5BLv4ojlR-3>uY$EKG1y3pww}*(1pU(lG$vlCSA`TQM2qhVGV^+1g&@)$q0-PAiaKYh(`hP-Z(ue zc8^cZ4hm3vBiTr0ect|)Y!k*o_ZL@zU}?5@#TTfI{-g~c*jo5OlR$FZ!8^i8S^K@;ivpMTv!N>FJ1=lfE8FE-$O_~t%#vV&2ZcPCc>oMLaWXCHZEd=1bG<(R(tj{9DFD17 zzMG*ZfG4~x!g#t$o^1R1DbIj#!m_4yC+r}dY!O>dd+!44b2{@$*ZIjG@yb(c**fcc z5MsjQ2!k!Rxs;Z)L=mx;3a zrwgZV-#dQDfO=+yLWwm;3RS-N;2N}*K6zh@cf>OPQvY=ieu%MYHW=Simfzx%E&M+4 zyt;ZpNYRnX1;1;~L zX{$QnZ5Vq3YAbWilLQ{_ewO_>M2v?}<00854*Z%n8osuCpulI-if_g@S=`&*>kpo) zYz6tt{sYK8&JoAK1I-weS#$_p=`{^T8y1Fd--Ak@)ThEhhPe z8{02t7=-9YaD`@`gPKi0Tzz#MXDis_-HOPga%BupiU$!mD9m_WL}s(8{lV}`g)Vk+ z+xqf@rnvirYIFadiy)s!FhW`&1@dX!07_}?S^W1-u?Ry^L+RSAv3IhvCR5C~_#h*V z`vu8HUOD{UaY{rXg1(ZCg3(~5|3^jliRT?{%K#wP@VJd!|=AXRK z5tfcksnR|n*ZxX^C9VdJsGm;@s=sR{_nO6kH|$zyWXPv|1#gf3J!g`Fd**u;Ei0Sh z`1kPoWwy*l2|gPnvf}{q@Xa+zLVNuyUz^kUu?mOmsF&PDdMe%^1c+1B1h<6^_YD3r z`hX@YOGx0j6m#6xrCnLtzCB1~&>nPD@Go|5=A11>`UI+e%Y!){BByoYWEao*!HU2l z7(^XX{|chb2`{XA|2D*lF*{CIKY_g!Ukq>aX1%O#&>bQx5w!lTP^h}O#~ zUcwx+B^dJqnLOd*k7v)mYwG+Xcmf@u(H+xH<|Y z!`Ism#VE>5H0HCJHGewAw)+%I27Gwg7o+`931n#CH|lqzWs7K5`p#P3nbCPFlCoL4 z8$Nn=3Q?<+Ffs%?xU?eftlp7mmO~8C3CcvSk7bN)N++AQr}c#?7fTKMj@}4u_8wOr zOb)&+P+L;Kcd@@3Ih@fEFi>QwxcBkjCYZV2()%YKfYK}h$_^1bHmE&&FFnOZRLKx-6?^5`M;DBs}rHh zM;>sIrrpE|kZe4C2Ch~b_pO?Pieif;HR>kvITM8PFNt`yPu5Cjy%e6No@{&6=S_10 zwBCv7IOCUD%ao|5<#Jlj&Y#O&PPEGD}sWuLh*h$Zf zZeK20)*sKF(G$RSiU++6U1@o(f504+I8(0iv&(NTy#r@n6YYFx4ZTa4Q-Hfmf*Zln z#am3|?8~)H6cBLp)<)|@bJa0^!(Q79!NU7_V1PIZ)qaq_Fl6L#m84_3m-ZK>U7+do z+1Bp)Q!Wb{2>BHA?Au{8icaA<(|$`kV~5zoSLX(wcf^%Z+pOHDUwDpGLte2&9ynkz zg%K~nR-(Ao%1S5j;Q&3z@}`Ke4^&JyV{nl4WWo7d`$BU+E&BW2i|_br5rZ$1m@^N~%r|7-DkOb6A4YN5(NdTr;k9qb+SXKqJWaKHMqi`68L z$`&zFRESYCw{pv;hR=_NWOt|EFwpi}Bg}(hC_PorxhGBAG~1-4-5C%cFn<>N5)^%;`X(lZjo4gF2xZc_-Gb)wFrmY{>JHMOgf}TO6 z`7P|+PFc6ZDESeAse?khm3>hH(-}!_E3gLULlGw0bx7%gYm>$;|Le#S$;3LVKJ9G# zQm%rXa#mH8hq!vZ1=G!&?|*A7iF^Zh-{6@4H)to7TpI6R0R}ikp#0P;KO5g&7h$;( z^+};69^5yVglR-G9(xQkI8p0U z+elYz(dTRJ9cMv#MSlzv!2Hgju|4DQCP%z3p;ItrQ&}v7q67|w#pY-T6kMw1&jbYb zm8i;jh_RLh_%GD#UOXMlo>8&7w}tgJBi9?FC#ZoD6H)i8yOsWcIn*S`1_F#f1o4>j4nejo^a5Yur`;fL3# zd1ob}h+M&)*+%tdGMiTZiW?TVwF8IA@B}U-1{&ZeEhPcP9t&xM=)J_abvvQf{i9Lagtf zyj(!8&RBnr)H=8j`_=aJe8N!Ylee=MS$I8~_DsY~C}ZSjJmFOVmXvu;qfE*g5r)vA?*+P1*x|(I#J2+WSIzeqNJS2>kjEEhsP$ zr}=uFEK^x}UANY_HraJH?aLq7T+~356OeVvZ6Sa7dTiE!klALwHpX5)`)feMm3Neu znleuQ%`faV9@tdkBwJOWDEn*cy|uHiH)71n!s>cR=d+{u9eBs80zTTWx0+gjMDF}C5?7g)VnBQ^e{2{_RlePJ6L}Z#n zt8Z<|JBvDrss=P=UdYzZ{S6&K>(BeR1+>DUf6h9A3ru(*+FWrH2n9qiU+?x=)0hpdd3;T{VHqM#N{#|Qm_Ei)%mR z&UD2(Vg0)QT%A)igl~OH(`XI3cao*O8>9gQvvpx&u6Y<-()O6)>{-o!9}EBFNxR7y zkzI*1&l~KIr+-uV`t6WvNHayWg&=4rR{KR!RGV0n8i-n=-H&8mwL z&WoJhGu@_oCn%f23XYIL&lyj4#mWwnH^%CkM>5EfC;3N@Rlk4L8b>DNRVIcBiJUBa z1zKC@s(mkG&a04Lb@9KbwIE6RCm9D`JmNn8#2eJE~C0R2=d>8sMC#RbH z_5P7(MfI4T#Ebk)iv9>(SLqsY(BPT;W8bq0&az+}{DEwP8_v_sP2|J(zh4$jPLw5Q zj@QaDVG{p#XaACQ$ejfumF;441Ot%OdQPQw@IBi-*y{iH>+H}kGGmt4%FT~>qHPk$ zI~L*a%_Ac~QIDgHetGGQe`VNv{BtD#KYh9WkEitifAa;cZ9f7SG6^7(+(84T24;vK zHFfm@`QeAqydOZGZH!kw+`aSbwJvnRQE9pf5D>CkTaBCaW4k&H@YpJF8mHQW)+QVA zcfbme#6=djG5cT|du9V~5jxFUs~_O1L^vs>SHv9WSl5^5;U(~oVe6)RNcI!gN(H@| zuzYUu2f(6RwFul930zISxUTs0_q$}4+iuGZ3OCx9@E{3#w)hz8ClFu{{$hQ&ZY$x) z&xj4&|6ORg8O*8X(wId=2PKeNE$4!pKoaW@h$U zcF_LsRmJJOyFTd0glBr7iGAAJvj-c!zA4)>Mmt~+$I4{ zLrW_Hg4t;5Qg1&wqh__4&rf_$&LQ820~PP%>N6gT#bn4=vd&u*sBaD>r#2H60P&jM zZzwH9kWpdarwACLzCiK!+t;JW(m-?fFI<7^p1|M@dOSUfB4^RB!~``fl0{G(Lbp_- z^Q7$r{htaMO&35P!sb=e>3z_}XvA=Bt4TGUQK~KnPR@`1k+4I^X0nyUSVd@u33U6y3S3yea zp?}yq+0ezj9M#$c3G1$_wO7yif)>h%z57ePb1`D0OXT!p|LoqNJxk@?&#ZpD4&q18 z(7%H!CH>ek)E`zjNlxQl1v2a^U+}?3jeX@UYHBJ776PN8qlz&(><>Py@}Ez2Yyblh zDNS~5eL6*ww<)9g+s}At`>ZwLaiO1%v0fkGC1;e`+-_s@j~C{1PDYmb~rKkcJI zI-au^hMx`&w*0PZ`|Wv0aRFBXkNqRph-~7rX+ubDvhz`$Oj`GBivw*)w^>E_3C zGUcVqx+lK~jCV$U_G8$o`H!6XPSamNa>I<4%5v zEs(C8jXdp_;Fl0@#b5boe12{pTeb%H0ZaILfN01 z1Yt{Sy~~<#Xf%#i^gAn>YV7u_Zn2^l_#FVk(CfXfK~-eTPN1YneEGnx#SUNrz0HW;HUe0NA4s=z-R7q zD?zWn)v!yo>1*673({q-o2L!>KGtVG2~6y!GLA{qiaAS?Iv7g;xx;SLdkFAO-XFm9ND#Edm zsjHHX#(ZzI$!u&XeqTn|_z!P&#*`u1~ zqU=>_=A~`==pE|1L!JS810|f3FepkFw}^Lqdw1awMIxXA3&sriV#!7V3uh=bMug3( z4EUko&GDi`k!C zZ7o{jKF3btT6m*ucQ1@$*L({u#MGuF_T%o$&#s^GE4ZcU>5p@9wS4wC|7zX#+(33gGoEPzLIXBe{;N4z z$pO<~5!czICg6P?{WHMGQZ(&HBGYWZqv93#LIc}O?Oz{2*x%ZA5uYWoEJHq=e%sLRD=9iwLlOm zNIGf%^W_g`K0!*JD70ffOhG^MQk02+z_nK^)~8cXu(3U)5b22Q;bQp+r)F0`%MFps zAYFU&l-=Q+5YL&tKg@OIY>$3{J7eS5q#9!}>f6R?w0G|qvD-NdtKXN=ND&%muP})o z#MQp%rGdn~x1Z!zh2$1HZ?k!4HMv~MgP(1DZFTdluq?+=HC)_S?$7PHuq3x~b5A+q z=3J3{sk!UIv;YNv`<`&daVeT?>YJ%kRJv+!cf! zd92B0%A;ExmcBltaO%vAAtnl2*whzAY%EAREg`%9(`Oa!@(WL{%KFJld7MM;MOoFO znFP5Omh!4)Q^vtj6&tK##o97m`XiYkiYEBPJHP$dOslc+Q@ED>E9V;ib)P_0NA8EU z(l!v4CkXMUGct^xG)%I|GDH=e#*o6zxK?4H{39rWaW)9rEMTuF$nEV_CGJ~Ay2@A*qI6)1`!!aa0) zgs&OgD54mv5h)+7F{z1`xa18o+3)Xwn$DK#ci7vk+x1;PE-XC0q#W;NVbjpS&{UdD zZ)}6~Ina1S+3dWZS1?yzx+QvI!SvMft3jB#5YA7plk-aEvmavfE&Bn3?4N5|4;g+0 zSsvV9k-iwZTP%4+OxAq@IqKt59c8tAqDJ@YHRbvUALH)@EXKkX>hsF56|Ld~MPTqsHx}YlokM^mt6^c<9lS zs+c{&tD4J`K8QwgCpt3j^^gQ`?uTQes8uXvey*(JKC4PYjLQ)ozuG)LMs3Z2upfQY zUz$R)rxiFi$FJcKy)qZ4VNF)=*dCRK${LtN-)*bVJo!8VI?;sF6fmofo4HApwT=7; zYS1*fy|>82v)QCE-3sl?YhNDG4LR#q+mXTNpxZq@+?VSXBydzJQ(41V_$@aMmd4u5 z1F4)pl+372`>N_@BqVFi6?OyPo&v7>clR2-s=C9t?vSt$bj<>)bGNd&KQYE^;ayWa zdh1)WQCT0#!$_!7l@knS!llo(Onak~ zbEB8t7t-=d(Y8UoX`?jMkEql|ZeD}xO8+$H2P1BM!*g4vcwD9VuxXnP`Nc74B2TP- zoRxUJaX_t5K&VzvyTZ}AM$BN`l>nz(D21I4ns7L(r{eKm(~hOF=X_>CP6JMm?GdOp z;SDBpp}+W9FYth9F#uX|I^pTLUB4I;Mcs?4&)*B5d;Dx-e@exPQME`muv9l=Yiz$w zd~P&vrobwEVtBTD@Vbrewbz64oB@mzIc>M3A*K;br`qQ=8zduO$55r`#JYzXVr6~h zj>a-gT$^TIpuHpml=&5hx;<>m-A=vp(=$R8>ZhXW`x5=R0%yZV9F#^X3Q5t+&0__B zO8xA)BZg;l9%NBAyG?TsGZHW6^+C?fJ5h{%0%1%m@4B_8RY(W_nO}CH;jm zkBY@wP_o{7XT3(MVKFbT`I43K5IkuO={#eK>XT!pn`MqeBX?K>4fTRk^FEB+RvAE+ zm0pMQ9zCqNgq|pI8}XiPPxmn2I=__i674?j^*&=Q4Szr6zQ;L_Dam9ddvg}#p!|dg z&T*=vl-h%Q-5msls(cayqth@WwtG}m#vOddZ>)`ml-Q^kf?^BauitDG2$Q*SM)-C` za1CN6{H5LLmF!GmS^w(~KINCEioC5!ZBzHZHNZab^u@Q~lSq*|`AGI!nwxQj7E5vw z>yP9Lr0-$16fH!!JjTk`ok;Z_H*pod$N4#UP0n|lQnKS*h$uc zc0hOP>s@zxzPiImI1 z;S#c?c6=NO_CJ!c7od1W0bE0{6N`Ne%|VNwV5Z0FRfF!OgT`RgACiikHV_fMR(7DX z&|7hQGC zZE#;&wWLx^N8O5qjfrl9hYsdxJ0MLa`$D}de|5_iTy6P>xMm(Z*|;;z{kgKzKIUDU z#x&#sLp-N*d?`d)C;9wQ0AXBW3r&xw1e3+b?1lVV-GkiSGqqj?!I1q z+^)6kBNavc_0wIpJz%8Ev0$u@ixOq{pS`>|qg4oMT!oA(KPVaNBf7dJvsOR4aFJkz z9Tz4zoZi)ZM(Irq+Ivo`7A3c!aYh1cffV6|-0Qq~g`#3d%JRcNRl zv6g3h!h(QPB=D}5O}b869y(nb^fik5ac1AQ`81+-@#FAD4OVJXa@Immmw=hY+*UY& zob*9sR<-6U_Ax|-LfJm*eMJ0=YC)VeCO2i~saY}Z%j;)yPQ_bSuH(sjb*M9QaB=2l z@rYpLyBg-!{@C@0@s%B{Ri*Z^%LfieBokbYKh-E_tpu!00fQ9)47RuZ(Q{hk#&igm z4zFOkf%g+r?Lh%pz$HQikp*u|Nc`12LF7A5y=q}+cf8t~V4qo&YK;}#+9fB{DSvtP zK_OBNr0s$PhohvBzY`6tpPvEVu@Z~sWI@CF135ik$u5)QKdiFcqzz?jf8`Z2k^yYR z(S16qlKI~N_J39+)MbPS@|;=85)-}Pi=$fcr-U}i!q&<}mDu03Rm5KQivWIHXcNFT z)bM{)?gWzCDahYs^hSFX8FGMpQQcdA|D%d1n1WRKD!Kw^B13?s8Q2etUU_yIYC&b?iup(1-@Xdi3^xSgw@ z2IKFkfURC=$RkwknzeZsAfKb7#mHuI`2#Wu>&LeuFErOqxlQW>1nXbdM;5SKlY2afDgF1FU%5y?#*Wp{t|wU)mRj0qQZNa2|5rV)zq#Nt zziMl$W7dQcZrqj-K>GPw*V{4E%iwPgo8hMB1M1?H#ONP!Sf9cyKalX`0bnm43(z#7 zNyWFF*T=IkCjk!)*Df(wA94LN@z=lcs|N;;=o{{Y$&n4C)iw^@?}eo}FF-T#Eqn3{8C&lHO-W>ptu`1y=>0Wv z0nr+T>Kd+c);Ir1tz_xZrwyX@r&1$-ZwwmR?I9AnE1s$Ujv$0n#2zVz^0P^e5xB1Mumgz!}wQs*8IQFDArhDJViHZfU4K1R~?J zsS5zY&ZgDvgIWdKHd8>Uon~w!7lrTvbpX8MfyHXV`)mg;wwR!~4*(xN?24+%J#_(R ztnbDb9{>M`z4s1katru`EnEc)QdB^CC?Xvcq@xmw(nUbJ^penPyED7<&Cd6a-sEgrB+5! zCGzCvW!=-_64V9Hcz29g#|;+<--ag^dL>h1srsrPh`Co~uj&;HH;xRd7F87@<4%}r zT#o%UtZzoxaUCKBYB)7d-g(b)E0SARt~FK08&VeyVxm)a>)0TyZk_Duh!we-q$BPP zGvB40^zxN8htrlosZ#W%Sy1NHSyq1+eb%ul5+;n62ekb`J!wxoLA%*DNeWLW?-+$# z2aDA9I>WIV+U)7n%gCE(i4T68?Z41AVDY%u5xKEg%U7~HkLkML4H7`Aboy;!CF7=)K z#ih$|5<1#WO0~eiSU3E^RMKJ0f{4>V9JP zxzULOPJsBxP*1Yup-y48&bsT@AF6isPDYwR%$6PmCE3xP6@8$;cYZsk5Z3~xcwEuY z-p`R(Drr)@NrCZ9=You>6_)Ic26azX>38fEEiZvBY;3eTf+lttOCqk)HMNT8WJXJl zoQlJk3ktjMx!gX<;nwTS{pyJ*>Zr_}Xp*-E(j>BbdMZbSn!ybaul&wY26kX>nN$5C zikZ6_YlPZvt$FZDSi=Xu3<8BmN6(%rKVp7CMs~6!VAMpxxhlE0*qs~~@EnsIKni%7 zd4X6$bZ-6*t(OMTnnR`-s_0p>^iNpHn=V;7hiLPuHe)ZWb^7Lq;JIWL-^I*v*PXF$ z2J19;yA?WtFH5l>(TUnlO8v0#ZNnlN-Xbp&AVoEDGPA3VC-=Pxy49KM*0nJN995`i zEI{6Yx)0G?ClPm)l*-DNqXRam?hb~Pv)}u8r7+wJ)zU>A%74h>HQ{Cihmo9_vrE=~1tw8F_ZI zOhetcE4g+eC$XA1WunQwV!y5QHUC_9kGn0ljQ8E`Hs0r4iLKP7Wna}PI=v)U-W+dx zlNw8g5RK@4sNt3376HY4MAYeeiKltxM^(m0kb6N&Ez^XUTMMOJ)GO0e1>!84%P=7k z3krL9fBT*3qK1i?^0>WBQuvjt{Ba2G|+GiHJ0%z4~HF0 zr|7YUcr;UID1PaB#ay5AlZ5V0nx75lcDmlAuD2XSMb zQa?Vj;RmLedOv}r^`e{Bg{igWO01_-b$%omh1OU~V41Ef-OdeL?ENtxuc{M|(8w?s zlud7*lsFa0`1T4gvr>7TyIUayp}XKyT*sJ8e&3Pvf&@zlFQpp(hYK%xBdU#tUcJnT zd+{z=bZKno=&{6kk++ZaoPTab+zGF48SBZ&)rqPG{w)oY;N$>{*jq#V4>nvL{|| z5oQ?sEJa_mIb$LleqUBTEhy-n7tVgXQP`DTw%NS_CV5qS(S|iU{$qp5>66KWMKS|A z%1pl2rDlGfli%sKXKHWzeBjS;4XQs2CF@wc)L^f?k{Gt2O%l96^Mk(#ce&HT`j+h+ zFoc=fk?lVbrvMk?;plBtk*Z*AHhwqyKGC{*)MLWKCTZB{j-vX&uA-ImM9K5Tt5>Wr z=NcxSihQJ(w_~M_T>wPBFl>Y!qCce4IAN>p8!CDBR1_?oSGjwg$tu0dGaMSQn9V1Z zlg4RLhJ{kgFbT%I*z}ye;I`zhaem1q9G!%2Ok$2%vxQ>S0|BId~wX!y1x=6*Bko@{9 za+6WIUnDRXHb`kwU|S3|ijhz4qoI!RzDVTywqMa>aC!OJ_>zH(K$}BV{>}prmxcuz zvOvtkVySX#K8bshGPG@$3-!)B<*t09W5?L?Wf;~SSai)7=!MiwuL-HijkGno-g)<3 zBMmBy_eYb-*TZQ z!BD7G%w731t&r?>WScM*O!6|J`Xs`RDb|Z$A)wOF4$e)VdPi(Nf}Gi~IN2R?Y)F}@ zc&iXbH7CV+hlUtdi-)SUE%|U^9Sx1BI z*_-K&nCFVRx?ChD+5tjWv29XmxdCuxXkt}$!)!jYiu ze*hJ?*QxQ^);dYi7Gk&`{H*27wm9>HmFs7M1wO%X)n%hAIxL<9NpHuEP3crU6Ny|o zD9}TgH`13!U*TqrHWIjX3e+dJ)vqf}8e1Z|6p0mFHy)f=y$KNC_t}p0@@l=;lGNDV zF4}VJfBmI6Si&F_?OpLIoxh#o54af=*=e)yk-dY#b1~7Vn!1!h3r5X~`CqU*3HM7X zlj81#&nQ63TzlWp5G1-?+3zIKD@AeQXp#*-qt_r>0)!z8nF567%!i{lVU{>80sQ*_ zs+&2*CS!WxU8hdW>bHs`N^s{+eh-T_3A4={`RtWmnf8}-jLOPWAS#y}7|bv^5T#Bb zF6_N|#ZzX;h;yIXezHBFfxa+R2g^_1M`YXC&vCdZG*z0G!9-#o_ED5h%DS_rYi@C0 z;B%%p0XHF@q;B0-!N}?r$LI&%tE#>Lj)sMFMEf_{-{Y-Vr!7(@3t zyp1W??U{tK&6Got)Hg{f{m=l<{wyf{8oKz>>AjMS&2uPUdE9kC&^)GV6PYnN;LXOc z@YRKUc;ZjP#VcCc#(V0T`6t%K^WWOC*ac4B@!;BLz~eBB0ZT^EC}Nv{2EzKZg#5@M zhtvpPELmR!=W66TCdRqfAKlZOAL_-cQ6QMuefT323+Kf5i=RZSu=~>Vga@OqQ$}1P zu+|IxbFtJXnzIG!%@*!F)JEbW_p7L<)UzVv9j>=i#2sDT?q)Bl49_XWhHa~zDFLeV z6)~?QcPxCswQqL8w6r-fQ}98ro(5%^?cG}F+sIpbEK9HjoS}f2wpo=eqtwJEyMRxU z1TxwsFQI_Rzb~%NLf+wBn#=aA#CCSNs^-v7n;zH4POY*o1ax`G%Zh9>YP(f;fojxz z1*IFib91dy-__hkNRj}dbwXDNG%Y-_{W1~~N8H%qKIQsly9yQ#B(IMz?Zb)H;W7bU zSe3+cmaiCWM>*lVUJ*nTJFP!A7uIoqX{5=5qcZCzqI00scvv)w8r5Rm^P+2Vw94W8 z__;qYninsG_N6B(EqlKt9Bq_q9E(a*Rl2P(w*KRZl;&)Bw6+<3IlN@!?b*}cFJCw_;4HuYfdj)J6qx~TbW~n1R36Ug1x3rmoC$j&VZ$+62|LX=i10{@m1Cl7 zE!5u$_Z6Rh6mF3Qv8RchgE7S?t`DK6Vt2sn4U$Ul%h+|`pz@0fq6 zxvY|^<8{EVXftlHDk;fMi%0mXXmw(U*@1QJV9PEkQi2{C1kCS z%$lI5a`|Cca7YyE#uir&DIZ+mc^3}pi-g_sfvWsGS3YU{YI4|Jw*Cc6d^$6qd`Iok zQJD~Fq+%%U&!|c_?WLl$V!J8}Kd!gXTlr_1(NFTr4rLgLKRiC|yYE8Vx~2SR18dm<5h~nd%F@YrP?lv8GD+>I1vd??F~=+*>QM`5Iqk+?R-$> z#Z&$EMpKP^t$X=Bz&vPHfM$e-Jj>F^)u#Y~yOOfhRA=3rR|0`d5c3h;)ajy{W#Dx0vgln|uni@P{u$aoNZHCIc5j9Q4(H-Jnm1-X za|!A%(LcA@%dlr}7ST1qFZO^3E%OKe6|V@nQ6?Yq-8w4eZ_6_8pX0)R)<)i3BSDlQ zy8rzIT*qDEP`aTDQYPz+CO%2i; zz`qNOWmo-9+J*Ggn`Qj-no(8h?Q3YT$?VVn(+bt)c9rz;z<7@Y#`}N9ywmOK-(3>_ zC&%Fb`APm?{eknTAiMn6R9Gabu%{I0xZA;BDu00rm8#z*LB9aDx*tZ$&oDa=YFbvO z0T8%+-0%z7iKP{eNLs*YQc~rw^ykOLK}6U(sQ{>$K+=htPG?*Gw$o331?F3!t3ZP} zVF6MfB0)B)NAYBiZ2Tk%U3N{|;KP$_@cInKU;1lfZm;sX2Q@d#lW=VREEuy+YN_A# z-%EbCj--2(8VfJdw|j?MCwcM-?@o1Z0pq*%Ixx5g2#Dv~f(KL6l-_Odg&$vJ3{Dqq zBYhPoe=oJq=%q29N(Rd)0fmDe79*R}m0lO>@1#WK$C!nR@bxaq)Z>wWR0HjwVLJ455)N$tlhfUBm6<7hVLJ&k7uRO7_aWI!;*7OSu_ZWd> zo3B4CC`2bD;-~o2JqGSG+K*+ggK$_ES1{^c3$!QjINnzvFrk^Yy83rVluvOaDIXwm zEpWF!l2l`IU{v1#eEnBJx?M9xuLKVdh^+i7#1)Un!QD_^$9GpF< zuL`ln(>4E6#Ehutkw#-*_~;1a4VxJQ*Sa6kN#+PBE(fuO@C%lqfhNuTHHDStQ8!{- ze>;+&0-f$rUm)W_v@kr)1wMJk!o4f$y+77E$#KPff)sY3zsL4NU9K49!I(DFen=O( zw0E>(E06DP`1kqGUk7TH&ad40%;L#Zg?mimS1r;K#XnPU0l2!521qDwC%MMkNzt^W zee`XaBvEPIqQ(osg=aTU-78(k1I4g=H3%#SGEDQCPl+T+plS6kS`kYNtjkovpGY=tLq2I%Z`ykk}6N zw^}f3^;?Sf_{`Fe0U+TC5efS`Thuv>(-dw3^ncaRc&#pE-94S?g9{vf(RB zIp@_zs!($hop9ZFvjze&gy~BgbpPCGR~|uaens{^1z{CZVcS4P?s*%gGAlNN>Kd|l zyXQIaQsdQNp@R6G-#HgO-#k_UO@&G4BZ5Tj21d>Y(D|R=Nm0ZFUpyP`cC4q_Xoz0F zdWPZjvqh2M;0+V^uVN+)&0T17a3?N)N8+Jq11X%-NDZ=H;U4x$HNr^U*$1rC!eJj# zNKzy)ts9Ku!VFvP901PYn)sjRI6v2BMSQ-m{c*(L2KjsNsfSA0W$Vc*n3z2>E9adG zRhP@C2igUMhxC7aHcVG3|Di>VAL;rBi}pi}n12BCDa+WCsB#PfMb@AXv>)z7LY+C1 zS746Ub|V@;L_f{(z5?uM`i!2(l}T<=ISNb!>E)h&biAVP(OeB(_Z+sL5-OMEn9FiL z?pzvtp_~pcr3$SglF~TfN-uVSO1(ti&l;an_~c9e^%pIA;OXD0AJ73KbA5tVtiJOA zi1v~{C!Xc^p&{e6^JS{DhbAX4ESDlXh<52dw9?D^2G|^Ae#QZ~Adcx~n2ZW0YDDjv zl_-)(C}f{Nl+gZKW>doBo-?AQvScRD5It4bTIoH(&f;bHdOgl@r0MJQSC*LBQ@{_!ugyuh-HpQAUQgLJj3wQ3yzMZ z85a|9oE&e4mS=8%*nR0qQ9EYN>{9@<;7biD@6xasLnyg5S8`Rou0!e@6A8zV_Ni^m zn*V~d$=I_zt||@r0egfv4N&>8Og}D9C2xWA?m2SN0HnIghT%fhW$6c!SZ(BBNJvA4 ztM-8>Tyk{8#w!55SsZZ1bM37|c!J@J_U-$c(x*Nfr|Ow5=%>52-VR0>&v{+S79!|q zNamE&swXqKw47vo@mI%pt5+NL3tUb0niBK6;&d?tEO}zLS@xdOyW2#^-G3{l{?*g?<6WQLNx-r>yj5cM>?6Kd%b5jY(t}cJQtJ zIlU2TZmX`II+SC@b&2atoHGklOmrH_UD%^-dgX9ah-{efyxVIW!Y=k!$Sn<;T;vFU zAx{ROjFR2EzL3LNgL6z_+=XTQ(ryxw-S+`A4e#)kQr(a;UrY@2W~nV=1NRc~l*)Lx z+&xk8F}PeKLmczTt@cXZiAUPaXGgZ`>N&?!-wQ{<`3QZ)+YFh#&_`5Ob5p*?VO4!G z1>_FmAGQwB;#=_huT1ouU+<;LE|{t)`v%^Qg;*Fcm$2SGVB?~$B{LylHhoB()>8g? zuPocUkKWp119ZeaXMETm@kNHy0Pvd3aaV7v(zCsh`TC{Sux4yS2JP;!WXS2hFE*_~ zV3#}QgZRZMzAJwBQLH3CiV=m_akNF86BqAi*e~f&E0K2YloQEVI?!*-w2a!gnzUYU z{Ja6cr$)AL(V0Av!5XJ1*NXuMj%I(JXv!(6Yvs;7`gSEvIn+ndW3k2aZIojN?F$C{ znG@*plP`kXzfs4wYTJF#e4P`p#~W4D0qpa$@Ay zj$TUV!Z~5KbAH2*{kDx1qwg!fKVg1{@#-+1K8}M;2^NMNE?2Iw#te5PE-SnaG1^0B zC6?#Cy3<S%qk zS$1iA6=>D1xYFck{p6L~$f4wF=0A7Rzaef!;oyk2UVJh3Zh5Gb6*HgZ+f2tZT8}~r z5q!u+@t6~56Q;(Im7{@=i?75gir{4wE$^GXZG%x($QV+m#OYxiN~YL;KmC?m0kAh^ zl*mU#0DB{}9#aJQf_8omuz7_xv2^KB|vmftt9J-k!wbd{;{x|O^7g5&he?hb0mz1 z+6N?sF*fIGdiEj7KG!53y$)-DeO;hAn2V$ltN<9E-V%>yqjjzZRQO=LGXH$*y+mz* z%6wgl?40m&N>X{@+Q|qZxNe8`iMl4rjx1jl7>E-}&tzqXuZSfsA4JpjjEJ((#A5HF zVe(8yFy2Als(*v%i2{&FCH4)R88wR(y!?6fcQ@4}rA8ZU(;2?j8o5G@8M<|aW#QT2 z!zI_=i?@D1)D{MVOH;yzrkq(hTACo$&O}|jm&)V#(x90!X)HA+36X1hXr z7w*sK7I@BioyYb#auVg2mGo+WpH``H_{%r@57vfIy85c76OL(I8t=1*7@(n z;;c`7Y5XRoZ4q;?oQSqlIuwn}%Qu8+A80#s)1-Z|x27E)Gg#IoLpqv4t$KKP&fgsj z7|0;+CWun@xHKFZ#wO_$3OvsE@{k#+kb)mJ#TfJ${8CT#@DI4q$EjTOu25EpHdYbh z*94UHIU{#={CS#v8GNm4Um}dft<)PHHI~Uv`RKA(Ke`?=MMI4s!qJ8?AFnj>8LE@j zc3tsYH7$G2w~?adl=W=979Ncz9aH23z_?fxI&ef0ZQRPzN5!dm^5BcXpHc1s_4hm3 zm(}0zoc0y=V&sc*ooF#=e~p04+(cHe28#uj|3%j32(aP5p#uO+DMBlI3|zD>NiDvQ zt_(qRN|~II_+q%wqqV@~lEE-ismFYM9LCMW%M4Sz;hx7d;~ab{EKTA4%yub5%rms_$w>A_nl~GCm-s~dB?bi zC!;!Ps#Pz*o7pDr{qhhXk~3w!#fytaR&41}vp2L2{yUqyIV)Fa&!kNoo;J8u;P+$A zFsORuDLiRw;Oe2{sb{in<|CA&im64#FwtkSHgCqQ$Uf^E4m{Dk5mr{&BXc2BP_3@K zQtfKVxC9P;kqVB6Wp%mw^-`l2%Pr>P(vr~!D)9Rlb6vzV5 zA9ub;R+EnC8*&BVgokmkL%y9RM^S1VVTE*=ZDJ3kn zX2U%V&+6+1K+cuu+ba8kb%9AIdc!cejGAMMrp^J$(8*zb6Tsn(>>^n$7*0(SC6c|93}{ z3Jbf(tK|-6ci~K*!U!dT?B3Fa($4odQ$MN~k3g$>U({V8H1YH1k6;CspHg}R(JHo= zCSdwyJ%iK^liu4m-}hPA9xSV@y7CQEtKc}?sE@8wTH3*DW}w%`G8E>FcT#;l2!FZy z=dA?J99Az4?GbRjWR^Wv$54ZV1jhc)@%R6$2=)Iec=>-Ff=6@$ z5v$Mf0vKTS5N-A!F{Jd1L!c?!CLu6DrX6vo1>B2%*U3R=LG(cl?VEBEE=~f_KbD1; zF&IJTv$uN^{)X7Cd&K8=fHL<#vN9TgUNaJuqF?_Paep`l=2V$Bc5Ylq^Iw#Gl<^`+ z=#&lEtiq*w5p0w__CL^VyNwgT?sD#?g{ys%mgCfAnKGA=-hq>P><;oMHW2Oxm67PCV_ z(1lzFM1b;n^=Le*PyV=pFYJ(%b{jo8$4AIKOw&08=p+G6>WiA7$ax8)m%^->1#Ii; z&mRQ_1p$5Y_};)LONRr1ZVsl`n~<`Ldk<1W#eag3TtSc<(G1UTsxb3zz!&tHNtvpb{n>Kb&8H5a4x9>nW|R-@Qj%T$r+>gYYF=G@Ue`B?_sc(Anu$Zt3ppC%!9IJaYi$Dw7P4y--vFYI)^{;|y9EwsYgx&~C< zZPPK*tB+*>_8)nB(%-%K;KPNC{LD>KFB^QXuIk^|GZCoOnfBv@pkrrOz&rd#Vaek{ z$~}5TI`}gHs@xbD1kl;5Kx%GWy#_{NYMLbebSacz`3vi4(ALC5fUb&nnLX zeW4K(2q_+S27=PrX@K+&54d8J1JIHS@N9QdT^w3WLOd`|FBJpgSSzPIYby5wKcT6{CE-`S?{EPu(>2rEo96XY*Ov_iiA3=2A3GwDU)cT`t=lqG?^=r zL~MAoYJlqwDXX%O9jAe#%AgqPHpL*->-6C%j7~L~jDtBJ4nr2@mJdlaUkxA$nrOmE z^>LV;aW*rIWqBEDqvY1n-oPR=RCi2SF}gbwn7eiM|3o0hb0}32s2O7 zJjx#-X@d1DWoY&V7dJaz)-e*5CMm^~9O02Dem$P8wQE+!HLHasugvX;ypM@PqopBG~t!b*V_A>8;7p%D!CvXdhiF-CQx=qzK|Q6Ag0ADU|$bv zwfJrggv}iF1tVIwdgBo*wZiLX;@tP0h%zU?ePqQ1flYC0JUdgQL!<|?A59Trc`DOY z&FrH*H{vbZt-zJ+Ak)xW#dXV{G?<22Yvkr`hu<)piE1Vy)lhw;)DED(yXCemfW*=V zlZ5rO8DRie0mU0!9(5YqdGfc9X!w#8OCp51&m~DL$w0(Kh9wGm2O17Hdg}Ac?WIiL zqAob^+X<1m?sh!>bdPkw$V#|5d^qdiTg zIkQQ%{8s%3OR_(b!qn5Bl>IX|WHTzTa?R3z{JJ$4eZ3~ylc}_xJJx8Z8D`84fdz>@ zf8SK8zRk}{K*@fx%~Bk5`O2+neA_7a>*ov{JdGR8Cmmb&#QR2Woksu?Gs)G$N5`nr zZzOxn?L7Ky6$_X+91-O;sPqnGkVjXXw=eq!Sla^xg30D9I6J2w0$Klfcq`NWnDXa3 zXZh;7o>|@z=FvO*X%Lfw;V*TUTnzM1LmPT@_w#)r0kB6Q?|U+5ztgH4QK!(hWzO1N z0oy-e8tlxh9u5)=OrPCU!0r=^-pkWxmYht>W_OML((Ot;_sHdHf_+bW1eBKNoB;E;<>j!`YZD`7Tp?aD zJR)4t3(yv{FiyK~b816O2l3Mr_+`?`{=)yS$eurgGN(?$OvO%cM zu5ge^R)gMj{j4xzwRYQ|F(F#WoV9u==P42d!QNUv3p>;b~+#HP?sT~6M zdd$463y2x|){k=0IyQS{+UoHV6Jk538RPW>J^O$LFw{pIQf8~3I&gK5Z(rx=T#%Ac znWpqj7J8vy^4q-?N3pi~8I?4#+qrGGj*IP-7Ir(X{>*{;IABocX9mg^RIutylmd=V zR@nU$i`(A~7Tc}uzufWB3JeGlStKPxX5GAdVW`+AbnE1HL~|X@#WAI|d-5u2E0Lxf zIvvpa_uM0491_=hh9tLakGMy7Tv?6Jnw!*yKXZ&P32c#&x@Ggi?EATJEz{;&rv=r) zQLRqpxBHJU8ij>jx_KNHa#6AHXI1iCDzKaVw`pQ4!>(o#UL*MBcQrzab{@{OQE{`x zyQ{2l9@1EuB>4`%t6vbn<@;qi?jL)BTKiJT3iSY9>DFzU60G`ix7D83Bcbs-eoh~X zD=jgN4hD3)>KWX$zjr)}(mU=>7 zzc{U0m>_iUfFWD6kfu~mJav*cV9p}x23_pE+7;XK(kXZd_d(8eW=F-8@RK{V`9w1u z^1eZi4lNsqSfa4AR_2X_y$0}!$V;{?{}}cY5*k?Ehp|*Yei=r*jLtVke%n@^Tt5<9 zKh>9q&&QX)YZ(Yaw;qD*)5})jFeo1_oJ@%DmR`4qOd|1Efq%#ws2>u&7M~ttf|eN{ zK#_K8_DMesAa0DY%h-n*dSt_W4!a7(_>INfq`ikgj8&ICWuPmL`9RpjJb@@M8=q}a9HeU$F&P~;p(n&zEC82f7<4HHYoDRmY744QHywP!wF z1P(H3^r+Eds!U(ci0;VQEkzp$`})9aJdd$`mE{N5dvBo{yPe5~vEA@f#6}U8efRCL ztUzYVUg}4RX~P&QB^bx${l+V?<3dXSq^RBoby)&M?_)yqcHS4&qnbtworZ$NYsOWq zXwM^c$DZ8GyF_fT9L}t=HtzqppQq`Vcjd?Srb3qAl;NDMjBX#C*3Jm`XxKJWR$3{g zVoKNYQKe2*>V6r&vR@y3pxE|vTNGYw&q_wDioQN{rLpMWcBWhqw-Fy}*&=plKb6+t zOx>2}uS#?lI_<*-s6MEs4gc)o+7L6=TiR(A{7NGzm(mWb$`VS6D*@ECPdXzxMH3ll z4=iXqq6S;EotW7)K9s(GZ1>S`>Av#=awT*d{&yi z3{%EMc>OqXpquM=PaJ0@fb3`9QJsM|^4ComMlAvhxE@R3yE#}1Lu(tBan!Wmop%$; zVX_aFc-+C>7+GRfop4k0j;l;AE<3DDI?N=GNc#Zndp*iH+=qP*o;@S0l!~Dwo@T=W zW6lr6K=(_E_j%8oKO@gnTMbvQJ^YBI_!`3Cm@>GwN3GtuM|Z122xQM1P(;D@g-X(PEQ!rk``EW}RYHuEn6 z)rW3Clb18o;`!hseymC0Z`qC9Exjg#u;+|2n z9!L$C4cV-qz%*d9vwTO+UTX9idI*Eiele+s4YO4A8OkY6HZW5g92ZhX@on%hwl@#B z_s!NORKFdySbmzv{5~rjR^&){54DQBiK;vbX_lhNl;4_9+OfBopwIVTZZCiGCX?HG zSkrIvJL8OJkmPwsoa{}dHGot93t)S`IaB6a^l-shxQLwRrt29==0;0E-2|&QO11CV z+=~B3g2JM{@=~V^eqHA(q^TySAmDdUVrVGi%L`}@?H3$h);~2%P6eE8VB?pXaBZtQ z!aYZMXs7}&NvkBpO6uKfY@ad%!Kez&eLd~;q*T_CCX@Zk3!diRoz3v$`u})IzSdf= zxu~7j$gOG>A4nXPucYLaNVPsq&5x*+1|0$nS?_+xQB#xGZGWmgP+t`G(Z)Y_%NRRA z?d?qQVHC6;X6$f0Mk@QnN2V(eZQMQ?g@C0{_&8i*GQHQ;gy@9#`l_dHn#9!N^To6h z)W^2`C`NXA%uJzDC4LFfPqd3~(Ei1waGAqD{-@@OYdZ8V&)KxhIg?iZC1nT<^nh=Y zd*jXpsl3Zv6VfigeOh?dXF2#$CN4vIX^2KRS3y2**!@z=dOCaG@mrOG zcHX{#MYWUNJ+&=lqboi!-mRtbXac@heeUN|M6ir_wPpu87iPHMCw&|?+5D(ivxiQ3g zZEee*`~8mze${gS$Hc0w=-_<_CP&^~cGWV{L%pSz4r)Q@niXVk5C{CPKb+@--2J_+ zpd;Ohz%B1cJV#np)&0yyyGuI^!LvDPkS~t=ceYJVHM_#4A!YY3&7^OnO7M){;!dv& zYObDkWW{tgM zG!taY8RQ=@2}R+#d50G?R?h#?!5SnX;4Q$Ua>5Q(WkY>&@l4iN=&Wyga4_sdGy+8V zn)3qMp6OGQN_Xt7iAuZ{%|ujOQLsLKZJYVQsgt>uc5v|ww~Xjmq9^?PTx?KKw7yGW zbkP8hZOKb_(^dsD<1y^?7|G|$E!#L%l`tK2)-O!(GRF(FAl@p#Wt>I_7 z2bChIH4r>dVQOn!@`fua2frWxoTp84l8)NIut-HoP4Kg!NraY-!l!bYq69!lV#qYY zJ1e73Jb-f3yB!X{l1UgZzr*j&DnV4}vrZNYpf(u$)Vna2R?g$``gg+ z`Dg*8YP|b#9e>B&UlaP&wIdfsY=;L#mQ5Z1-Kf-yM}{a&1U&>#sZgkD8ZD_N%Ts zwuu1Y%tm3C7*Xt-&VCPye{Q(C5(cI7cRo;iz*FB9akM`qz|}g^F=xonmBp{nb1P3D zldE3aymDik*V$3!V5fnfHq@ke0eU=vG#XbXQtx6PqE2j2GXdM>jL)zVZ2JA{>I2I( z&ww1=Eq1qpAVU0|l@$1xa;9uG>&PZQ%`EmwK&rj;pJny8VIPiujWCW0#9u%N&OAvn zH*38oK&SO9y>o;vR#I#m*K8T}4b6d|?t(6DnMK3dw{F-@-UZI61suIBC)Jna9udQx z2x}9g;~81VD+yzQC=1GCCXlaYg%3~VR^21DG5^T$!SkCvOGr$c;tDk*xn!E7V;+

S%9X|mx&Xa`{p>H*)#G|r6oDW4Jk4TMw$<^r`+j`;&UYX#G zCEpLwN47yG2HI&Gt|t4ueLCh@ot~B3@4q8^)T@qNGT`9ZUAPV3yRT66BvHa<1H z^U9JSO?@#3{Y-pObKAKzThSOTycx0P_NT4l70#Jg;aizID|1rW#jyC$}e>8t8jHPtVz(%B}?>)D7;8nf8E*H#eUA^O>_d+kFLn_UTsbmnhpuO z5G6{A^FaXwsmq<3T;I@_tZ9Sry_#CQY4%!c#mPpHv8s!#hNEzA!V^&39oB^FHU^v?IE5Xqfj`oQ|V{F zpPJk{GDwN2(4{P9VXEM|Q^=k#+zGIE#xZMu-p-R=qIn-*oh2oky1{$a zOp5D!lARLNJ!*f0pDGqHzoG+a!dq0Y;wlRwt~PPiEX=*KJH}}+&V8C(DE}E&!Q2;V zYs#8;iT#?-M_Jl7Xuvbqke}I?Mku#qQE$YvMmNn`ts+bB90;1fm6*pe`SRs#?E3%g zA}`ptB_iaq2l7-5oiEUzj#A|k&q~DE4Q?AhZ8Aw(46uyBbKY6Tgvu4#HoVc4?Rohg zLZoqEHtMrf4FAO1SKdU$^nL^*!pUKJ+=rZz=bB9==8tGHvUfê zc4rBrOr4}IJR-G+)XqSibo}DXH2}kRKUO^<@MRB|RWVdeab<3BqJe$j9h8TB1;t34 zltDpg+jdaSU?#4=7N0VR7cVuQus&@+xSA^EM(i2U(Zi4(wE7@Z9kW+(^$S@OkhqKm zEOBTD(Y3LeD*o{ZCVAddZfhigBKhqttzFcWZ4+@O#=>KnmH*6A)4se&CN=lgb*QTv ze_=@iMr7+)lwjZx3+{a^Os{|4;p)&6SOHHGUHfpq}{PIzPUsF9;agd zO&$Wr*;Txlab~vCK)UncKABq2#Ul)3L;abQUvo@g9f)&ooRYCj(?{KS5k<6b?e@X8d)8 z*eNJ>=Afl0!SPJ~l_5z`G+Q+2PeeSzy%85GqWL9bi(`*{WHfY!&mu!5@8WJ!dH=_R z=db5|Jdb#_hiIFTuoa1TODSi-h zG9GZQ6jz(6(qE3bS*7c{taKzs4Nc=Zb16VVkJ$NPHMWH_;H7AtptTbRGVDjevU~c> z{?=qsvXJ_2^?*mv*zF+W>U|e^5H6whF2}h-^IeX7VN?5Y&bZJA@oA}*u%lk$@~hJ2 zk!?myk^sf$OPI6QG%L<}I!?bwd1fJRg+lkQ68c02-e^Rdt=cvp1`H(yVjgEBD=aZz z<{8R_T8^|vMnc!CZ&zgJsy~2XciFb4P}tUAQ<9qvb#w!SAiL2`*x@666nzK7CG``@ zmvtv~%?O4|FU9eai5iNF)#7P81VUXwpWJ^CFga^l+7*m7n`GI zeXjL|s-5GMcywyHGQ#ddxr4eDWyNIW^CMBtm+l8Kwc^UJSDm5s8mk&T8ZD~}Sje&N zuHliwoD+NN4d>%)2;&~^oBKScV~dXu`6ENdc%k&CK8uQQhZJR9-2Q7y4l4R$P_Ms= z;&I!M7m;W!ks+P|-D^aaCHy0#w=m z(EVUFes8s@M1JFHj#Vv%w%VBMdl*r>n6f&xQ$pKx?7n-@J%1_+ZmE)>i;$!Pm~)u0 zEx9j*IZ5F9&@^`4u%Vi!2`%SPzbLJki!}PNfr>=v$7!Q5gO4_&$|QwF~y@IgdTjmp5+BK@g?xg7yZ)fD|-k4^u-pG~Ke`46A{ zXK2_VAV_L~`0lqIPwC||>BQGYkPX0k4NqPH$AS=H-cu(zAewO`Vw@@#)DO&PsG$Eb zwOOZj13m7$XhBHi*3y6RwE@FN#J|3_DNgFR|HRujf{KI4e|&954EuB4fBrW{cR!K? z&TnUBDfLeg?b&#-znf#lP{v75d;f6_#)|!89Q{ARIsZRpQ~aNi>i@ex@axgl-wU7} zG)+{K2*rI;6jR83%icqhbLzPEz!5akM#}o_@RPcO0UZx`%JxY>KDU!BX&6$Sj0eOT z@V!6ggNE+OTnsC1Y$Steferq)VV{JF%z4yS#ka&o5JpweMrv8ZcWPtbIWcup0 zP_(BEyKc(C0g(v&J{}9I1u^*qQbPVmOnJm_&(D#Rc`y;s-D|l1Bt9@Rupfx0^2BJ(`qr*DryH9u&m zWEB7`c+Kh|iSKm35c(95{Z~mTYe_M;V!&0ULDdFoRk1|DBJpC7iRQKr=p*lr6>#d5 z)-;W&2cu2*o5T+w@zLJ!gyC9cX3rrK1HK4t0w3;L+wgZ%2|tZ*h}UVo^RvQa!9}*Hie+oRuzE5su4~X8)r*JEDq}JSQ~(IwqR% zk0jHjMo9;;W%^<#V1oKx;;a6Gl&8md03D-wxW0Dt_~wh-};jRCYZfa|0? z-jqKcRA(Y7l-j}at*Zg(ikj6NPQ)O1h3kQ+WoodUO3Yp!^@=33TY5C^$t&}T&4rLa{VahkWJEM z^;KKi7Kj>vwo*RsH14l&$6ygJCZ0O^d7KOkiQ}k{Tg@uN3dgSzM&ma9-3@R_joT?J zJk=c`3U_5j*Cef!gT2Cgopjj%lw9A`$J)D?uTBqJaehwZ?h{BBPA2!U!LpQ^)#YyM zjbMSrqMJKEfFrE>Ti_7Kx~{3X{c0qFpLs3;c?c8DejP;cR)_b0vfbVIUFmCbC6=H& z19A@=Rl}GMf&QbkUflE1?mfxB?n+Erb(7kqWrIoWCAaO%2CGP3C>Hvp#(;Ly%NP11 zI=PpkJ;jzNoj)*?Lm+*>?{C0vH~-V^8OiR4{Bn6cJC5`6f&LaP$GZ<*Kjz^#TR2FS zNWZwv31+Hd721eEj%@1x#@>5JHMw@{-c%_H(xkT}G^tXgOAWn)1(YVeN(Vuj)I@6N z9i)l^0s;ymO-krhsR}{>5fA}EI^P|aYrXIOV}Ij}Z|v{vea1Lz440zGlRWjF^Ea<~ z&15(W^*QnDLhoaDC~^1ZAzX6_x~ePN z{RP_c{O(W~`$xL>#eABL1h3@?(7?0K+Pa(+%G85=sH^ZZBrI2Ynq-~I^Ywe_9YLlw z&_g=^Fmf+ZnS8A<0S+yjADt=SI$P+Ys3bbG+|8{Um<|w0dgS4e{Ky91J zFjREvv|1H?(^k4Ki_dIgSMTh@?RkOTyGrKCA!Rr zRfh|1HuWTKg6G zH>C$*vy|~A->7zhMsQi#=ZoQfjT6(1V04yjvw4g5K(CgV}T5`GV(rOiS6F`zecoudWm z%@Wo&sCWeWc&oRwpXqimj*b=A)FIsqJ<}?;ny>r;^#b3}1aF%tN1Y6Fd9-sd3bolA z_R5rT)B4}cWUPc7i3*+SW9}|SAevK&&0U8ACOM}-IkF~41zGgH#8wU7Lv5@=9%Jd( z?CBWu{Z%0G{P%K%Ej1(aeWmZq+3 zFS>FumwPqUDTXWr=W(*zlNkFENEV$foL*g zXHs!LM=qK*q*V)NQ+$=hvu4UPJw{se%jIx6WlHj};hD&>k@GgR4#DRT;m^&*4|kM0Vnr%R@de=|>i z^I}t7uPO388$BXN@@(e$$qO>~sC5a~p>6D>m+oTZu^Y7AkS`5UO+)ZyuAUN=`%-|% z5BV}Lq0&5h*({|ouHt0?hx%l&{}s5U?LPmr^-5qIQ_ZaUfJO74%Z@^u_5pH7m7)eZtV*dSD9*6`lU%34euxuN0_&WW8F3$bL zg|2-X=eq4C6*F$jtuxczN=QqokpmV1cb=HNCUzQ*Lq8bp+cPzOpkrgq z2h~!J@VOtf`djhZ&6hOZbwJY2)4Cc&}v*bhhX(Ed&)uM zQsE~NVyB3jGZ88RgF+JvoCjYfc-|%?k{L|iHmD@LS763Q7okW_h3onDZ-cOVkm4tE zAA^fkysw~atlDQy_ii9-pYrDxd`7g+TzQ<1&{qFIKBd<=TFH+OgU9WSaX5}J5bXFDhU9zh1JWjXN3azreZh2$1(wqp_>_$L5qWgQ?WvSJ>oe1&}$V4BU z*vXTXoKAgi`L~j}h+$DcX&Yb)47l<6x;Vbl_ixOVH=agSnbbv>%y(xtVu4-@W<4bCa-#gz=J(rWi`5JI+(33{X4yD$r+|**eg3|o)xi`IM z@-N4`$~?_x4tFOyW({Ae;}>unrqSCn&l9>Suiw|W6heJZ2l{Yai*cjk7TZVsW8LG? zQsTCmZ12k;lzf*q#h%6r5UW}C$WXB!g&<|)mDcHi_&ell#%vkXFU+P8)dc3}exC|c z>xKGXNY3wmkv@H4I)kyBd>Z7eHTHW9-2SEs#o7ann7hf%-wv@eoW^dwtoQXp4iz?k zuoigzw53GI$&{sTv1zaS^tQo5{ZMVN0?S6RD~L`r*tm@2p1L#@ew>h~s-4$HGwnL^ zoRXXe{S%x5VVC;@h7D%*AyT!}S7YbTE=OmL3NU7lV%c|~5qlIB?yNf1LAJiS7-uzQ zKmV~lJ<+C{Jp2wlU1hk7$;DMh<;vVUi!9qut~tBjMHP4JS%E&*6*pb=cTkhjzQSww zGzltposz|cYYKT&4Fb&1*}@7mjG2xoEc>sge=Ri2mE^=3-@5MI zqce4Hk5bng;I^Ss{X}&9A=j3nPaHN9Lt|&{Ar0mbHR*i|6qS4tpW51ByQsj*MLruZgF`_YL-UJaT7> z8tOi*Q*a7ZK37xnWNNn!`QVk>9-*p}h$wBKi3Wd$$8k!yRK>M@|Fu7DFd6*9A;4hf z2*<}#BLNNrL65`zU53dcttgYBSaK`om%YYIA9eq|36`R_z2GB+3U%!KdupVY%0#)u zg9d!m;Gq{qiR5+8u&tEejp3jCU)3M6DOOxZktDO*3BKw@^qwL(TPcH&nMRFl0`K|$ zvSFDlG?#p@Oij{PrhUqDK-=@PXx!|T2G#O+i|DTh;xC^DxJhD@C zPiCtsr?~&lL!?zOg>rU_*E?)2{4RqRC>eMCZLBnQ0yMGT3ey^3nop|gX`Vcr_k9ce zZ{MoVZWc>jh%j^Zzibx$b=7CK9J^tO+`mHn#R~sV@h$qV1Ooq`jhWyWtQ!B^3mho^XJ@?$+)gOTTN+MXIO7m@gdR&pg33d=_pVP@X zU|izCeq@`C^@pG6s|HSk8n9hRGUr&6CQxYH0YJ+o;KmF3f4J&E47Taw)h7oCFgS7y z`6WmFx2a?ZXrAqLe)-PV1K8B_lY<=lFKe9Ny7KJbj~m>MuK8GI@YC5}V05T&>-kqT`58WWM=YoNB#d?TPk+6O zapXS(ohj3s$Jn8oEVFr!fbWdpko@_HE|1*rzK46#-Mf#zHc47{d-e1P-3pzqe_Px3DL|EwSK za17vsAy`V_=K&16`LXj(yTC+(MoCPfvb35%H>D=EIOW_7Ctx!Xv!P<4V!0eo@0Sz?m3oO#hmShSTAsOQ6xPDVebD)FOQ$m#Aa;jxxSJ0>sQOq!oDY? z-hYj{WwsW3VQs-NSQbisQ)Q^m5$kT1OR<@?4MwAS>r#I40X=1A3?GJdyAOYp$vmc+ z7~cn;U{90|#M@P^m9yU1JC9mSX>qOjiHx%TeW9>GYlqxGBYq6u%jQ}yW2wBHq1gv- zE7!$*5}}Dw{R_pcKujM4te9l_?_xtAF=&Di7(y^~!2P@dTDz@lgS6!%r7rG%2ie{R z-Ed}D5|E3C!CI*H~4a+q9`?5Y~;ADMfK5V6UM1tip^FN619R9%_`9V?5#c5Hf`)k7(@{R}&w*No}qv7ljJQ^gz+RGW)wf6pMKD-31Yy z*b%w3pkpO))yB*Fl+W-9bvJT>;;?m+0XrbQ3)`U)j6GK!95rIe&*uL!CgYk(8*$iN z01UQ1rh((q{{64if%hn982jK3T1tq|r<-mevvEbr&}`wKpdn}0IGmo~fiCgm#z<>L zjBAexsq+w4rtisCw#eaO+ zw0RyLYfijwXcxD_FX!H$1k*`A>r}i=*ms7-OL*LuKkf}acEVFQmR<kgKRa7RIGo4>n3*dnh&L~_AxF*5#VgrA z21~i8SyDAypE>2gt%Qweq^vt8CGX)9SW+{_th-ii{xii3OhzDzv2L5j(71aFn6y7$ z&<2h|fno@wb}7m@s1+ynI_FYo-i!hEGDYw^?#?{vfnMn{W)^aUEy4u+GJrF_iP~p)$bRCau?{`O2bDD^?!w?t_M1_#HCI74SDHiTQ^$R( zw|x;|4UjmXHwvZBQi*xlvCU~1LqmII^Q?NdqO$T6)vFRt-cQkt$d5Bp2F9PO+7G#y zp9@a;&s6kYc6M@ho{9Ks9df5S)b?Io(CCjU#kV`mo5i}n%L-K8X4Vgng)sqd2k!RH z{(C3(5Mfz5wW0BPbm+ii7Z&6A@~>R+$-2p}rhQAX7uQX!k$G2a88k@cE7kYA+Y@ro zyXU5b+4Cyw_o9t4r4EBn@L0meEgLVcQc}*Ah&jpIcimG`=IO`w=INY~SH9!lNkLEP z;=QJLXJ%n>=~tGWOzFLb$JK9=NtXUdY+@!j8>k#Ex`y)1K0!y=X(`hXoKQI;!I)+M zS`2pfM!TO_`rXL6b3A1`Blf5=&h9lw%t<{VsuSpzh9r>s2DgVibhva1T+=D!>SQ&+ zC2{Qz0?lQhgQ5K_CyHvNX1e-!=nb;0AqvXIppqo_4IdTq$gkAv^B#8nhm^>rR1IB? zx8nNPci#d6Ts#ZDv-Wuc(GHXQH>{FhRH8MtVckzS9@N|n74`(4;Zu|>$gs3pdMZhk zdI~oz;s535v&h<>z|JGmaQVxG2q{-oG~8wB^{z^N1CR6DfeUbMnY3#yW`8)I&IWhG z-PSih?mEj;O?L)7YK0?Zwy>Rw8^u&8t*tz9jKWA093r+^)ZTQZ35C-%;E+*9lR_|)yL z%Rk z)vmqRj&rbE!|6AvJNP91>eRQ4YiWwXyG--6ONAdYQMki;?6EfBdy(Y941jFiCxE}3B;URpWp}vRt$OK zcFSp*Am%vepI|BxTC0?s@RI!+Q=FP1Pa<>2x)WTra4%eg7obAq`C26pwVz?eA1%7s_fVIGrmk3i2$Kyxe^UzgHz7l0TucAAN&@3L64EDu)8 zU1u7wiQdbYwsWa|!sYjpxaIo`k>2;gsGh!;n>Fhxp9}@WZGB?J&8RPr2?4*k@Exh! zvs5+<^oJQPU)8oiK27Y^W$dJF%XXKvE{*J!_s#j+vvmKqL`B_Uml7$w*_^e0T`kup zXhN;W@ts;cWtkyO{GIE^>ylV|Mx#i2P3G?*1d$88BFR%)aL$hnsE$JI4J@>v;6*ld ziZA90PdqfCuK2uBqsJH_vBX1(`r+`Rl_bcW_34NwkwApL77c~Zx@q>dF~*Cer0{0y zEwOJDT#+iRB`j9L34bE{n4T8dGbF;l$l&(@wI-YzW(`1#@X{2ZLvA2 zij6HmPoR*6Ybs1zQrzPUEWmcsUrQ%=CiKWC9qzpO!As;PKe#f8y&}1+PHbS)1Ozv6 z+I;_3gEyKBs$!WJ*-`0$r_w= ztbFHc#bxWh@3jSrn_nW&m7#scu~_wJ+pB`^%X~Q6Wf$;wG zDU6)8;HL?74b(jmH?U2VUHeZhkA6Mi^en-?-hBz8uEdr7*ULYoaB*|TXxQvLNbmXm z(&TdQjxX9eI`!E?hPc>cJQtcn{d<>+FwLERzIxawIv!p{4NlMh!~^$#)Y-3B$AB?= zV%ZXIy363glgc{)jtyB`t>bp_B0OdX9U0}-0T6d|mtSlRN zok+Q!ga11UnT!3tcoS|p^alaA=rs%Q@rP^{f!iv(VEA2eozsWsKes>|x$tc!kTmoT*9-!WXh&@>TtMC;>oZ1kio3$&RN(BdgDfI|5z)) z6$j0GB4NM4U@|9cCcgs8aIS*DErVkduXFqQr0KXRut9d5#;Ph@J~jFH9)g56J#G75 zg9i|oHmXc^=e=(qu7)-PlMbUT#)NUI&d=anUUlI7 z!BKDm@aUKuu{nv23HF*RJI@DW{9aLIF1P|m97h7tB7I`@#hOn(>mN%{cd^;F3&Y2k z7WpmKZ^2Wn2X7(>F&{{J@teW7?uO=KSX$Fh&eRC5nQoT(vZ zKUIS26eK0;t0@gsEJRrU`)@q#4>8O?XlUoh9A04!B+B2Bp2RQp1&>$78jPF(cF~gQnv9+E=;)Ll+Q5C> zwLj%+u3P%>X>R}46k9`&w=fjyd7^M55GWOggG)Dy;jDt#XZ4b%jjAZt2zSuLJ~#D2 zEY<-gD=n;h2S}T496uCn=PWA2I>6ARP*SkRF_tKCu6?IPLQdKcbKvZ%3@`d?}X#vp`lCVtlvI{9%c8y zfY-+$2dxQ~;tUC@Ylaj`$DjsJ7FT%>sd`3+HH0Kw}-pkNZcY z5fE6B$&hI;u5A&yT6dhL1=Nq`2+d^hXx8<033sI=ZJmE&A^p)TN7=V+xxYD^-rxrmT&01I(W9RWbs{NB$hxGKMJFj; z8>w|AA#rkQ=p>=_@u8)2Z$>zg+-eSLWi)N+SKXdNjK?ST)@fGPW{O2%BJD9FfoH#? z`dP$KuAzO)WN5w+}nEKT*$2)pFZv*Uz{)Uf({-h0XNx+nW*H00&l{2bCa% z4Jx`*(FCtX4-$Hu8TQ&87{j6{y2Y9qI8{az7CPKmfe?%%vhh$j8W!iNx=wL(q-{ke zt%fC7`F&nN)Rsdvb#rxWXw;@~hjk894FZ844Lc%5e#q-amboU{Z#HG7e}I&E9!A`R zR~bm`jM1j876~#CYOrWmhvnE!c$w>o>?AG6=D_V%U~#xoar8x%Z2{_SIE_!f7M~D{ zroo!R>EGofrxVjzjhD0ZD7q>_@JX7G3mJg&T}DBcpA56KigIFv^=F!e9pUCa@|OE>89UU!cJUbFjRMipd3<4QmaldL%=w|3ICxa zM?fl@$Ix1`Ed=gbehQfXtWb)IxQ_-VuujM-)`#+r_p_)=7ZXD!+GF`u@ER-@mDH~F zQ>GG#BD$GXg`#n6-<@y<3nyjK;Tz`n_~P~HeUjJxG#8C#NtN4)mkQgB$BTy;3a6v% zkULkKh{X?CYLj)t>B(fDz*m^ry>_|lDGFTjM_aV-Kc2EZR@YZ(80)rrg2cgP3CDAg z8M^i00$%l}NDh&r&%H70Y`siHzPJYc1^6=o8!};O&hAeYWH>~6Xi3Pm!)c3>tTn}| z49?6YeNS#w2Ng?inpEA2=No^C+r#u!kHMOhwA0%g&z)IGNwhX+%OT&>RPr`+IAE*1 zmBW6-wh>aUlNnhcdW}k;Iy_JUfx4Y3@{%@p`_gj)CIULKw8L~Y1_li(s?0s8L#c3X zLa7pKG~CWyu0e4`0hZq_D|FfFXAh7&W~qvb)*|;D>!=Av5D8M7*od1*s?nysEL1YOifxV=*_a&E6GQ2^UIBk{ZW5Fa!BP&!48oh{l9RGnW zV(B$mVkGk<0Sj3S3aviW90&KGl2b~Ff7Ho0X|=&uWV<#nT-LvESGghS;ZCe4EP`Q> zwexI+D$`kz!?U@95~)o@6h@2em%Cu}0zWMhe5zq;BKeFzm4>bi?Ko<6>FKGq%!}C%R-GBv|%Gvg|pbdwDJ%;E#+_VH(xFF{Mo*QZrOC1G2EDQi}1%9wafHh(&7Hs)e<$C}XutMLY%rZR2* zD}i9KCfg~E^}iR7lj+$NgX>rZF`=wx)~hnxzk(bHcE7Z!5-6-DvTPf&ma*^<;_fdJ z=joRl18xWw&bm9jLNgT`_tY0mp`z}%O&lV#R^H;XrEyF`<1jYWpW}@ko@Z&(ZhbxT z2HMdz`b41!#3drmo$W{vN)5iD%Iv{g;)KLDvyUE>De8vjUyv=bv;wy*1)gT75OVFR zm5oT86XRzBMoMFfx-otGY!?`7Hb)}jssOPu^C$#FOpL+E%a-cfxWj_+wD{>cXmAq6s2CJm(;0?x*ZZA@t8l z5aV1RQAAMKJ}^~%Tib<0!cb<6Z$v@DsAK`gW=Ipb+6qwt6V+20~cmA7|rBjj=lVY&8n zZ@6`$UhZg=b?-^1Fap;heCN?jF|!td;BjXgq3U$cv7|APY8(x@SPpMt_gZ(y?D{((#rmerI~V3)}dkxqCJRsyt50_YNZ+Al3l|8*EwMz7I)O9 zc1S!3kxkRU&yEu5po1{G`Es{$6ch;S6lJTik;Fcnqose>KA5(U;W~C;2iGGnO3F+? z6~v&}uu&{y>{LlSJk+3Ud6yg(R?b9es?eqor11%TL_8mug$EagDVMvM#1ju$)T!mN zZ82CfA*uocE@H0vc%?+f&*MP)#7GHl@T=*r8rieYJ$nsv!(1(r>3RCsqh?TeiJpSr zv%;~5eVVeIcp(w3exE~jlcAGqJbZaq%PX#$)s-tkur}G53y2lja;T+uP|sx7x}OR(KKaNqaljoSO+c#JE_e#V2!Iw zcyZ@t-e8O_X%**3B%RLK1%e%=F+;hk^1!AP@rwREUH`?Y?O&mit68h(v9hH(n6CYY z(L^jlrn}lYimRkynFgJ0hxa?L+pgZl86sRprt3p-=%)w#i0$L)R-?TxUwUR4N_g1i znMaFQhC)RGQ_wwh*hn=<9TPe#vTc~w=%6R@*z)^bwga-qm)1c#bt$!LqH7|q@x)bG ze~h#Iz%`wDllNll`d!`WrXBN3e}WGV8AE!F-jkNtSd|k|kyOeB7beHdgag%+A;ML$ zQpVo<0aP}Eu<_kaKZJp5w@3*-3uRr&%IsO7J9E>F(?!?hpxCv$P3#F(&RL$-@|Tnx zw)o}+R^HJw_q0@vGB?QNQ!KbeEsZ?57=u9nP=t;{7_L?8B4tgnQQFaO_5%$X{c8zU zy(u!5dX<_2m|hVSc`Vt!kJYygz*l_}k5}}8M3=!ahg+3U)HD664*eQ4%rU{1zITv# zVZUX^Y`_FiDnq7;aoH=K6S*9DGnwn0eKCsvk!{Jrzd!q#Dgn(j=+^dV6T5)6!jre0 zMeCIk#x#~3M{L59{2!Z7I#^-Zf#(Q$!TaQ^+3EwR;hXA9vaXeXF|@xS4K}vc{NIhS z{u^lw8P#FiWBkpRSU&e8tWAK^QyDruFi?lhfDts2k3%&v(pWU72nC~?W%g_UDZ?VP z4@n@2xMK>{)6-)!6#NaCGa~&KWC`$H$@Dr$@|S6d(&rA|B%YkIzSSU z|D`1Hzu*+m(uK?8;BY_a?`+LhyY73)@l1ljSTkqR zin&Uxo&7P7tq+0wUrDWpoR?Lr$&g%KZ^EJJVeLyS%j2B{{LBk2^6eS0+?0_D2Yqg0 zScU`0)0Wuz=`V}B_D)W`r@lS$EYY1qn8BtKu5aTf%xseG3k7?t;XzoAIKHnM3F`go z6Wc{D2}=rxSH}aZEwHoDazgNf{^ss-96Ozza4(RCUk3xn%#XAQ8@$)XR=|vkj8`7| zG9ca8;XTJH3tRM>8l5{mn_Sc+O_7%R{ z#gY|dR`HwIg5axph_74Ya#{-aF?x$K-+Q5E1N5y)@}SNg%W~5Qt~LW+x+c@`2ndNv zmfFwi6rX>QymsCBWAS`a4;I{5tsu@R%ds7Y(q##gELULb9a7T31xBqTp_$~3quvjjkAbr$^-j0E z#x%IEGl{!62vJco#GD@g;#NdkvZ;o6>M5;2NR{WXZCj4H9*`AfVyIYeOoTgp3S#a* zmTdeD2!M2`=&wEm6)$n1eY0+ohY}6T?hdQ~_@`Mz54(X1)B_rP#lvdBSrwdDLZKTr zZ?RokNzX%0w$+ld(V9C?UFL>il;^ecDRG{L;ut<*fu2YoNiO0MaUgXCW)N6@0Bxp668#<00x8u759$CGMxwItnK)3%*~{j6v_Sr@lcn1xW?NLm;5 z!XO`#JZs7uK`57DEx`E`M@5!{aB+h3-tZ&bxKs@nOrgKWlA zz>maqKX=_3kkU<+cYoh!9-KQske9~m!b=d~VgWGtO+|f%>e#+G(#R*d**vKuW`3VO z$O2}0)ve_%;MUnd`o?`@B}pFdy|qL|4_UtXNL5c1-VW@t>OTCkKq}3Z3&@_3XJ3s$ zLv=p)w$8Ga8*c(GZYt@O-6o%P!@QLAJVRcofID>KULgGNlvN1rF3OI*U1-kvCf}K6 z(&|qtKs`5_XNkGx5{*NHt27(7k(%aKeVt((e9xhrn9!9S^<;IWmTWvvgpq zvGZ|fx|DWYC=GWo=8UAX7Oj^?h_>0!eOOCY!wWO8ZlkDSGn8gq?%Q@)9Xz{6n)yib zu>O7o?_%cCM^@wj&m6*$AyRAAD)grPxyg_lQqNkNY|$~#t~WtT&WEr^jBQhME+!Dv z2h>-yP6x_P7v_@Wdt$HhOUfTgeUa!y6Fc_x9V$vw_j@%XqbY~MQ#jyB<5f2W864a} zkA+{IgT!}+=oT}B6f0Krc&YO(fRFJjxv*W@ZTTLKMgJkn*~P5oEDj6l`zG>j_%3sA zg#9CZWg=u7<;ufXUrhG$HRwd+xrQ>w3#{hH6G&U_EOlQW@GqJJC6W+EvOF!Dvnp(( z>rcPPI*WG!dx}O?v0;DeivR`A<&;1B%214}WUHv6s-hj)GZ@FT37nU+;QZ_QxS*V! zfchbnP?kun00|J$aEZ3PdH3Rt4kf}t>zlG%oy~e}gQ=uRa84agb^ilxLIs8cZ8>#2 zMy8Xzo)jKESwoe^^1z3eG(bFZcA10F6*o*?HOU~_uxDtGXXneI_0@0~~P+pYy_~l$(m|fSZ zSfKTOvB0>7`(@JZ}OJd=nUuQM3!UYNg-3VW~L5k&7nNCrWL{|(u zH%x~eLMEBU{FdS^1a*~d)@t_T0&ccmbbAB6hW3dDw?Mw~gj%#h=Wv_RD;Ku* zCpVYh{17rsAN@3;Cft;Ls_>+?74sNJZLC&qR?*w@4ArfEE1qGGHh+!4mZg(-UG#bS ze63th;?^{m59=6BV?TS5{>{XfhNc9z& zc%mJq-Z*MYrW}htwT)q^(o2vIYhAk`rqu-J3#0J|rToISgp>5g?>)K%65WVR39h$V z%S(nZOQh|-rdy3_pzH~272S{XnI1m;obC?RtKSuF0M4`thvTT$`gD?+AXI7DTsKg2 za|d}iNG9=Amaq)cP@k`yB~}FznF=u%E-pU}D|)P4qWV7U$&=-|gSY0j3MvY=?G4r# zPg!aB6Z2br?+5cpk>fX@w1W z;N|?zN>CE6^zkH{Y%|=k3WWmi(R^XL92fB-w?KL{CNXFKriYz~;VRNHuKkAW4N~GR z`dcw099;cncP*u$mlKCNjSgvW)UQjE?Urdwi+h-QC)QhY%ZVB$`_SjkM1eN7w2SU7 z${UfUDt(dgp{LZ;COy=wu_>R~ZrYjYB_h zJ12!tVdtLj^y%f zYd8u8ug+tk=Ri0;HjkUXBs*%3EWd@8 zTku#a-6g?Vn)lCQ4p1Jxgv)~@59*Z*GihjkZU;PGTwX(7;bMM$DgEhJ0uKgme*a0U zdRhF6&&{u9MOox`phk_V{^i~dC;BM;eN(8HOJO0x+z^{I(yJa``xf1lU64*GM-aI* z^4E<_QuZ9bJEe^Lt}n zod`Yls72Ocb$uHy>PWz2c}Kg^H** zD&M+}Qfvj0vBsFsaoU_)fzP86)RX~I7-Rt)Vf3K%Nx`KFM}!~d8h(xy~+5 zQF!*nt7P~Vbe&DX|C0D&uzCy zdG~Fuv!ut4*gXmi2HyRKfrB_NLq{hudMEugD|4_FNNI$Fd}<7bik)Yr0ePRA)Y7si zMDI|p^<(H7>jBwnX!F#|Hw|xwc=%wMSB1V8+^FO@?rr>H>>*3BZ^~!N)cM^d+Yu(2 zkTct?tHvRUmvfZ46ib}ooE-v+K=F!p6?8kfLCafEY3Li;!?%4hS_e!EjPYyy?xZQX zBNob(TtsQE%=?xjT5z_9I>@UIy%0r3r(C1%Go16SjG;G_LRS2UUQx>Pk05 z$($TWuBI9c29{_8p^Vxk0%quCq{@a0Gi{Nb7c$ntx{r`UG|UwwQAC4e1MhMll4_{U zsX?Qb&bm&MMr#^rmd-I(grUN;`N*1;07XVyB}*vNt}dG1AP%iOGG@?g-`Bx&Q(8Dv z5Rx$d#_yFy<2A03#dq`!O!{))hYwBPr3kGguVf_B9VS^xL*bkv^bl5|#U&(lC+D(q zp%J$TG}~Z;-~L`_8?#Y7)9M9}KKl!j9(hw2U?|+oIog*Se9J0m{A~dv(`pNl!1RwR zA}m$OR*O2N&Nt}RUy;xD5@~qwQ}!c5;_s)!RTEc=+#sb+3VB9b%K*M3H<$N#x7Zg0 zmX5wivd1nR6(&D^m>b9=5nS-8)KctvPkhFW()j|>P-o=nMWnXBRo)uAv@tq6@ajDo zw;{IGNb7E?FU%sI`Rgf5vBq5U`(iAnF_^f59zFL@-PhY=E*u_Pg`}e2!$IC(dKsy` zgs-99)u!Xgj2zI(UiW`;_d`}PF*t4u{{BJ&h@C^sV?)=*_y!wJLH zStSZ+K)J!0YIv-vYUu8Z z81|Ky3EWUm1q2^`7@;l9)ZKDxTOCB#!ME-u5s=ukNP zwHg~fOdUzK+C(QZc4fQU!90TN+|UjKslB!m%(Zfn2eur~eIHj!U3Q9i?VP6<3`_7VykjL`5NM@pV(f%2|8BjJUpAKeV z*%mpb-K7=%prqhm?uW3Sj^6LTB&FBcj5hvw-@jRnII%RaN7&4PL&O#yzuNiL+{z)| zOAsX`-}%d5jn^~Fm!SX`FvilJFa!8xot~lZHlC-6b(2lG*JwvBAt-5Et%#K7p zEww~@jS5EORPB+)lv^Nf2;aj<$>@~F&Z*8Z*8RcTQu#>{ty0pL57JM&A~9aUuu`e$ zIx2n z`sBQL0k?Pieg|I?VX;lg@z$z#5T<9VvkC76qyDU}xU#N9(jOe1DRRbHA&`{J%c%45 zs6LAY4(J%sTq}KfOdTd~fcSPXE6ic(bN2${g&~|U)Umi#CS_XZgOjv3akgm53rFlr z6vZyoH9bdYsbZ)wlVRL)g<3GN!Z*A{W!6=-7SCbjW<$Rp_xg~CrAM7k>Td&Crr>@EaAdFJd@C*Tw73TuhEfKf2PcEon=>OiEN5|Cw-~=h@D& zNKB09;=FKQK;9KE2STcJL!VP?bwB}KBVhI@8;LtH6pFw~yek~u+T11Y&)Diyt}8Pr)m z$jb72u+sR94C`U3f`OB~0M7ipu&V+YW8s@MoPNB zZ`!X^<%?uZFdykHJ^)OgVYA1g0t10W1KRVW*S5XPHKSjotG`GC+4jxpcivj3{uIWM1G+s3L zj|x^V{@*HD2D$$qrKj+VmPqT?2pFyM_0gqw<`Jns8b!JfdO=X`W%#Z-u{OBpy6AV@IE-vA}+yzkFC`7 zusUNq8COthtQu>8tvG?w&!qWYH4uCsqaT6y4MwvCPX&Tz*IV_`lH4ilKvi1)+!Sm< zEeTsGm~QfM-Ua?wTqN1y!)vlTTOR? zc=kG$R3)K-t~Z!8OZAUPMTa5)Oy{}|@b31MYo#3IYl$hAsNY(q0to??etIhqoDLGw znI;gHn z;8Zv_*nu87pmKg72)ym4143pST-Shy*~$3b-5-BsAG?q6J^+LtxY2OCdPKeKJ#M2; zBXQs{@0}5~LqMsHBE(5@%U-_QzspECFm5z42s}R&?w$eNt+;)Thbp!JQoI~!cM7zp z^$kDe&4CP(qnM$7;E4E}k*-t;Rxv&EqI4PKGavNpI<{oNNES4%BXJ!&geJ~l5qih#%WDEpflAx}c64na{! z2bSf)eFHS@4uZPIm~^1p4-FZ_Pf#rxiVr*j1p}kwRnS(XZWQqeD-#CQsQmcrkA=n! zuI(HH7c}+ut9oIC1f>M;LBT`M$?x5dj*Ejp1w3BcrEXKo9n<`=$}D@v3`me=d4y(? zu*H7YMwcgeb0Z3x4Tla+7@qOKJKB%+_*GnK$NJTN_uhoDDxQ>&rv(R-JYi z#D)Oj4<`^^7kQHDchv1U9! zGlDe;<2Q2S`=H^K1rnuTkaW5R!U^VA% zsruj|i5JtvC<8%j#V4TCc7o-4k7<;D3L7ao^#Mh!()~*bz~k{ielcD5Aza`w_a@Vf zvQ`?u##n~$mDw+kf|X@reJM7tHfYJLff|+id`-^Q^!GuD`5d@_;`$d&8_bVD|IO)F z-1ToC!qX{@31$&CGcFZcCd4U@yQ3G12i0a1hFua|DJNdJo#dV3u?m&CRH!i}*6xH5 zyansCm0|s`(T@O~GZnw$lBJfSEujmYu7BJ;RbkwU@$X zWSkC{V7c*o(|Qv84A=*ZiDL)M(cnN57;=?H+P6sVk|5~2!bj{nu}v_{W+~|CjxFVu zu_1`1%}v*4mqeVDhRgjFwb=5D6W{3P`^ADqX~Z`jx7Z6%eaypOb5=c!gO%mbCV8YV zcH0|p!G7!r$e%m{R!6}1KzCCxZ0LO|8tWZU5y~YrW1?=o1I>49IpnGzRKiK7J7((7 zoQd6M#>?D?6urO>gY)KR1)s$+$pBIHXCUnMPzyD>o#$ZPNq0;$=HJ3@25RQ+3mPmZ z1U`ITSMWtLo4BKY$z0>9It{b@rYJ36zRnu(_36(Pa#(V?Q91W4*2c5ENHQdoHMe9U z`BEa&-*FQ-gpW18l7w05Vj4L-NAY3)4X9pKLiso$Ei2ip_fXHsNl&vm5p9K>0#kyDfu$#wma*}!I7 zCV&=4O<#gjfNw=u`l-03_<>y$mn8-B-X%FmbeYAo^ckT^Yo>{s7q=_3PRZpTzqPcy z$Mo&=A@r)Hn5fUv$I5%JuG2;TsW$*o%g>JXK&niC99)lBI9Di=lCu#E(SR5)tmtmB z!eSdWsL>RX!aEtP@G!cxE^j(a@9Y;)Pse;Mgv+HQqJ^*1osXS;LGl*3Dzo?X`_QLq zWk>l^cxL*kx?e(}V=b+8m)SWy2YS#GbZD!FNO-AEq&w&El;A9fD=3^0w#B5PDe05% zXv3n|x#S~+X074ZYco;Z!o1QPJ3Gt$<24-Hf|vRx)6gmDsJkcq_BNun_G~;v%lYdi zDqwZFsQ8Z2=L+nx*R%Ambpu}_C z=dSt*K{-yqkd2PD0%3Uu;NkDg$^fZ`xLeVXkG$;>VdZ-|Fn2;y_~t$=Wq0iwR=x3i zhcl=%m)h#62%gp5qOa*kbF+GsY$`|lHxEQNQu{Lq*QP`~RE4g(xo;(*weeMx@}c(d=8ZUpG`9^Wj~$?-2Rejd0nJP>ZP1M zJ8%rhGNyuwed1}6nZ=Fk6QWjRY(Da3))jXL*)za<@^~d?`qWOXX0JOQ;=h9{L2Fq> zqz>`xzuwwS)Rf)_v?jf<|GH5x2yxA=1C z+&s4}_jVAwWz&?Moi9z6+X@%+87o=FwYevpYewFAQ30yoC?SsUoG_%|3bNpJ$&h|?~%Ju<$XYrpJv+ku&TBor9doG&}c zt?75%?&)${x1luHo|3#MBQ$ft?58J^A6Q(@doshtx_{HUn4Lzl!hO8A zdjf$+R48>fmUA50&HmzqN+7>sYlwh;*V4#z_ioj?(t+o z1IMw)u}b=EeYYoO^ly|`{Pw&_{g8y9Ps1E_t7qb$56OS(D88>QBkV2@Tz+>k$*`2K z^i9$$yKC=P$HJz~lO4hPDeo%CRoYC>Tq!a0REGRAl?Rzi11)bUnJ-Vd=Pvwh;?K63 zVLJXb$1P=_2S$UMUq&b!L?1gTG}pxJF3X*@uEE`XX8wk3kvC4nZ_pf4T_KXU$|_9c zm$A6;FA7-VEz#h(6QpFW^6AVOpX!OgNc0Gl;8+3O0IKq7UF_~zFJERt7s_Q_y0^D_ zHgGGVIIz#VH~;=V=ToVmJ*bmUh8VrQvop9eMiacPw4&_IjfwwEA?xKnsvR6rnrD4_Hr5LyH@ARSQw=^!AzgbqmrqNsF1Is~LEy`upX zqy|Ej-XuWiE!0r*nZzA@hYXAgE+d##x@uQ{)4&Ba?i9W}Z?um4F! zMMd{e{r)2=sy}k6sLmY&E>f;cMiW0%QN5&kcwhOkpZQwD+r%d$aQjW<$3SmHx>w#t zj^uc`4Zn`H8c6Qw)<#a8&YI3d`K^f*{+q$n)_o{Do?sO9KinEU@ZR(`N_iN5*I zVJaZvCjeg=D)c(}@z!g8seH)&P+e}w-RQ~h1NpGQGHbWfAm5V_KC7oX?mL_L>G!en z3YZ%Wl24%LQ1EKQJ^uaDUg8Atx`yNv8UwF*-;ci^|MZR^9M2bHrxde8|DT7Sv=Fap z$g4d2{q_?le{&Uw#TXHm$?M6!q`qIgg}*vtPhd?%z0befLX;>H^hooC2KJ}9uW9J- z&d2_g_gUA$8kXn~Swtx3VjPlLuM&glIg0V@wI4$*(Xi8_pCqI0Qi(3bh|DM#4BM zzdHKudRr^O+&4hBGwgCuX$ZM=swBvI7;_w-~FwY-cuS~ ztW9k%M>7h8YHMQ(RB!jExD|(FeIbCN+6e zfl6|`Qr4#oknz-OqGk!lc^G)Z^CpTdyV?tK<;t9=NRbXd1D&3kGB~VeG;k4Wh>|3l zW!mxXmJbPCZ;4JLVS(4NhgTS^=M*AOw}_!#-qp{?c7mLrjU$cL6&yAcn&%q3u(ezh_Kg|Y4A|9IJ$r7u54JP(MDnXwhvcTW4-yz^7LHYh z3hQ+ASGtt&X;CA1*}j=$dg(JWB|d$^$AulIdh4GnpV~G5nQpyPus?H;JXln4rg;IrJ>IO5j@)vCXjvQ$mgbg{7i0!_;h-LDSC+0vYNtV79`djq zsbg{0H8Nmfv75fTMifnBnv9>g{}OH%#y%L0+!-__IIa@MO*SL66*n7Yrr^Es{Q8=K zXr?Yk0Y&iavKLXOrA^aj}DNV@+F7%fFL=M%=`_hu3z!ENyt>-WCk=UfEV&2vj_Fg&;n+*fnLO!?Y9QN zLF=i>jFj>bF54;$9QR9E&Ssuay`WLRD^k@E4+k%0(N#gO+-CvDf7~@`I+b5$ATMe9 zKZ{}I)b0!FuWgJo2W@Kv*lof5k90-OjOoPyhMf~iZ#9hVy z%}$O>EC;R9)M7KFecwUf#$y?Wp(Sy!7OC2T+{1C0Frqg8pl-ia6n^V;^?*IHUtsN^y6)?v$y=GKfZTQV)mY=4ky_hv#ya z>D$%YJh@p{?;jh1R;|F8cizo&_02P=0@M#d!=}Vq;ZiPmM@4=HC}|>o)yNikqUcSE z@LIMt%t7cXeQ25BX))J=H0|!}c75h0dJqm7|LiSIn^22J%i2JOqAerJ!o^&gb2=S{ z*lj$Km%FY_JazUhztk*)+oPA6j8{cQT|IXluCP3Hd-*lH{9>kf3BOPR`zaL602$!t znsF*=P=cJiDNDtl?Fg3Ifzc$Te$mri`@}`D1fLzTSHxPs61q}TlL%1$bcACAIk=66 zo%%I1pAOE-Hqqp3I#Z)T(=XAc_aa}{y1b6A8ZHn;NEv-;Vlt$6mLHn@UtG@is7JW1 zc){p7feRL;rn(%VXS(zq*)NvEeDk$54b9*#*X7o7gS7Q#5?@EFYbJ>3(p2PJ1D0G(#A`P>|ZA4Jsz>Un`!SU4{bVnY92;>x$nI)1a=uc z3}-rS#cOhoMG;F`HDr862AZf_(k4)On>y(U$8T(O&9JAMeZAsmW4)r9r|9c32kmkZ zQj5w|gn?AIrnuvzKmdbDo$DAs1n7-j-CgofGlzny=Y6N(x`W$`XPy)5kqoY~GleZ} zJ)4AeSsFkJa0i^gptQT?zl>NE4btTXXT&7W#{*dVT1*yLD9w&#~yu06fMd8T9R}Fc>Uys4_>n>2C zwtTyWmX{yr;tuCu1}`5V$kCog50BQE8Mx#VCVn2gJzLg9?&SMFw*bdjDu%3+_vM=s z7Y)ZfDk_wI0eP#)E2%6`=mu^d;4%*8^#V@YdY13!UXDL5ogvrm>XMHwh;prZsLY6$ zo9r_uPwB}d$etH{u3@S1#Fb1j&q`KvrU>Tu=l6InWxFu#FqSY*GqQ3jdfYh1)omYD zKlwZ|zXwbQd;#$BKE5@{JImvD<4VBUtE_r7@a+<97eC`oRK&eV5~H7lT!I|qnZED7 zs^5?88#&Lao;$dzRiT+mv4VsP6U^qg^d|1ju*~<|2@4S~qxQxUBwo;D$?}-&DtZ%w z&ndH(5@mSb0pn&6_NMeh4$Q~z^ipV;*P`7MiG0x#W1`3QVDANf zf(U`iC7-iMm7DZ<;rb_^lJYd0o)=>Pg0cPgkFDT70_2i_d=%5^k24Ea38ApPZTHX} z*%Im)>f+t2aHXTELNGUf+2UQO%OH)1QiUi~iut2w#`blW27F0Iy^F`>jijCtlLdYk zd;r3J?NG{Te=?8EeGjxKw&T-PwG39zMdKUqzr=_+RAVs*+*;q;BIQ*)lGYfQrL7*=ykp*8)LA)7bj% zW8|arq62vu80(}zrK3HXo_}Ir!GfG=x67eau zHT0>(J0gUmEUb~t=vm^z%T z9Wm5+a7MGEJ`Au1WB}d*cz~WsO1$SqhIyXzsK0Zk1&XrKbyVEeZ2Ew6bR;6V za3Qw*YS8e@70g=V&2ipI zCi2bQ)6q%^YC4y_nC>&HR|sdx4qjxmWKyYM7;h7Jc)p7DmWHhHC7(b_eAeatCe9mU z_viU$$vl5SRPEc_7X$~t=x5BiaT+{NLZqrdxQfdd*`V1 zVhKyL-OX==vj=+Jif@zzL58?je(FqcUDoc#`eFa2<_oU@xokhRQp2-H#-d1iEW`Mw zn|^vvK)-a@llOD|xPB=Ey&;~ve)K8e>0QnRh2u%uqk!3@GKTp&j|Q@EDdwns4X#4Lw??^Q&O zUpN=Ve!zRd;qUnKlNJwtHL$?=8BF0q3xb9O(Yw#-mtX5H9^9hh?lvy->vZ@f8Qda| z=U)_sm>L5SxbrHIbZ}V%63k z)h>=pt>q@BsyKAnJ-=nu6_X{p=ncpMD_ejn;-!-=?j^YVM~%PF1!319_lx;7m6W!Vix@18a$ zLw+{X(7vAkY9K=W7AwCWuTaB$KBN~0*gV_HHZ{xxaOYT?ou8dk*J{=|U%aGAN}F}) zPIV(FJr4j5=a&R3kz|oPJXS2cuZ(d311RdU9a8|)Edvj!6x@2rOB@{(NFoF^NK~$08Vl;sC#zl+FxdZn|+{TkCb@s@PM*^j2Tu% z?pKOkV&WTIP=Uy*OI*iett5j)TD{$K$IcUD2$vZ@OnVu@vk&~ zs%ZT_>iRU%U0$NCq+_yoo;KRpn{)gMk6iof5!t7^TqCAwv|9Em?BL$|X@M{a|HxG( z91RX;ob1=MmchC$-r$nktC9&F@Zk@*qvM{5KFEMLj62DF%_O%?nmnQ%gdoY}r|?r62u4=; zJl&1hE_Aq^R4tYA*l;d``SKABtb(l^;0|`Lh=g9X_duNMoWF@4Bb&)DO z`P0kWiss~-0`K;K8mx+)lYOWslu#=+Z+O;3ey@g-W^9rCo!1db-d%(3foAd4AEJbBvI<8P8W1@hut7B*c7Wd-E98f^v;` z8tHxb&`87mK}TC7>s8GHRiwei@4_(i$_UcVGba^?W*jT0&MDw6tMbDJ-XAx=-HN$s znR5bgPL@DfruAbmP%@*yAX=ZM`T6;K2!711*^d@S@?mz8c8fKYqR=1-q!bTZdrBj> z6t9@g{aeSGB@uJEr~OaI+Zur`c-Bf0UOS))@R{4Pr&ES~+6Zu@w4qTogOqJ_(c%Ux zuP=V|F>P+4&91R{((@e2)2=I*so!bUe|s#A`bO=UIx`c{w%05NzFD+;!L}8wDoc7X z(+(qhl(b5*qjEgV`>x?Qn;$ppH7lQANuo`00VC4#wV0WZFd+xcM%S9n4&K;h80j6& zWtqt#@n}YVOCfWcAP1h+t=SM*{4!U8C>HPvTnmIP3r2Ar2Ye*QX%oy%$q}Rs!}?utew9pN^EqiiA@xciMSPXGLdDqSS;D$G@JORv=Q;yS;WZ3bSpv7;fF zG>Ggr?y2t;u)v9fcEm*QGRVwfNvvbYg%2b1#CNu)?Yqn)X4G{Yyj{0U^Gr*wLfv7Y zALa7_c)4F^_X5WJ}8PO0bl8vb?G%jrF(u-r-M*x6u~RhKTbcBdqgvI6~<; zUAt`EMfw=h!n=)ea%qBEuLj)fJg&%%aA!~2km$|vM(%5cqJv(2o0ArES@Z8VZ}*$$SUji3wTuI{aF_>G6-k8sL**AkYI?qU289 zytx!Ll^4yg6(G~9zRdAdltN2ZUWFr>l!&AWJm2HtrkyGWMaMByfu4O7Kcs%%ep$C^ zpP+3Z5np?Y`}s5>Y3`$`(j%G1qAfIe;aK||$ArYDphc9rEr&jFuS^2SKcitYV_J}Xbq+1g!HxyRWK6DAA6Vp+!0N6+(e$X%KhnkmO z)nL}SoX~dV59gM|E~dj)j{ZtQHK?eOrId;?O@Gufo_dbn8TC7Te*nlBimVZ->o1c+nPw8M58S_~)kwq9UupM@B_d z`bK5ets$uA>R*=+=E@fp1Tx};1j7d6^0XXM#&6)xQFy`qML-@GpaV$fFQ7V#yF%}O zWq+~_T!T-WftTX0nZ@{3kbuY^QfdqeM6H=gzgifxYpsuU9IhW%#m#8N~C?e*U%^RH84TX41l;kRf2HdQhX)vnyZ3Tb3 z+k96;o)wIH$1rP9csfAKZ#D>c#St*KR)-*io@QCpH9~9FWu(YXKiT%GCnKs zLH=uR^U8|FcNJhcUi3R4e<|;^N#S);*#bB>{3&fN@G4*cAbqy`8JVH>P{LmP_>HG= zG;+*dzBy?}U>R`F2#xmL!^p$ed(S@JR>;O6bm`Tb$TY~wD;$ZnM@PAaIZdZuHy4@; znI{;V#X{toqvxbsr9Y@TBoHN4SXB6+ni3LX3Cxp>XJXIZ!^O2PWd8w7T&7T7SzY(m z7rNtGvJX105!WRQC9*A+BcR7>kPbSNLk+TLR6&01`a~EjR zuW3k*xI}10-HX*Lsm}7+Wx*D(D}$78p58M7%&>mvI90aL^Ic(|cqBboza^jLPhSoJ zVs3}_MR0%Mjuk`qXZ13$r6fMk(;DqvWzt~&lNr-i;4-ec_28(ecgYZ{Tp7ItSy2{% zj&_Pi+-9++Wbuq;mus%}j!^<2Z!WO^h@F@?VCD~NP$0M!TFDOUjUd#!(#QjslTqp#WqB^M~Xrti1Eab*e^xB9?zRv1KcrV{1g=A zS!!uGo=1e+By93>$}jZSU~oe$s5kv+Oq;igaRk(U#(!fCOiU>e9%kEKaRSFLp4bdY zv`4N6H$LYwr4f#dUmSH8F6UAiV})~Av~`4XK+OlE0J+AvJ!Mv1BX$Ov7^hDTIbOLz zQM`lS8A}j#_HLk`7kP^~OxxeQ%wqSVokAk4snHD$*dz=}Rw|Cgv zw9oD^E7`la(EXG2m3+!Duvypx0}%uBGCE$dWVa;7IL{-Q6Aq~-!&cYdU`4)4 z&?sY|_wHvh!z4v5%I}c1#n_asC_g(tuNiPUDO?)rb2&kx>&sICZTGgvmpCMweMs*d zWC62)V%f#2@L744r!}{zZBuB*v0fY~vm)IF8y$!9(xjM1JE>c|s)by(VhdD6_CdSz z5+TaXve3v!YBg3>qQ9iq33YljqGOm&5er_bHi9^28D2j#~6}o$F51(~zJ?dV@s_4r1f@kNSlcHAn&Oac% zvT2imavC&QiPH7jwtL-Ni-W^JOTDB@Q*1KS_kBNM!vYiOSOWeQ<9ULNTC8LM{weWE$y)+uiD3_+i1 zJv4_rjBa|5z{`-GNL_|xA>H2K2uF}S!rC@H4xDqNimyyK`hA``r13DnzI#sKMf2bc zX?^A)3;g&iWo_h8b6K8@cOwFxBrLcfJRjPoLS$XK=$fxIuWbe$xD8NN2O(PIol|mL zsOq?O%x*T9MOvt1NCKeEVKBx@bYTQ&9mOG^&~mdm^)N)Ni$CJV?6MzG_{d@mxRP|n zotNFQ+MSkuK|_V*yrarJ=tH$rS*VtpN9z+qLRs7nAjleI(skrF6MQ1Nk3vMq-c+|k zN%AnS^BwZ&MZRQCrVmE;N9$3DatUBeAPkAl$598N9&*$sAu?GtBuOb>i0)MWdiSVw z%z(5&jWN;oM%IW*NV7Ur(16dy{z$a})VXRPy2#1ryg<7JsD8n+&h2MIt_rn`sHqTU zkUJbBW(20HR3Cm+g}yMjIKeD^dF}!Eat_m@3gKfL&hr|1xueJekH`E3YGwY_$;o<6 z`agg=4ytL1RpFuq5lkyQL_e#@LV9cCO6I}>X ziC2}?IR@w~hY6O{$w9mX5dA#La@m&PJO%Dd;RuOn@42KK1X*PvczfefDq=Rx+Dp)7 zX&~2RL$GU}V*z8wIgYD;vZlM3o+mnYfN+J;b{;X7Ytb`8bzTW z@)jxnvzGW>(M_Gygaft-gKOFE@Q8=-@b(m+0lV`7C8e?#Ir1l)f`HFQ+VDR(x;@C9 zs}T+e&aSsi-54FRZ)3HSIohY4CH2uYSTJm+Yb1csCcRz8*Tr+sF2fC{IQ{@^nk}wK zzy8KSeaXJdifsg`(5H;c0#xG7J1)?daCObn$`m?vfA0}McH!YK0|jkGOsj6{vKS~F z{NpcsC;mEFQd_p>3o5B`-z?Wjk>I4W{#%K8=5SJ;Z=0?bY*AR-EKD`fW(FugVu9+H1@&9%!MJ=atkt8A7Z?C)>^0aNsDb5?MKWJzo^DJ0ynWqP zQm)d1e9r%e<97HQqP*uM&&3Em!)B$mB{BCi*5R1n;K2{+&m%?H>cV=mj*gSKlX`=AN z^Dg#Y9zn6}wq;~ntl^@6g<-LI+<~gaH8u8j|8EyA)$j*JfV)$N9bRoM^J>np9_x}Oi?K$2dC-Dq8Zwx)LaJ8;!gur9^V zqaMa%5^6{cfPEc^g<0+)9FSE))FHVHu(lwi(tXEI3tCm7f{R)dTXzvp{my zJ%i~)@>!yiXo+h2XcWFz?!?@cxzuCkhLN3=7wnYX^U_k&oLqd%>SQ8oj}GM8CY%1u zoLm}=h9MoHYspz+w$kO_F1J0)J*-TMK4utsq2PXJBaA3E!+giKER7gkmgJ;K5cJYDVaN0b7ofikL~(bgS^QWHF~3V|jtuDovyii`KH*Ha@J?t zzIS=O-gu|b@)T(L{}6!PAztSZNLV#-VPE-yFlrh<`{WY3AidztAQLq$>}J8iX)K(1 zr^KKHW+ar2VaS=>LBxo~$A)Q<^_^*s3u{WP!E5<-=0busJf#M#4MxQv=U#v#)F|P= z*yYrdbF6)=s-aG&9M_5v;R@T?@l-4~igv`>q|a5%ojJLfxTNkhV)sOz6|ON4C(WeCrDb!$hHavHC~wYFNL19M z7_{K|Ntj%<8XtY@+Nw!$+D<+D>CmK{(Z;5yS($0#oM)8%T6FCEN&fe<3&J!f|I&Qd z4442No&Av;4h29Z$R~$-_l#G)Sl;RCZ37$oH$3aebTNry53mx<0Px-9!s6?$Ld zKoX{w*dp}g=i>bIcPTEdP5t#>|CWVOGEq%|p2-0krPhK9h34%zCa0drqQQOM_EKSm z$1hy3cr$t>Pe?5bB^N|DU2SNAF%I z0Xlx2G%XqkewqD0HDuK;0n@4uvfe9?!&%)x4~#|2#gdt@@8v|Ci;8Q(q7K z=iW0Jy4-(on{ujly{_?JHk*=)fBO68&o2%F#s7KHd+`782ZjB2m-@ZtDAk20hue|~ zg)8-x?*}C8*j;hcXq~~Xrc*+Qe1cAfLNYQTvHFi}1jZpTvGDH>*3jT3g&~l$L%ft# zGfE11c9PdShgf`~2%(Ix=FzmDAM>-XJ}v>P#kB*(UI_a2CARl%QJks6MaFT7W(6|* zUDM?@YAIlt9cea{Vcb+q8(!lwk+50|Bj9L<8UC>Xi-yt!NYt2_%)1uGA+E&qY zO7E|Y#a+LC9cMIrE#kI;$o-m+E}E(iKcC8ZE%lwCb{mg3G-+8xv?glZdb2}Wgp6W9 zL0jK3&WBsSj@pbH{WE&v)Q1I2H3byirZJdkhrztNQ<>P+Cz8X0BZbaRBcGo_RwOAs z;OHCVIx%xm{CfiZtPw!l7?zr+%lU4k$@#1$wY)%FTc~Dt!-(%~OdW^NX?%^5*juWw z#SSPQOvX{nVwhN{-rII4x?K+=em=?kHJN{rC=IRhm{n$f*Gbk zls9mkL>#Xl(K!z0X=}^)xE7o=BFSe8#eeAD!0Mjd_z}f;(82)eUfmhG+jaZ(fTUe_ z2O0Wx}8MmrQ~<+ zKCJfnXN9=t$yj6kSRu-30(6Sp2PG;qzvrGYYA z?#?fEq%sBz!uViuLZvO*J``U#nhEl&xGrOS-f_Cm@6XQEF;xb|cyr|G9tRby#a7<6 zt|4Hr%k5IH;(_TGe3X1ew8D=1VB+oj0`L|(nH-sEKi=qZ$`AZCbvRYZVZZ_QtbDU4 zRVH6K^y0#v>(})fOtf*OUHa2nH7>YRBo{=9ysF=up=b~dQruB@U+m<&7yT`*A~_=z zUfwIeiIG&Kcv$^LKfQ`W25Y5DIj4eho=a59-nf^NF^nWG(5731mdq_B6~$+5!`K8tRyE7X%(?aH$~CqqT#W?IpC z?o=8140_L&iz*H(7K;^>)jI+J3H)pkxL%$2GKpPoFE_$I!=Ep)a|W}`Skw}@*^r3d zxrWV-if=~%%5H~C__x?{3E^o$W1%hgMPKEIk`@5?|i~iH&gI+X|joNp_%s2OVvk9WOKWgLe@2m@Anvmoc zIa^Iz3r0l5q);9I3gg0ANrf&7$aqqNsN+`)&|DPwH;G~2Nf0*K0n82K*sk)1v)$E> z!D@dk6>=&A<1P`Fk^;ezv;_ZZ>oAxK9V?ey|JWvgT#H)f&=TpItAW&;q zi@lD(N;icU@xMQ3TM2Sw9kN*;uRiJtmKnUInJ`~rxjolz6! z)p4GaUk_iF_mi|%w4AD)4iGGD_nguCJ3v`lry-213>3P{+Si7( z81zv(ZaUe)g2n7e3<;9gQ_eEUOQnY9{tH#>V+`PnR#L%1;P|{vVD8b4hah?}_vqMx zh##)n7qiz5DG+&Wp0Bg%1LNe*^XODtn9PqPy9;bryR}m&6DQm6Qax@eJeXA{YZGQS8WD#bJ8aJYYF@aGLcfJwGg7Ii{znr)-ly!|w)JhgI9X=LDy4 zl&huk0%|~PQ&!hee8IB!9QOkH1hO||P!B?_tR>!7`H0x&(@amYMl}J`v*}xXgv`E3 zHI);G29GlTHCLCq`FVY)CbNd*cw$U^Z+XzhbeWav*`}3>gOVF@%7B7qF4DZSFuBzo zwzQp?-r-BvvWphaXpdFK9|>s7vwmy4SIJ@#Bwm{=cm9|Bcd5Na$Twh#4cJypEUJG9 z)p4mVeDWhkF}sL{0W|3>Ku3L0gxwYhTO=I~P>}f#-W=OT+cRNX3T8Z+k@bu0{~iJ2 zqh5RqGO!eT^+5dI;%jW`H8+*XrFtt#-OE|Rx-9tDSl#JaflzP`PvcUIOy^u=@(dfe zhka`6R|}Dy)=i9hSs`*BXw%`e7Op>zNoLe&Pt`7O!1Q0%FpzfR8wy=ke@C{yZ*l`= z-@tlXTQM6U#7mX$+HKoeu*pi_4dJeS2U#wOBX=3tct^*`=j5SA^l06LWygl4e9@l=MI}5^gGAAv@>B(KJ zj|Ioq5+K!V-`bJ3wO2F@ASwg|Wf+@h9Nj<|Y0vo%ub)p0 z7h-#<>o9LUY(&Bax=ZhWJEbHS6aYgFw2CF7dtMD_5%e$l8p}YRFl$u0OItMyWlt-mzc!v}r0_-y|j@%lq021z%G z(2heb?Td`OX{zQeJgTqfi!x22{nla>N%-QH@x=k7OSwl83_%%c39Y`4u@|zEp9AS(Xs z#7Us|nW}5Z^ro^JI=DafIhey3AU{G5N;Bm`!8rS(e~CO+@3myF|{)G|wZ2UWEa) zcgM3!xj^iZuG`8hYTP`_rMm1Mt)CCsljT!l4wLOOIwvIUrQqM}$!1@D7BHx9jczlI zV3RlC!p|ud>ULnh-yba+6i4pZ7JlX-IOC%MvJQu(hGO4go%5GirOBjJbx6olyC9;*fKu^}W+q>w_)hb$_aLN|5%C zq)H8Tr~cWG*tr<--4!)-$&sBBR}5z2Ywa`5ciAAoJ>#}3g+pQA&#MEIdBgM61ye-9%bt_*~m4o?rtk1A5j!)J>MSHIs?& zi8#unxx^ZyYJC>v!nW5<3My1Vb!c%lH+>UBGrL8^cQUJo%=(4CS+9RfbdyA3i`u2w zEA4t59cis3bRmVhsE+N6tta}Z$rH{BhLx6+@$Ebg3x$^vYk8cl_4!Ag>`W6(pW0Xd zNiNov&p%OWNwjb)g5#JKtT>N=4&Rghjx>>6iY$BxOgwsYKkovMn}8kd&l-j? z25gM&iCMS7T-LonI&!r)Wr!KCCK>C2#JnF@K-6{H&yM&^OwqOq#`CXD%oDv?3r!ox z+E$t7uaWaEO<&3Gb|m8-P5WYael#8#3qj_+YXlS7j|0+N zy3=%q6+mwSKeEk2#d?ssl9tirlEtR)>->uB6sLl`+P=Bz>{@4E>FVUX&Tiv0;#5rd zs_+jFb>V8eCzU5{bv;@!aQ`tUUGXytclqbulP4}8#Pxb_OYfTge@R@Q{2yY<2&<72 zvxmRO`lK3!_%&K?^S`g~y#3D}y&<5g!*30OsgV8kz;D^%#v_?axetDC@5^yz)d>In zqWYBB|BGPJqWA3gd-qpfXI1@v@68b1|NY>{S3Gf-~RKJcq0 z%s^E&^dDILRKwHh+~L~cp0jQvApGcptKDBB97URY^p8{ilon!(`tCK+uJ|b3I^Ct;D&7Po0d?W}>r4OLAYJZ+62<@Tiv3pfTqs&tEu4Y^j+Xx> zj-SR6Qt__G6f78;HC1(}5uNDkb8idS+cllIlARElwRsx7TqAA2^P(bQYR)n%K; zU@}p&I9yl_*Tmw&gg?woN|-}D=hZ7u$qv!luw%heLqu`zh)E@8xY0>i^x1lfZ0h>o z*7~XUMbVOC#?VT-_Ab|n8sP@=*>OHa?7?|$&3=c891r@yo%-UUmhD2Hh^`7n!p-$J>sS{-uJ#>qXKnP~3Lav5 zp)$V8<*%Qi@6nsbnkqSeaYFCSX~ijuo~1qQGvQK58NS2R7jMGjvoQ_0&)R70VlR5; zisy$BhhmfkrN=sMjYoDAO>LA7yS>z(?Xb}GVVN?19p7nx`!5u+XOI#lh&5|pDT3H? zD~p9zdo&lsW&#jGIioik4yG}up$tk%cD)%>D*?>2=@d1)*r@zQV!16gVRB~3mm-QD z-BsCs*8b8H8VyPh*jxGFvp)Wz7D*8#4Jb@+@bqA2X8tcB3gkZ@%Te&C{V$bd%6A67 z{Fg*mLGPVq0dTgUuwT}!U6yD5Y z>K77B1_Vnno^9p%ZuvUtDXaJUCI-s$6vh>tN6RLo*WA9nJm)}M?0ItRF_81{h$Dx- zKbkVvdfGp@fVqsq_H3G}ZjoZ;JkB0)Fj;#i>L-A@*dr zF_~9JQzR?`;+HAAoY#x#1lW8hKU#}j+PT9eTX8L1u3a2M$Os&u(AQo8lmD6Ndx_dg zz7Hqb{sM)c=W~WN`tFY&EM-S8#%58!S6_naP37yN75;o5Q^>If-MZ4m>used6y{nS zODZyNkSl&x{i(IA6hpW7<`jx#WM!6ZJR_|x4^g(S3*{qHrCqwtDV!7dHG>sxN~0*a zF^d%Gc;r@E5JhH5oywT4vuTe4V{Tr$iMrJ-^t!fH>AUq;z^Q$AQmpEO8o}2Cs2at9 z-Hz`wZ<+abocd=T_`8f%*!l;C=&C$0$IC`h1mv)v>ax1BZ&8IsYBB2S>K(Ekb3gu4 z)fFx$tON0XaW>UHfCuneI0Nkq-d~ag4FWEmSc7SkE7IhC*n}bC+wgr0_=GwUVdkIq>1`pe_Ng zzNH8!?V57o1WP{8)uc)89u(J3r?i|d@kgz~dKD9|QDkkd%U%?n9PNwRh*vWMzZQx{ z%q!ELsw+CIcEs|+Ag~MrC4%BH!Rpc;4<^596k;3SE4rcr=y~HbuIM^hcOt_h?Semf zb1z-ucIvu0eD7z_^bhg*bM$)=6%NR4RzynR-(sI*;E>*z|A`o(M9SWQAUFrkl*v5y`g719i z^3J5A4xtpSdZH$U_^Ir!I8v{mDF5VvI=Md)RgT)BG`B+?LS2e-#ONG3W51yKq9JE~ zXQ?qUh#Q%~?wodQ4k3?r7PQwF`ByL3 zNt{ohp+$v=4}5?#b3kJ>mb%_N?cy%GSrygCZWKn@a#Hr84AGq=UT=19$_=yK#_{5j zlJ{(oCH39xSYtG)5`v>^)@>dXNS%$xf0ssmPV?oU?0K@{Z~t>N>WioI*fj^uw7{zL z*$)Ul?pvl&g@y(4S)%R1a_&Kt{vnEZvx(cE#}7O zMq#Fhsb_dhF8MaQY%j2LS#AHJB0lMn(yD%`tnV@e^sKLRQnC(*Tg>=-@o(IGp)&e! zFn1T>(LoJ_(lFp`uc`@2Or1Z+mhEUWBw}Skoe?2*>k8hjhQG)<0C4jSZ`EgMZDrUa zmSKKNKKIX8HPn*YUuZYO#+4Z#Dgw_6SiFC^si=pTBuKuZ%6??5qfVj_&jrRPy$G>* zsyI0VrE}a9j7g8Kohlm-#kxmY#=1)l+Xs&8#HguvAXpo;I19LVxOuE2r0pzrMXf+q z+`%$?bF5p}S1WZx+E}%)HQcFxd zNj;f5Y7I?u*|~R{Rowc%QX8e>!2XKdcYO0JS{UTP8_iTosR2`nZH*PR@Lo<(6hoH4 zLLdDZUwkyvH|q9?Dm&EObJFdvx6<~+K}mh09EMPaK~ayAauK-JL;*}Z-tLFk#3Ori zJP+V_F{y%UTp2Q$I22~1Vgj%nV`&ey>1wpek%Fs^Vm(R8HVRnlGd|>z? zrP?FrU0(sK2J=0plx-PaIbbQPwq`xb-MLplqq%qo@wv0WxVbZuU8d%fHnms8^z-L@ z*#=`7FC%KiLXw+!3-~zZ-BjFEBomk)OZ@qvEP3|5H*oThExXu(>l`1Y-~%`?O7q=C z%jw>?E~lYa6u+{CLz^lXLe$UfP+$io}s5PgJ1>Ae@GtBPsNe%6(`{g#t8y`4UksI|svg>vkx`;OD44>U3xFkb&-k1Ha#ra)-u@o^BKy zT-gL*AO1avgV{VBVOu46%ADoJ!ID>y>53{kdHzEV-YOU2xH!3>Fn>3p9CX@T6JeS^ zm8K?MfhNAzOZ6FSuX^kB<}A~qKycj-yfC<_0HDJ8cV!2W;`b;U7oym+BnW>H*lgQX zxo{8iF7k-w!G#D4*-gBxd5ED`uq5dpkGXc~MyOA9xXMLW)cmxjWN{n@QRf(_WDf+w zsZL(AyLKGB0}L%of^O(3t-?w(gBBC=GKSCro@gsDK@1CqQ06#rUpLhuP!sm;?`3j? z`M(Ks=G}9yu*(BX`%D+N<7Ne*SD5*ju3ttB|CyW2_mZj7!zR~MMFpx>!)Iv?Z+?8f zaTxKP=a{GLc~cSB-i^r{dN*fpGV+z(q!zsR0c92O=Pl=3Vhg^M?UxP%EY7M8zz4uv zyg|HgY`moYANJldD#|r}8&yC7krJd!a**y$>2yE^q=XRxrMr=CMqubt+KmDN(p`hZ zAOlKw4k0l^*LiSn_kX`@oiFb?>zof~y)Kp?fQPxCJAU{5yRPeZ-3tu)r6-yG_dxe% z!HlQz72=Abhm>cUqKlUc=kZYXNwG*rIre zT%ZKX6lXj19Zo1v)>A&3)QW##*(G?!CSS0%|KuTft%P7xFgR^^CiF|}+ah8^sC>$e z1o@=Pm$@+Y^=~cI3Yy0F5AZ+ZFXBt#v&NwBkL(P5(=uH&$s;;nrYo?=X|86>ci`sO zU0%RF_&|iyF3=~XQVziFkQ`?ffV_hzkBOVXhTZ@oe$dG3abkx!^hEszy{iU1@P5~* z2oomB+TNewRAU<#HLRN#iHIjP@>jtYp37hd9-;yOce zSefPH!aXi`V2$J9SI@eMl-tU@AsK8L&o)&FrLrG7>HSK(AbJsej%fKPwRZc1gwF`h9+Isg-ChH^%l%@6Jy<*8?JC5bc zhhR8@I20)Rf#T0YgaK$U60Bk@*ic6`Uh@(s9Vclt!M}?fEP=`wmhf`s> z~$4FyAu z+zQ>F#dPs3A0D_{FMHv{ghFu=wK&^B4nGMcbn$4ZwY80@PH0&IK8l^a*NF|aauafA z#)%7nNa&k%soBL-TLjQi9%`rcITy~MD1$A`pnNN_@{NU$GY;?-Bx)yk%P_}}?!C_sPrVJ?J0=zRWwszO5D^ExN}a7wy+DK(<7SLS~8jQNh~ zBPs8A0LAI*daNR@TA@Lu zUZHwJeZ9x#30BYIVJubA3`QMTq^w#+jOkmK*!xL9R<4E=3 zElV+XKFYbTiGSY-tq;Az8N~TG8b$(>&u~h5m*|g9+O}mpV3}rrS)5I0a2n@d#Tfqp z9{N3XKKb-JQ9OY^`Pieck7@8>6Jj^H&*GUa${I)po6^GYR)p-qP7pYbz)8?MiEv5Yr_5- zt!SDLQV03OoVlB!{(54Hz8flM5@jYyp<@DqIh9N2i<9qLhgNcmeuvRXhCkcmya^Va zq5zhV;-m{`5JrzxhTgzR(PUW~)w__-+D!qm@SfDS(hwPYil=M`&LPJd*k9tZdIO%(ALj~_m=9^MM zuLqjx^B>_aQGM2`53M_YNkiD7cgkM`ns@KBe#y!5HKtu^MTyGjGEe@e^{D6q2RqDCE>CLl;_q%&h<{>1?&Qh@%EmQZ`|*m0%e1_G!_Q^nJs z0SRsNsQp^3!TAdNc8W8Y#n&Cc!n#)XCF?D0i2|OyP3^2Rf#FdA!DHTY@q6R408&s^dN-{vEM|A4?k(|eQCbSAlY@E+X^3;koFvsV%yMDy!x-@vrJ`bEh66=p0Y4qoxv^g(=sU#e_VPW7EI zpnc$O^mY-kF8~{jz4t7u6)Js;91kS&OTE+N>$}lOOAPjlX<&yeY>Z~?`I(%K|3ccSuFq%Yx+OLuttF5aVSf1I(rD$7UdJ;l_> z?GDur?_d?IX1oC)E_EM}swJ(BR6mHzEXJoQ_9V>Kd;o0nnX!cO zgM-!s(iOqrQ5RqAF>fH(qYYBalH_xV@Ilh2o*`bBXnV5H^%icpk;^BzMh z$6^!bKf-_ztKqAWnp95bj&V(Oj~QuIVx%}LXmX6_WND8%d)3-qLp}x#03EtlCrcus zqbv-sm5gnq4y=q!bXR2U$`}(O?r&wHh0OSV#)($!cv0y~hdakk$RdUoT~46^FS9M1 zOL>X|s^yB*qiURWLAkly#oQZRf@MKzeJmRY!T6i~HbPck%3rVjeRu>88M0*mqz!|& zOg?|5OAIG{c>j?sRv{jH%)+z1qvtEns=jWDaGd3tt@!uM4f4PN32^;zjc%3yn-t2i zkB85<8Vk2Gxv)!r)Hi9nTIE9hDTmTuPgReW4{kTsao4e&VM}(@HT=yHPihYw?eb$hWrgfhrCvJ z_dkgB|1Or?_ODX(>xB(y<`9s>ehm7<*8cVSZ1^8|1pEI9*i9$pu@(DYT>TG(9-?2Y zBk}x?IQA>OTKJ=Xp!jD_`^(?lmU|OMuVL;%oqtgLr*g*EF#p|xDO>A<%$2ruaN{FFW5{!j!@4aa5A+NLdK#MPZIr$=RxgFEsZU<9_9Il~4_Fd~~wnL2z+!`77k< zStWyA8C@w`=3!AM9N~Xey|TV1v~rL#-!$&KYg0Oef)dkfp3y!J98^04z~z64M+6rk zA)zfWsZoHMR2vYqXkP=oztPt1na0|tQ|pdTMpeIIcOnMqv}^rzk#26z&oWbEv(FEW zkLWZ4haq;S3Oly8y`yoq(}~-t&E!_ThBu0*KRL|Y){f6zzc0I^y1q?*m`=N8JR*0H zv7a91q8n$Nd3n^SKT^x(K6^NvzOakSNUt&11o89npH11%!yM)jBRzL=(jJ}Y_IE;_ z6=~*XmuyNf9{fLc{Ur4iPH$wHfd>YS`QX-L7biVoj0Db7aqxu8<_e1 zoECENKApWt8j&+V=q^hytPu|Js&V`U!2fhDI|nN$DNivo0=aEDE}n_$bN8W;tT(p@ zwda0Gd~mtc)~Y2k&hprnGBMMZwf>jv?)45yfCJ;Joh|9Ek;I{kcp%)q)8vhIk(YVV z_7)JI*F$He4XElbel_kCLEtox=C9?Iar9C()z92(wj^MkqAKd0wCLI*xlLJ7XpUKx zy*Uw9s@*AQM!?ecv^gGv9DORK$(Hr4x84{zH!UQvErpo~Me;k&L;%gz)tV{|MdIw4 z*B@*PTOOXRDBEI!KTF`4wl2wggG(QDorCA7S5ixndtqUyzT!2Qxr08x#_3jW@#Qm$oQ zmc_+4G?`WOc``Y`P@*i*^J-c8b;H0yJ_k}aCoR3@sBKx;b|U_~Vu8NAa=y2iB*gMo~40n$8x9uYvWc!Ay&v7f5>Y&!#J zGzz+7DA#fUy|o%ps-b($SLA1;avLRLzf_SDBE+rOzLtCg3SM*oyG~Y`IfM~s@sr%O zV4ozz#o&`+0=8@44j}xb0i-!OuR~vCdsJKw=Ud;l4MoO+*u8EmqFS5mO>$=QdR1|u zMRGd=qB1gjlq?luJF;omE$usHy8Uo2)N|D7e7r_!!U8KKq%$77YEqRNC!F(`%PI`;%x1Wykpi{96N) zfbJ;)=kL4G1DYv5zP=#Aax!v8SwC?Z^nlPufRRH1K)vSCw5#rQZ3IzRko@V@m!nw) zAG)>XU*@$N&t?G6wk+Rxf4lxe86DM>mzp(MtP+|;E{SYLmB3deTQn)t&3jB#H-lN# zNMc`LVXM7UQ$k3rWp(mKX1VrY`LCgr`EG|eQZ_QR-1j*2{%dyff*lk33*~(#w$t>e z4+>}S<(58KPIA8VKboP`e~a0g5R$&YjDXJrmg2+~_anap(c;cEv*NY!v3Rla6B-bY zcL}BeRBde-DT_+i3Q)d5_N(>kl1kvBi%hr6{y`eqx50ozpfY0`kWPb}coUUuyR_7R zpcovf!g`=86(3)pVbPP(48|ybXOV=hFJ210S!!a{NIJcdu%EC{+G*XOeRpnCO77fr z$aYxt*I#nQ*lP~>xlPt4$NNb_okwO_j90Db&xOtm9h@H@jrS0nJtql}%k(^>jp>|_ zbh%+3N%}1pCAwf)uirUmz2Y$WlyM=}@I(P<$E0>6v>PgS|)tXNLHFf}w32R}&) zAjrRQ2d^?PahnB>*HBX~mV>!I)4oT`rN^@|Xh0B{S+z|sB7Ls(D3jw zz~MZ6_v6-i1)o6nw~$ok#q`4&yUg7F!&Eo6$a?dEzD4QF_pPacZVq#J+ri9S( zy?&2yUF(&^!(cax!0%Qj>#!fx#COLd$}u_j$1W^uv{$bjJjx{4yWEi<&oA*V1{lVm$wh%k0z28N;#KKXf=-zmw$OJyV#gCfK zJ-=DZMh6xC(22tx29xuEXB69??qwdmJeae!lNQ5wDb*k2(`gcvAsQn(%h06`iM;qC zr5bTUd$g7UtNie9o!1zllBJ%7389`f(?b@(n2&5vt3Xgu;Hv!j*N4|-D;8VhUj%00 zfVwvSdu&^5#h@hKfELSX`o3XKiS;UJ+zg;a`CjWd=X8oeiO2bG0;cF;zAdQo@!gHk z&585C@i4p7yg@@7uMWxPtTEX&V{aiX!bjgJwB^#FTBSc=pBT5|atW9>G*#=S~se z17=KA6&Nq4D!`ol;arvxE%X-n*muGM^5pmyGMp|_@u4L*6G`Awx`eZle3sI>*)w1M zJDUY@^3p3NHx#9;ia_YeW_aGai=Uq+kEMu7>m$p#YJcWp<9wG}7nHqRZDM+_Yi^qB z5Egk7f05dS)_x*nDcJs1-uKj5!A*Ln_>e=vwB4PjPbo7%K!x5tt{UxZ>xCUMs zxwR$6*j-@4oDoIBAQ<@O{Njh~0Vl8cP*&Amzh}?7C36;OwZ!F9%GyWOqQvwRb`5&f z%63=JJQx`1Dg`fxLhj-?=)K;}#AU#)MDGtvslpG$UZMl5mP&Ssp*k^(zWR)>3S?VO zrl-e=Xj$<42*k`M#WwI2&i70(Zj0NWw9lVHITz-?XpT{H>XzjDVX9%o%75f~^$twu zhwH;EckGkpmz7_tyk(!1AVx{_DmMpQoL0OV%y8OiJ_>L4-5hzyL_s#3#hh)?9()Vj zwb~h7K=9xeE|&1Slzd=c2K(tSNme<&yZZjXip)ZrCSlbn4AktUN>2DK>U^dJo&SJ_ z`BQh~=;K`GMY3#o*PK!r^Y_HpSP5hbKGT)iV3UUwOW2=|2 z`$1kwPs!b}mCXyW=gCSPp3tY2Yc4t7woNu$}Ltc}4iJtYJ zR*$W5*hQ|T2~#>EO5)ZIe|e^K_rB0`WYP#+nvfv0vLE*rmVLs++_cjLOlRH2hiAH7 z2bizEOWuY4pJsjPEKHg9Pk(NWlSF^)oUO;D@H0%r#lt&|oKZ3&tJi)FtB+=^k1@iQ7J@2Wg%l>agES$^G5rZ-Nc3z3pa7y8w(EmYH{yzRYA@NiSl{L z25d*JkiO5n?3R=Bgzew;qC@ISb|$w>T(`BrFkX@9KLSmc876;RTmDEpUPa74psXQx zl^|}Y1rU`NtCf6^D-0nC6 zUKG}f5;&+_vb-eA=Z$Le_Tra6CCE#Odd!ZbV?Z3ayo10UhNiE5#wr?`36} zo2;qi@w#8z*}!8qN=?pRK(YBAt>+Ipw_a5V^t&JoOMwcfqM0i)VR^*!(?kGkQS$@% zR~Y4g7W}(IEDP`HI_GX9fxh(V!N?)U=7VQi=mezI4NGB8vB)%)ItP1j29#4YqDKjp zF+-`0ieXZ@`k9^I8T-{Yk11+i!f}c1L!4i$Qe_W@K4xRYljP~?^7_{WM#fBr4c=9m z+emQ|aI@{v@a~=tq=R!my&hT6TMlEaT&efz))l$Rqa}}pGmI&{b z5l^#$D*x536Hp!V+ae6Ti0{VG3^L;sJ@e%X>EzmaN`8-yz^XXwdlV;^Cmi=?^8EQb z&$DeK<3$OM2HMEVRcVHyl{|Jungz@Y)&c`8Rn}9orDWTV3puRA4T?-SHIf1|F-+=NDsAki7rKmT zRKKVzK3SYddxL$pko3TI?^??{6@kMDdMS6NJwvw{j{DsLyWy%O|#vkQOI8RpL9$J%RG#LT5Ve$o@z2?&`Acq8K6egr|#$dXU}&LJKC9( zTa955dHrM@D4ZJyq>E|7wtD2HfMk>=^^x1FbYRDapF~bX_7z~5nsMukNR!cdZslgp zmg)F7)}e2dY-Qcl$UV$p>=&wF7-9{k@?8fd1#2N`az%c`r5t5nGPgp*|%h6(49jqVTNe)L9g9#1ejGK^W>P=o!d&DQ7d$C5RyWjYLMnkpY7$TI3VXSaiOGs;3f|qkg)(0DfL-|`I-Cd_W znVz54j8UjQ$X+_@)336W{lyy3A}9nsxGaLhzPpZZ?D92R_mETKrAJPEQtcxO)ygZ3 z;1LG? z91)vQ*G8xf9I7XGoUrI+h$bKnnOv^kHni431GBSfnJa>bpCp!MtRLY>6nd*E+#%3w z_#WO%n7sk>pBPq?xmf=Zmb&6-*0S=L+~1ZZ(B-NC6kmM7{Pgz7Plr}=byIxVfyjy` zSxV>Z_YP5^s|)z=S~JjXCri?Lx=GKrUK?IU7){MvsGah&ZO5;B9YS5zjcwGzSPGTF ze?Y#!Ism<<<-_=lj0`8BzI1U5k6A=fcu@%a6F=1A=TIO&>0UZ98+)XaPCt{1kX*a) z>dHt+g~N09R)+xZyT>mqozJ3JHHw?QMMeY3Kr=UwN-*)J0ACQlW)}$DKcM;eHY5Ll z{xgk{7w|RY=#sg9l;uC<=ZMB9EKo< z$C2dBU4=|I))-6|YgCvyU*Mc|V&9IPp2x2-QeR)Cb8Xr(e+XT-j+bhcJ?MCZjnpfM zz`>8t-e!OJ+1yqtxuiN*iGJjj7k}cMjx=#x^^7#$(iZXA)6jjQY;iKgN?(a%2nXaf zxr^tT#Z|O@_7k3u+N)5bzL>l}r7LO#@-TM7+b8a%*8Mk7vZt}8l=)6*#_TtoO0xlZ z=<$lm1n6#5mjm&lGbc;(GR}&Qhz;cFdB?pzLm= z$M!=}5l(6#8Q6wZSr3Jay!jc*V_Xvoh)G}Hw*%DG!(SwU5{o^|-=$vFzL7RF)wcTM zb@h< ze3=k&K8UhWen(NCL!t(sn$a`+{V|cq<)oCNiDuv0f0m2dtwhjv1ZvsGb^ZGTWmIB5 zhzF?yhB+|&Nq3ULu$0#yK&An|pN~ck{+ad#g3(Al9`eF|XkN7%SGzU$)~qDAiytv(9M-yXvbx|3za!EysnlO_d9mV8cMag zK76I{nVT6_HGnk4psyp5;&y)3L3~feZcq6Eg4a$!-XBx-dyER=TmU&ezb&%hJFr!X z%WnNBHK3Akc_|)n?&UoESEu0;P~|{KD{2QqycYdWQhuc$2mn#9&ufI5qhWz8X!@Mf%R&sL<|jSUxT_~>LU7iwjpD|7Hj)vW6AIWs6e^Hsi@bT} z#$DquLlZcXaP1&uF`TD-K-=1TaPK%K|FBROy9z_Un@*Ci$37}-E(3@vF(wG zukNN&`cN|p#3ngV+j(kbSLF{GGpnf3`ov9j=xwB zLdfHW4@bz`Z(@CcxrA(4M|;7I3l>?#}}xGXKsYYKP<6 z84qKiv`Cdo+RG_65ur*z3j|x$oRz-F#T8nY)ax0 zSlPl@{PBY={%4lnd@aY8u^sIDG)^QST4#|V-1~)qw-8BoO5o*MAW`jHfNuy$8^&kp z)*5iJ2vq+Rg_Q>!loB$Z>e6PF7aiWR=HpuzW!KOzA8lXkj)8cKyNHJ=(S8C7m8M<# zdB93Vh~qfyV*nz7y9Un>G+^Jp8#zp3mvjO!xAFatt*gnD-q0y;q^vlLPCUy{?W3H; zohlsM-3^{z2GIQIysy7w+O#-{T36b;5}O{caik`L&9l2KSsAWks}tSgQ5LFShm zOMIcLM{|L4Dn>%HN=*>%@OOiTiSqY*9!Aa-Bq{a2`Xm9=#)zA}%<36uy*2z!BXsY< zX5klhONk{(y?Ej9|4dq|NDmd5A7DXc8Ontvu*d?q7R`&(7wmWeECbEYi5h!j6Nmm3 z9%s|6`14W|CBr3gtsH3ykI`#x#7*3Z2E=tGOk2DL_!rc3H{@ep205+>PPQ$QmGPKXSRY0qQ6geyjZ0Y&b@3j~ z<(?==nmKU$cBoUhU?Xqf7c0vw{q zYdIMg8jd$|e|O(GMxkltOrS|1ig9Q>6zHp4STf=y7$i9}c`e3| ze;X1goE9pse1L*VXyq- zqa-_`I@6c=5G;%}tQpx?Xwp5r@zGoH?D4e0f?$FQfUKf&M}&mo#Iui(w0XZ*@U~;I zhpyWk21&k>>FZ2i;Ui|&TWN7|^cK6n^y?iDc8j>S|I*5AH^jP06DTM5lgWx1 zb&02FF`(!Zf9<>!e;?mcas5?RJ5_LrHIqxmoazIkr8>VKPE5yXeMrFk#Ia$qag4*2Ib+2@pg=MfA(>(Mvvg}&->E!b`w<6sJNen#`=MFEOsJ&cKJ zwn@2-eV+4{FlkALc*vI<;O}jQLBdSkwa=(=nFGj|A@-DM(# z9O)cz>hb#=O>A-_7y`)?X5?P}C|=)+j3Cx$xUOdGo)afRu|%Rc#p?@Cw8=%7JrZ6= zA;llfov(^PPVf(N{d&b{@J0$IjuDn3+kZs^Pz7ic|=b9&J?6E>3yI^h=>Hl#T2|D>68Z})1c}AZ^MubX56S~&fCI`}<(#+bnhF+f3{+H9 z1$gHY+AIm?S#@p2?0q>MYGMMXyXX#S9X9(LYe}cf_kwXLlq7fOZw^ZYc3S0n*+Zc@ zkB5}BE2K%+loW)ZYHwB(Sa??odQe&t{qA>_ua8m=+<2nQHW6*CSe{gxB#%3enc^6! zAOfpP*hj}#vG=Kn`BRv#04IN1=Zxw&;5(#L1k9k~tdY0#gI;7785k({kYkH}=naH8 zb8|a>lU2XrZE!~0Lu4dB!uGDR?MI4e;e8OzG1t9r9jf1_O*+wZPZaRj>))qYt2fIN9&VYtx;*0R&mM#e zUtY{z$waT?{lqH~=dw@eQL=4oXX(hs&}2oBr^klCH{DLwy}BH;I+s5^P|}xEN~vOq ze<53$v5mZ~hjS~?mP1MS`|W-A)JcM~prk&^rV(s{-@>W-FY>!=Hev7Y&E{BfKRyp9 zaa6ij*F~FHpr!>O=rN5wOG6m477`ao_q>I5XV!bCA!?%yBU;tZU&Sg+;M`-QGUiXQ zXzEK#I^o=Jxc8q^r__p16jEAU)nqVcSzJu%24;IoOj6eS%h%7tuzz4lWX}I`)E?7> z@k;&gox-J*!42Mj@Tn&%{}Y95i-FVh9|kL-?>`6fZSwz|*Y&9XGg{espc&~uY3#%Q z99ygP$qO?h{^So<iN!ZyH>QPJMs-nsI3fK_W;1E4Z6 zOxE|%&N-zQ*Ag(}yNyhyyMCq*I9KO^nt@VsC*VB4u4I!sbDo?y9gn`x)v@uT6jD$PqXlyIsFT)euDIgMe^M>?=p{}dLn|#U zb>*Jb`9BH&AJP-RG~9J^r5JziNYWI2o}pc}pJeo)mwjrYq!Ilu@)8+Y{Sdgz0WWaO zSLOHz0@A@ZEAjAL?-gUCr(fN{Hrv95KRhNz^V@&AHR0Jp7s_>^OrgJ%&z6Zk6Zg)#Y*sS?9WyR7|Nx)c^A6hp{?$@Y~B1A{Gb?qkTac(J~?$ z!I1wmbO@7nFMv1mvZ@w}$B2^T{WRlUP}cy%0cOazxey-Xka7x#+x;*?yy>Qj|=E-r_1zDc|c?x%lQx%T1tz&1+oj{+mob~BWzy4O z-2eE=xcBH zLu#vXeg_;W~dar%BpXjQAOL6I~NJHETqZlTqSui8L-W&n+hGI`V$Ht@xiM5SyLovUQPtb2nh-m9 z`>G$GYN9(}B8^K8*|+(K(PHL4eaEwT+Y>t%79D-23x{?yW=lUKLf}JPP1;Aq=J&^_ za|a=`PO@B{%Sd`>DLL;q&Rzi{#qq}ON*1U& zU~Kri@A+w34^Gj`emAPL%v-s@zemXgS-aDgLOYsTL)OudhGR}`au>L(5QB@YdiY^4=aGANkqb_}G1~?egNF02D7tiD*ROVRtmQK^mI55BVu_}Yeyrw zZ;G87W{y}*NQ!co8-chS$NBbzKb<0mySE+>Y}uLjM6_h?FnlQa8Wtl z{B)j+F{f+U9r)yxpH}0cYtnMo`(wdCYRoZaWrCMealUZNOE!Ui%v@HK=e*pGHWEb~ zSI4Ts@*L3m#LoHI1^^x%Zd6T9#wJ%%7Zfk8xZ@bbj~x!>baj6pL6npVq_aw;7_l*t zFcqgBl9JW~WD~#dMN5g)wCoGp0gmidGR}*i(?>DN>i9qm_$Y|i)Qa?WJFd#n@5q%E zWIMkMAzLzdMe1On`uRZCCAxWO;jl;WFRcocfDpiitDPdb!+n+J-uw-LjPImG7ZP~D+tG|PF$ z7a533pI8u@2-uc!0n|~hh?L#HSz+N=!v>a175=N2HABrkW&I9JlpS-2IizOUVExt) zv8pPc@dL9yo2Sm07#>|UpqkTIqU~8qJn1V1{XXwBQBhy`8cEc)RtaZ`>t7{cDYka65JHBeMJ62{|W!%){Zp4fSa$Q)w-aY)q2K- z^7+M#9Zh}7)$;Ji>u^eFnG}du@Sr-KGQ0g(91Oh<;@5k_@gOe*!s@o&UCqjG_wDd_IC$tVgW?zC7fCJS zMzNGX5qKp8mecA#cf5MtGy($sQYUdG#K&%{VK>*2HyY4xAKddo4<6 zE$@Z)L;a$w+NtmeIWHSM^1lkRuCc`Q9$B~Jo~=n>$Bc~x58>4=N5-qa=M!`v%-QY3 zWH@bTnI13Y=ywv_S3-q-#a&V*^lr8?{prndY_lMT$syHunchPnOQVrW(i{NE`dk4P zAZ?sO&LlTyp>)1$5ll?UTza;3S1va@Nr9k71h}{F_rHXFrA;fY;FArW zfYivsk7MXnyO&7*^g!jy{`TFAF^V2cQNfu~7W;R?!G&G?R*|P3zt?T8nGuQXWn1X( zw+MEXy<<>d^O??5|gI`n$v2ABx9F`tMc&3Cs;^-yte{hbjs)GDl@A`QG zU()ZoC4#%OQ)r}m8D{+|3~Ie;d9~MfzI%Cj+JrbFG`{b^P*GrEdc2QzwS)=!wax{9 zcXjh`u}NzQa6HF@$cFW_9%NJIWLFDB_Yub%(xc@PqL=!I5K)T4@E`W$eQ9E6{X>IM zH+4`+eu-+B&phEM^Dq;Ljuu_{iZko#o*GEbWX^E4ru<_6vA#zDamwbDO?#qS1x22%OIL+W@-RJ;GNw3GmVfe8T;X+&82f4-jE*l_(F+Is=8zZ1x0Isoyt$Y-}_`ub$EfN?cf z(HpTEBgc#)+ZC}>@6)qIH|;lEuV{ITjmjs#Qf2hT(@RCW#K3M`udhl#`|q?d`q62j z0`0L}<7 zFW?D=Mh!8q1b-2Lyi7=G>%6Osa=8xD%0TWo0Me-aO|wqoMkA3w0i_*BG}46!$S8lu z1K`)JRKN|}*CT|KeGE&fiO+MB021Xl3gCuoEyfd4Wm`_6Xr%phdYf<>6bv{4D%aN#7g1Mp*i zn%f`O1>pIe05;zdgpJ#Af?ngxG%zrr1d4*F9FGTqG~ZnuU>QG&Z4?_iMiPx$yPg55 zJ81$f+36Z^u>t%tr?tCX`pEe7Y<1T2eqrI)c2xsP`N-7IlZV2V)`?@ZUDk#`LK)6Mk>FPPrCLb2cCWxpV4zrj2x z<+`fn@VN)*TKdDCImV#_z$vP%q!w^Jt0|eih7KIG)^m@xCdkfgP7j;jtL6ef`6Uju z${2~@($IVpVxBn38DQ12y}-8!FF4FTvIdaT>kg|WE?J`%dKD@F=?MkIGoo5+K|c!A zlR5yBqZ6=}a(yEl_h(cFNei9(*53j08(LsdP+kGvPXdyzcLH2w9w1Xh(<#NM<(Yg0 zvuXV^=IDZ|B=UqqEGLDwGkW=KC8V|9n^CQsex5CR#yEJtkkc9vX%k-P3E~hj|0=}? zKPxpT_h@5g-oA5c-6eV20<=FcjwE|$CzEAE!~pi{eeTZM7vMq%_%_M#&*$@eRrY@2 zi^d9{36BhZJp$?-L5M{5_p^G&D|D+<{IXgtgd;uWKf^P*G)8px4EX(0^N3>ts{ysJ zxH)*USl3nAc2aGaSX;#(*^?3M0k6b+PsX8h7hs?9ICOKkdD))^`{V%Z7nPWUNo<-9 zP*BM|kS!&ql`Gq9w9N$Yv%lvT9{u(YC0n|Wstn$&*R;|(v<#OaX0jH=6 zfV)|I7#EDb)~iM&u&A0Xyo-8ruaK0=R&07XX-`Tn%_ep=5-Cj>xZ94~xm`QQU_O$V z0zmGoO%$7ijnA8bS1HzmS%Ne%(cofI=EecdYQ&iPF=W0I} zp7)eo!5IUajHXhEbpS+qOawyHlaHYsB=i8n(C@fVz?LW6K^&MHczLor4uN*x#1fkZ z*bxz_xqAsba+iD1&U~n>&yRI%+d!ZfPhvWA@*L7tGwpTGY@Gj-00TCwR z+8FTk!#Dw76G?*`>fYu(bHIY_WA{^<-@LX6z>*Tf$?qwxTx;J?Q(+S{{?C1j0bZ3f zqal;uLXk=?z^@7p1o6LvShW&tbyd=JbwHf3IDRadB#erL={B1ep)KGaDZqRRP|DHS zGCEV&yBXjNc(2?3CoSME_y`_{jtCiKf&i+z528n3ZraKL{g)|JE>M;V+(JyjXGV8@ z!GXBp{7xd9CQaT;XV!00DzqFAgKKm#I941~>#pE8|Q#@AZ~M^ zqNt>mRijFJs_3e7|72ZBsZRxxhxxH8qM`}Z=DY}PY{g_?ArBwhZgEjcaOvMp4$+CH~Jf!jl;i$_G0Ko{R15i6Th%B2VbwvO~Wj$7EW ze3#r-i?~cQEDm%)t=_EEcsX$Ehw0%foq*guf!z=Ii#N{k`qax4s#>?CjtZJUgZ0++ z=a+rS!t5I6_GS>17*O0C5{Rz18Lfb_X>dsytHzQiQ!IF>T7B{r!|XFF$pGuuZ%{$_ zvOL>CHR1rOKAvNg0)Z9TmVrz|U2}qKNJ?lm`IM#ddpRXjPqRer zpJRkZ>LjmfoHK@+Zeri=`r<<~HB$SIA{4_!N4y=hcV`|KUBt8<-nuPt|FJ$Gw2W=Z za#jq;8YcfU8%#<;+4Jl@gvmE=h7i#G!kpd{`_Z`5B89N7zePZyWX%u}+bXTD()9tY zGaPr#p_0SW#-?fC!Xk=7*NRp}dH{l-x(gXk%}>(ZW`3JQT(E6E_%KFIW9k({h%$2n zQa<}2Exf8vu-aWu{;`K~G3P@4NAKi13NTH3`c2vLPC>l+bD@%A?izrF8uI#OBzqt- zio3}VmE}Cp#JhV}i<-5-per45iV3kx?WH>9?6Nx*pTutcI0dNq7R#a|VZnblL`75C z)()_^bk1JsPgR{ulBEF?lLX}Y{;D5m91-IP6prGbcJwxeik<7de0b#Xi*QSbiN6@6 z+Nb6(#Vl;ea3z#2Sm(%}Yc;|5iypWSCi;lz*WMN%dK`C_WLb<@euBM2^oR^`Ghucb zvg^Ocf%K0!!#_e`GB3_Ak6IbKi&-_$@91tO*M^9T;uPh2kRQcq>36v5qiOPi$g+(! z7bK}oT)AhwYh|NbyTyYc{0kjq`uf8N-|U+fmM(1Akp-s2zv<;zToYB?lMp={hG`vD`4_ZK>U zhL7e*zt|H1w~xc(WFig_Z?@gECv*$(GH5>p*mX7c*s;LouzSr3T zi0=O0s(Jy9K`L<0aC|wvUU#yHw?I|6avU>q4j7YlXL68op?>(>z9GIeEzTLDa0}}} zpwVr(8N9)N7`8h-+SF6W05lYrh>?grM31SDGq*qCP%D#d;jkW9QklY;H%YwheOx;YlgZ=Oy9gTsB{^1sOyGH+ zjg+~_L5#jq$zJAZr+VQa*i-y;l*fwWL$D2ig|-KsXs-~HliKzRvq_7IlnXp()+ol% z}<^)LMb4tHoHq%rBkrpW_v0y5B&%(vC89>OPDuyG=qT$pf*P=7*gosa%mLE)^ z#>&kf{a@U@byQSQ-#@B?fKr2?bgO_;Lk=(u(nyM^goJ>^2na|Gt%7t)x3o%kNGLUQ z3`+M1LwD!hJoUcMy1#YTz5iYo%e7eJoU>=oIs1I~_fv`2*Ee?eP8X&=QwMIkaD^`& z;svxlezriGSFRT3Gef*?HX)@GzQ!6d=djtIxQNUtQQN;^gg219BaP2IE?+G3It$K} zW4JaoI)zk5y?q11DCDAZ$hJfwDc>a6jz-DI#ZXUr9mmM7-5XzE6*J|$=Dzv-#%Vgq zN3TtuK)1HW$K+<4#!-*dBxc0Fz~z(*1;=2+khMV3lPr6^2g-xE>?^lhCQpyDc{hcl zpoKYJNGtt=-?V2hwv8T7Mji`g{6*sZW<9%ccIVgad|7wlO$e>cM+~$l6U2*Y13dlW)y_0{Iil|_>%V^#^^xQc zl-YZ;0>~NeCkwufpSWw^VqaRlG|TURXKr{aSvOW1uUD+;78zV4)aa5E!*S8m^PFe? z%Rg@WPfnAp zzIjZS@f$)B05hSJk)z`*wds|_gDtbkM`e+dYI+|DvFsYjCC?Au8qlWDmGRtO7IGQ} zuCRX64^6JHT`P(T4v3EY_v#?GM8&1Xtl}LG@&D3XPY(@fNC+@puYk+RsV@A2W(a!< z9=P}t9>0}q#7A)K1R{Ef{g=Ex!9E=N^haP^?7aJO9ZZrY$Tx3~aGaXpTR!^zikVCB zmuJA45(3YQ=v-)yy}>RhNUaGazJKc9x}~_y#SBv)6wpuk*E(4dfGH{;{bq$J z-IOw6GZC1(Hz(LNP>Adc7+Uh#3CXq0EWc5BeMp0Rgw}oQz3)3qi!9bFV-|FAF|1Dd z!k}>Jfzh%itWZqfq7Wqaykct5)=AX7OR*&TtsLs1@zDSUixOdX?IwpUq@xw6S#R)t^pO^l+M$r9Wnl5ZtnXw9ff>HytHbbw zAo4WLu)a&`H&-5~L4MVnV9c)Qk$x{p%U6wF(VE+?DFU931paANVi2ms$#~A5pT+Td zpFDF5`kA(w#Z}S4PryEu&5VN&jzf}aCKC0XzE3-Oq&)ax<}*fgwrozrWGS^L2}(q~ zJ4p2C2HTe?%O1)1rJI_Ah%M@?H?DGCq&MGvP-0M-K%Kqi5XB!Ohs>a^F?w}KH6c54 zUo&VT=_x)p8^t&pt&|~j?bf)em^o`8W|jN^cCP5lJ>@h1?rUG&W^%5u+|urqPcsb5 z!WGlM5kNT57&!&fkNO*Ete}Vtwc(*5g^~~f3hD&38&{A1t!(5taAe zhKH=2L7@~A&5X;PSyp^6u3^z8A*_C(3x1@dLAI1``C8Ff3L7wgjv&U)~j&zidjtQ<{MHVQXdTBP^ zc>ZV4%tMh~GmAm83v5C0pWd1walon+5#5XA^b`UcA+kGwmf{YRp0Ko!w`;LaM3ts7 z2+S;WJKI$+O=O?WP(2xj$&^1P@Pgr&_x!z5?#HR-cy%r5Dqfq2{Zq@q@--^jPnLm{ zBMh2}&S9I~+xmp7-ktMp6@#(z$X*_w)h*p>FVLZf0PKEvzeDu9`POv9e@1WjPY{461iLqOA~nN z3qNBT&uz@`R$|sR zw(CFY>A8ZYqX-`!41gRBgnLmD-t3_$$y$>gPv?2ZuJM9_>X}w_&f5IAi$Deu^cnV-vKJ(1@N<-tIGyaMrzX2^R+jqJ7e> zkBFefAAgetzRutYoZhD3q7S$ebL@Vs`!$ENM_|Hdus#a(Q+B&1v;()B$y;-?(#&;z zC7Bp7;SLw_5m+x3snCe1M@lAibHXRmS?`XEc`h8hXGM1?xNC~A-NO7IF(QsFE z!mn!A*>-9Pi)4qlb`vD_VeNaJp|>qk=-IpGU^|X!Nq034YM}jF4nD0%7UH7qr&d8v z-`e%kj4Tk|Ao;${W?mtOOcp07KiW?bB_M;b!H(o+=3tmYB#y)lISBiBsvgO~K z{AW3!RW{7dW@m~eJpSZCSIt&_b*hC$iTENK%D)%DoakLF$NaG|=~M?i(+~-L@$5ah zp<59A&6p*vMJbAJlnl%txlh`iUF{TqWZG0x!w5sr0iV?~1aYA&sV9#RxsGl=NmbgQ zp9;FYY|pd6+4`k9EW~w9`&k8_bTnc*wq0?&*;egw%{{b=51DVUtr3n9;sIZBg6r~{ zpG8{Xh81tZs&9ZxIsNGgZ>7W7y){7_38}`X^nHCeV^XpQt~i(iK{(0X0LyRD=vtm% zPzS@O^|x32K2Yo=p74LzPGnb>Eh!9giZBZE?3P-5FvNE;$Z>m|cuzfLKCa-|bWBcQ zK=h4cnB3E2(7ZHhY?plRO7x&-i5PV7WJ`){zc-qRn~*qHc-3Nrev0;-l-k8%E{KS7}Y$fA)Y~&8oM)vcBmk-4mm}L;f5CXZj*oR73tE5gaMo&H6jf zy{{T$3PHVnQDaDDmw3L+r>@9EAvJKjDB;SWz%BdUBH=gLG9sa;8^bs`V88 zUcXgtR5_X2vIu7oz4`$DRc=B97wDC?lDN2C#|jhu)A-K7 zS;#rH?nwvBc7=4rJvu)ITe%;QnD;Dd(W>@8$M2*gzE&yA2t?lH`6fE{2-FOK_+vm5 zF_@-J7rz4u2Mb}(ZBsDtV0}JImqTv(w7W-NzLARh#|B_9&Y&O1W5ZnNv@RR7j^qJd z>i=obiw1F$%CT)GGA&u5CS$1}qMZ$sO8?LO`0miV$|^hm-<#=rh1Y@d;QxpC0>D`S zL+%o1`BFV9l0LP(yc`rIFPA0P-^UJgK+iX!x3jwlO-%td!3FOI(8fRREnoWeEi4)v z<)!v82|6O_y8ZX|UZ_&xslM?DM69r)&xUS!4dA(^fW<*3I)} zb?sk~Jm66OM1LUJR#sNnJd4B^B6;7w1r`xd=K>&0)3){UuzE+`=Z02Ye&I-Z-n4%L zS)iLK_75lqsFxJ6s%dPT*_97IThqu(L-aU;Kf0z>+JFIcA%ieu74aF4HUC1U^b){W*Al?QBlS-B<&gzEU;tuj zntDI(fI?05UkKO3GA@9%ffER+Suun2R+xbobHOo(O~5V;&_yqiGAlDM)=G;Y4EPS# zn!#tO1Ou2Cl1HxTscP0vTuxRb00;m`Neo1F13_I|NlTZ|q4&drqy^zh-nMtGP7ie* z4(9bbt*leZRqfsGSy_)RSaax_;_gXF+G;;6l6(o;r@eeOxg&+;Llyfh%4uC=IxuhE zlZ+%S766Tz@yWWL~zMY_n1Fmnj zKBgU8PmLy4x7);Wn1alZJ=ED+FM-(Rts^Jh52o_vm!tAaKp^3lLKyamBDl(rokQL0 z)vs(WXl`~}1B19TTV2)WS+U*V)c2b#d*kA?e&*wPjxjbRt0k1VP3`x_n}$3U=ft>q z2X9Gu)=mWtCYf?*q>xc+uM*+mY;J5AmjBwR@=aKA{**b2c%j0-z`)SkOO1PV@XPf1 zxca@!w|9nDi3Nw&YfVyfJRw$LoRh!nVs)~uQdcbd(y_)*oJotO+`e>pE$Z2fB^nt2 zOdbpNY&|h<|9D_}vDG`lqigg-*l<{Fl5ixK$34tfdQ@Ea2f4NxbA?qIcn%#Vgn*PM zUlUiFhBH9+y`&-UZVaM=OVqOMuVC?pOl!bQv^mjM@$)_iGtdcJc?+VC?N4JDa_xTu zmzHJ_X2ZMlD;^CMko1;(I^=tUa;`eB#9r;?f8PW-0B8#VfWfEB;7<7AjL)TrEght& z+o#-3ZP*{PN(VDM3!yvF}PXof4s8d0A+HWcP(;&K`lypDT6J7F9 z-0txe@3}w@UI`q4XaXaNU_r^IoyLfhNpJ=0h>#&o(POCvphysJyi|Wt;MVheMHjiT?@l1Qh+;`&)z|aoj0;^tX)k@!SlGPKd*Ihy>hchd~WJD8dLvFr@i*RIE?I=)tcn+H);T^ z-1WSjB1ldVSc`Xd&~0KPbtiU*J%!k&vCY&j28#{J#}H$Jq^Epdo967{X^M{bsG z{=r!oi4;|@DbHK#X$?hls=b_JeS&t0^_vZ@63Gi!8QigfR&4qE7u_uC8FAEY@!n^L ziy}P_rM!fMJnA&n3T*Wi$zg4 zRWk2`-yy}G7bPf9?d1#p!o445D;wSyvl0Es8O*~eo%RjeEstkoRVGKRrnO_XYoBMa z25QmjruburAaRNTEfeF>EM(IXKw(eP9FNa@@kiSz)Rf}%G&j}lK^QR&9{-(J40b@{ z*7pA9>4?y}i9x3lwy;j4>jM8Te=Hqvci{z^{`Mk}FpA5F5}Fj&x4keZ(EyASI872UN;rpT zjN5m%h0;@BLep>FyvcyL<5FDwK$(K(*nn}vlrL>GCV1y;4%R2J`&=g}E>nR!foz;X zd8_nY#U4OBkSQqdA8x5429p5>p`pFBVXQkK^qPUORC(SQ_B^16=qJot?(Pn~DbpZq z{Zh3x0RYq4OVIJefSY5D9xbm6^XIGIecBAK45bXR3mI_*d3LCCyVYuYD03HI7rX`8 z=?MhKri?7LWpK2jq}{2UAvSvM*$Z)n0<(j#e1JvfTZE!doPmK3pRONK;iZ|{cZknjb+G?szQd3_w!_k8$sp0O+gWundu^qNj zJyN~LxRr3po|9o5&Uc)UyTdK_c=(MNS>-R+7Kj0+q@>{3L5sR7Dx5*}V_RdR5MNLW zbG?gxeDym(EGafT-*b+=w??p1VrwsjbzWKFXB0e14l9v{ORILCdtqB^Mk{lGiOzUp z`5VRR2nh?M)Hil7W^p&R(;-i6bM^e*CA9n`x8zix(foRg*~s~)+T-bx)Y6^Z#rQw7 zwrR;s;{-Ae)_Bwhe5n`KKfZ;DRsK!|mU(S`j%if2BnC|KrGpmg=r!{Am-GlnTsuF~ z?99H{`n&%K)6Kn~HKs`%>Z#w;(kK(>&JE!%_Gx@WVi~*W+R*UGdG5Xv+&FH2J~gaS zOEMYmS|a(k7iw1AJ)r+1cx`4*O!Y)FZgf&M*5BLRSY4lwiGIZ#=XYFaBw{uFTYlE< zcX>`e(1$rYKS(;a2v@(+r>ZyIkG8jYXER_ks&kB-d3xEbdB!V zdhBHO>>#;DEoyR@S+Gm>PSp;oghHp) zYw5XC>rW_v~HzjYp z`o(weQNU3MDGlB$qNWfn8e*uQab0t5z}R6vP-{u?i*)5qg1$x59$?U5NBmYIX6UaO zU9x;U=jQDB$-Z^8xghDfo`_tPJ~gK=ckr0r{ecRxX@wbrQZhTX%?nh3fFYF?1mI?@ z_2x0P#*)2<9q+44Wt|?~>>czn2#~XEmr@Xei4hGDc?7g%6})09wtm+JiPzU>K@&8;m4sEmH>$QXZWf1mTjN2Q zaK!1fCG`H`i&wuo;?2Oztx9v*rSY=vnj-qo~1^l^3ZV~R1 zoR2QJobV6<*Uf|J<6crp#R4+swHJKK?61^yO4xy5WOTF^IENkp!g*>fG$O-fWr}@v zsEXP@u5{YS(^&m*#c;5>$CigI$(b>1ynenwewN(;{YeAP&kD%(_8}N~mL)+z4gHay z?2US(fO>N}hBAaGFflX;D9E@l*E~2y169k1)x*z=O&cJv|<7JuP%{D2Njr4@K zZ>TyD6lj+^biB)VI6UmIJ_&9A`m?Y3{eKTgG+bhinrDuw-F5HG*1gI;9EG7M!Q z$xLmM5MWkkcwv=-_Jwv;%BSDl!UbBVA(&Uvy)IUmKPQuFSnTcSzqr4*HEMl<6op?2 zFnKyTTD*OBwlz_o)cijDQI z{m`9rX25M$u{!vqh|UkR!KIaA+2@1;hqbgfLgwwrF7`WUOPG+v6+gmT2pI=UkIG*z zSFH5k)*VlwY<wpwUR~vv3+P{r#QO@u8}fb{Sw>6-J#Ys{7hn z*c=XTFj1a=J^917>T)cc)m?FJ1{jKlIefOVgPq1^fwEL3V#253%h=Xd$rQa?`xTkGmQ0uBya@mGOY) z!T)UZl>o7Y zyD_koNyzevtk9JvzLhIQ$*k|$i8uy?H^5_IQ&~62q!rmD-E@`x-IYkVJPb=yk`_>@ ziq#34FVHQ!uRt}Ul(SaSxQf?=y7hjPtm(DZl0@+{-X=$=C*esLHo!8FMNOPCgZT4i zhpVyp3HqovjXL(Bf=8KcbjL2RoT(0-JGb0(20Rd)c>P#=UU)f&uh|$eIXPfHR%63- zTz+5IdmNpMf{!UeB`r+>qplreZttT-tV&C0TR^2u5ssDcK&{%!D(gQhBY1MW$=L zn6|!n$=64G@i31Ze++EG{k=On&zr4jJ15irTrJutYwWXVoNnKr2YVp-tu#;hkleG= zS=lI=Vb-qZ1q1)4Gz%P&vBkchB-Cg%K6`x5R<77bQKO9zjNsh3^1}AD&8M{|2y%9q zbMLUfc*so`5}jKFuWjan`+9GtRoi-48^Hs<)b}qa3BJ>)>`AM!L0L%Eh- zBEAqA;k`DeV*Vcp$QQ4QoyEo37m*VHgO~*)%KY_=8+r*ytX?rxMNn^~OO+eG`r09VWL#zJSGW+P> zV2#Os&hOeGHK}XIWlp%Zwbtzw1_jRT*ATVb76X^4@H%$@_D^=2skgBbwWK_6* z+1aD`dM}q`bhF61D?H?QxChX@A2A0|C+@J+4$XXp?NGGdlS@)?Ojp{t*!4ISlcS)r?vlgM=}elDehDVV}h|0 z{VuB8R;H4174LIQ{mI^xle-KPKl(aT&MU(>NO5$?On9%yR1#k|nK~mh!QaORudS4Z zjP9BdqF(V`0Oa(kA<`NM5bPxa{9uYO4F;nJeA%rmbnl(C?h5<;>=LoVkD_d?H1ybN z1@b1Em6;hZ>6NVy9l5+knmBqHcW8Bry?3GEI@ezaDTR~cE3N(7ZawSg z)xD%N3RIlbai$phFd#fo&1fN49Src`I{eP=Iln z-b2rKRzU%2LdW(x$Z@=S$BqhYDr#A8=4YV}ak&=q(mv86VsYMu&F%ay&Feo|k}Qp4 zi+Vs0MuF<#U_gk%Uxm#_RG?_^@#m12-Hfks)&`=#i;MkU!S+Hde6X?V%>nutZ65b} z)arT=6<3Ge=(~u_*tiuB*)!B^L| zSs*{NrgnDa@hr&h8_tl4xuoVU=OwtyxTGPzoval#3W}|cjXRFvkR;D5hV7;%_B6G}k6g z8B%!R3YVa2zBJo1tB1`e<_aabOV127vwA9X5frhhsSVKc)T#znk2fucI@3$l9ED#) zGq_SCQ1?KcfyZTy9y2(}Y!83mB)nN6k)yED@R{1diVAbV*(GsPf|lG2cUZ)nL^+J* zInR#N$_UU)H1`$=F{*qhK}cCzi=cVyY4QB9rP$thX{cV4N~ZlHikoL!c0{2?M(AWd z9SfA~UdB}AB7c}eoDSCK7@=>4olaNtLp3LmFuUFIVyb56*SLssF|U>xjvU*7`Ozo? zdIr(q>*(Wm&ejt93}L{bE^awnGSweu`8oTO#i2{82cKKWHKwTmEh1&>AFsQnIM0jv zwtq3jSFeMV@%izCR;ySm7YAnz)&*!l^IlIw0M>h_rOvKP;vrTzkVeCkAF_=j?<1S` zdgnC~nW89jwU}({LSIVh+w(@Jd&;KJI(74Dl--NNCGNr_h@^*&_O{NTZ(A(C#~Puuc4njo}bd zzMj!r-daO^7JcN~2Hqdzy*6_WS8JDiSdxe>#Y3b$GX5-{DxgXTg3fzgu?2f~lcHX4(%y-$YqZpkd{h2zO*=;eM>>96qF2(C3 zn4(upvJfQjhLm*k4Gkonc$P58POlE(vJ$Gb{CnxJCDOZ{is4pidiQ`5nFS%vm$iKT z8r4s2h9M#!ebO#7QL-2ltg4DHU}-)m|N8mN$Vb4X;ix^Y(1BoL)awhRmqSNdH@yfHqdVfe^>jhFSoraHUawK?TZ@?T-!{z7vcV_)=O z3`%ywUHCu3QfmZX41elzBFv1a@t`+!$cBrBNx8pzK}7{qfVXWC$rb-}r`>?#>ogg@ zYV1Q-yp5;1V`CkXej7qP%4d`t4U%5QHm3`j=&w^WJ%jk+^}ViJ17?HZy1Xqb%kuhs zYEm$R+K%dV>Ps50P0WdA3yUOvOPy1~#Te~i#ppS~1p&`(w2aM#JC?L=y?qp?KJ zU6zk*KMfrRvtUY(3`|K0!&^2P!LZ#sBp7ECtY*SD(r6LT{QE&Fe-D1>o?(qs;7(j< zQrp4+pWCbM-p)*oM~lgvi30bSA}rllpghh!cf{%p+nftgbUO#joPsTV;zL5DrA3dwC< zJMNixTIXG%#KF*P+f)YGIJu1;>rtQ4y#zSN#om#;ZjG|xV7@!+4-TRmJ`uh9d8KuM zk9(XtW^Opq7X8Ro8D{)w*i)KP0!JNV(uweM7SBhK}m8GN%I&!@%yp>zuS z>3_v0wyi|`*<(m`{CJSn{@K%pjP__87)5Ps*E}dl| z;O#21=`nX+8+yX~ke&5CC%fQVucd95=7J3f;wz6|V1|Bo#takAwIx58y42LvbMm}- zizfbYR3A2IRjk=P6i#H*#O9dGNXk2^uCGxdX$ZsMsqft*{c6=T7z497Prn1;Is zO^g_FW5(EqK%dMf4I1(iQO*y}c-J2uyObM@ig-!EjDC>CFyE7KC_+d-R^$7<~HaaIcI?iJ= z=X&gHph|P-lGEQVfo=w>QbTt@c*-H_;rCr*WqQ zbi1f{$TdH!8|1eNnF}Qeo>Zqkop{$vnR+i@7L_BNhIu8mPK}uv%b6|Ab=)YB&`d=3 z3%=M0`?vufHVWZ-zCP29?g}V#^NvB{zRF?hw|s%fgxS%@3>NqTUhK*&x7hNxBPHbU z;qNFMC^ELJK8`2D$f(lnyFHF>3#NtM{*bJb+2Yq0?7j-z&dXUm(9KapZQHrWGkhOOukj6z zGpAJ|+=8VDAXxJ z4pB6z7=66ca28+AX4}pWRm>)0=rGPRZ>AK=@Fejz!Lz`8k}UZYV^hQk#i!>4w;}Fg zy}-re!K8;Wm5J-=S6$~22I^}Ybxx$v9#Lu=a)+{=M-zIz&DE>$RV#5i)5^ARt<+mi zHwK^5d5loEz9kT;u6)`(dhg|ai$gGHk2CSR455+@fGcACDa*=XhU+|?+uGs1p?E}DDrdKxeRqzh z+Wb&_GX;H*NKC%iXIHG}S7q!*2`U5P8V8=6@TlO&U(9nCJMbA%AK4brX}pQY3opBI zuQ`Nz-My)A6EWkx&E9q_%A7=>W(D^TkPROe=rcFJ1<^_%Z!|m*hb|8*^39D5^>nsK z%#|RczX`X6<7(R9|EATfm`wM64|*Ar{v+eqT7b zFC}!G_~MS4CnVUt#;=e&%E;5!W;#ePgE$h8F@G9$Sa-V$*~5qin<(>{fuctqZN{8p z*RO7+Zbhhjvar?@ayO z1WS!U4f+5ZqyJhnv@;xUg;8YPDEhVW_fs;yIU?eN1$6vsYL1Z1Cck`y3KTUO=Q$E5 z#BZ{wp0v&Mp3#`A;}Ot(_5nabxeW@ab<2 zM^b+%L)DklbGI!Gf-%Vt={8Z0EHZeFAhP;`B#S@(=Mi4rSPeBMt`or9I(GFlflbcI zFY9u2+r)f!w^Zd;OxTT9CTc0Omp0j(W})=WF24JmzO!Nn6ZvW-*buur^-bV%v=`4L z8lSq=#y5RiGLHf}N4SK5Gc{Z^WsuF@4-gFf8bHm&)MU#y#U!zhnr8hNHI(c`YF`U$ zx?#f6#0{$ZzGs$j7YdR41rv-2F`!yQ9agrzqx3$E^4)+r3f<&DP0DZ7TN|0A&6Nbj z@uqqcZ3d*Mwua3cRSFhz)Dfcn`#%o9wAzx3J-(TMnxMJkUPtQH?Sz zN4MN*{EJw!DR&Zl{??TE?!Pf|pQEXr`pO+-A|Y>lkG<{Y#yHDgxeVpsxWJCtYyO>Y z_H5}1o7&J#>C@9@?pu1eQ`O?&v*=T`^HwJ3wVBK!M{=7dB@s2o@OX3Drz|7B<)~Fl zAgD(3y-hePH>4?p^GpGku^S*7=Z_r~?s$fZwdc`y?8>T^yvCD3F^?O8^;(!)-}F9P zoY2Wdae#!ByVoc$V&dmo=FN>{NFZLHbgV=$!)^gNNr*MFh!dawuNEuVn1*Cqf#t4=Rcvdb$^qPsZgxy4zR#u_L zaSaiZ2rQ}F0 z)%5>O>J-1t(zah@<_Z_de&$8c8E@Yr_!(;sfk4ljlJ2+Ro;_%Eu1v|}RFnFEpqeGp zZBtbqOWp0cvK8MzcJ}eN{}&BEQx!xOp(p*ZM7b74%m*+0GR>nlb&FICPyr#MOi9*s z#)@`x52ZFBQF2IgqMUQ>Tggy!qEbFGhDQ=~XmRiUC2!?LbTtORM&RqarCD) z7WDO2#~|dp2^9Hel83I}&&Oj7%*ZXN3;`s2DmksdT`#5ZVn(8)fRQ7&bZZpz6W+@7 z0WnXvC6$R*0ZYcqia`1*d=2^Dp}@{2o7^7rH8-_pUl%Q+UnEpoqws<2v((`)a&`26 z+fyP);Z*`v%b!@y9?#DQ&jxBM*X5A#>@AblW=htBjZ`nb(@}73UpaiKtH>;UCEN`0 zibi5Hfns6KmWgBEzGEkPH;_k_GjIbgEOql*i>!H&$dV~4`yAah6*~3WaV&7{)ToG- zzG3I@DT<8Z-r~=lq_wSL*%=l|M3#I!NejMA2!BOV_yVhAu_Tid>Xc27U0Lw#Slcg@ z{DNaVytXq|E+7pIh^LZQ$B1@$4@zLQKj#{1^f&aWo?zC0b z!9;iDe1L1k^o+HC=4#l;$C)jaxJF9J<<#T`);7xfVU9sDp_<;PSoSY*2$_*v=X*8s zv(p!cYl~~4N?|xg+y~RF!c78n^F~^l0i^lM%$9!@^DHA|>4=jibryx4-^pOI^{|rP zja4AIuOK=;8*|drO%pJK{p8^oZuNc*f=%Sp7gvJPT>_*E9(*^#yr%%UHZ zJ$YwJr<}*bWU;zhYFzN=BOMZ7%bZg!3lej?I_>&ep_>=-nsl>^;TuV}SE9*p&Rtjk z8by-_&m-)8I85X^6Z$xj`kDh_y!;F#Pv`^JYa_9yN}6COYg*nQYZ^jH^i$P2eW zJR>-6*pA||pSinnz5A?vh89^IcJ+SC!bO6?G|^Btc?nZ^ADzjd17FtRZ)l@+qj&eZ zECIhOIsZ)DI^H^aYLpR^7+4PB#E||*ufAQHv{Z}R2a*TSGIG=uWnrdl%4;*T_n$ZE z@+}}MJELc@a!Y`XKFi|c_xz#KP_qhTZcbLrlvFzkusZjxQoFHM5kcxQ{72ZSi0L8hK6 zo5$hjWa$@{4jA%`vp`gjYEK=1vK(@oh00)gr$6`Sw}6rUUDwaDs(Svz!ez$xS-%{T zO79n0pXVAWTzv9GN<<;ntmTn5#elf#bjg&J2D^Zx9xAAuh|DdPAroI#me*GqiRkEr z z_0#8lDfi3DRY%>RrMsq#_F1%N=)=ZpNmsTfPuJ&aGfLKgL5L}>L>D0|?UvXdqc`na ztyYGwQ1>^zE%szg1%>Z^vw)+2eCJpTxom;`=D-n$dz;MKt6`*b455X_z0^K8CpfZN z#Z;N@CKK=f61;0bpRC=>H_XBe%*Ca}^@~y_EyhNNKvRd6gw2=rEt@`@9DC}lu7bZu z)ArclRQE`!^ZtltHOrj>?2FzAse8#g7~m z@+qjMTQl}*){C6CUXNpql^3t!+rxdf;PR(%`%%W~8M@VIY?CRMv>MB48q4-v({HkPLVVSA?v-EDLf?Xvuv#p2Vcz7uXpHAlw zR99-XWaBq>-zUq9kz6Yob~0YkHs^laMHMNyih`{t-NfMt{5u4MsQ@&qgITChl|K%5 z)bU5}Hb)Jku`N4s#ms%ZQ&gs3ZzGXT{YRE$*q`qm;jd=&U0=8BDM4?y#;zHOgy$9{N4D@c+ym+dsN*^wAAFv z!iZ^p2|ZiD2AIaJ1NN(pO&If1en(-rf7uRNxI{wYK0dcEM`fnh+!04O$oDhsh;qcb z!Yatft;3=O%1AWa_QOq{?I#I1sLiT{FRIai6}Q%EV>VuDToZxG@VXK6q}E?u;a8YL ztN+MXY784b=`ifz&OwPGMo;7N`?p_P>n>C*tmps{YGxBJQB;(%KK=9_Kcn2~W#tyI zek~UqS(JS;vAGua`kYeTQA&L6seDVL@gX+DtKyKVS+51rKGAj8A%}t;>lx9$rrqvL z)xV)M-I9AR_Pa;pCT(5$Oe^Ql|7I2}$(F+@T`R*3t0fT+Yc{GczJ({Qx>mhY{C3(Z z(2$nhESTm(-6uZpj=KvX&^Q%u3DnG-=GY4wYh`44+T+aT#FSV^{#4SA&Q$JfTMl5It$@iQ;=ImqE*wvSqzaISlv?l3(4B|mtj%`&;U6huX zGJ1k;P_66f@sU85hYt0D6_<^Jc%uyCJyXf?d&YB)qYfY0PZm}PR&+SBJusE^J$Yr~ zDl#h2wWs>H*hQYlsz`*%9U>0<2cdPv?-o4%hOIoC;+lMhPWhuYR^?OS)Q#P3$n8}_ zke>GqYu8R;&a~&2=h8*JZ)0CpWHYPOz}d=E%UqmTHF= zL06B8*m+g8sX%fqh>3N%jcjOTxAL~r-*Kinsj7&reygZ|1{u=l4L zw5#$ieZDSx;RQ&%Nw-_s72#=n3eij?eoBwxubX@*oYK9S`D8+2#ggbj0LFXgux>GX zYD&aTqBT*3CSMdt9z696iZaSY`2VYn1yYUYK<+>e*NR4%s&>4E+Xg~|V6SVS4Sz(Y({MMcFU{*9in?k&;au8Q!YfnsbOqm+Vj z9EaYmVQrGDa&mIYVu2zC`NtHFbDg4&vl@HEf2>{(t@q38f)(wCN*3uEg{+ugoSzNM zYJ3MyZQu*`p>W42B}(hNl6K3mC|kC!S)X~zJL64_FtBKyYyg0|M|h# zUdqV+{iD?QUtWIyZ{Mf{HjKo80|Yz+ll$F@qc}8t+0kMCp?m(#IpO^xubnsB{Y&7j z0%USlZ{PA7#y-vdpOccjBV$G|fV7zGJje4=SUWmi^JQENXY2VX52!DAvU>!1G#TLb@Nq#j85Kuhx>8t}0* z2^*hXe*gda@d0k}+nUjZ4B(ytu}}XwERgznfOq(3IPU$gPo@N1w3y|WI)MQA?0-$y zQy^dh#(3|*`7a}({?CV2k}3vXi2R_w+gwPp^#f5zexRA`x4*w%9oA?Cx)!9Qq@U`0 zr9wkPjX|VMI+&d5<+`S=p+A3aQ)`oitpMyx{Zf~cqmed+tr=?1k{$-UqSBOiwOvC|ZYJTKWr^R3x_CS$lcd694Of9PZjt7Jwq#)Zx zjHfq_nD6^#;8Wf4a$f6mPm`nVdE>?Y^euh6-dYpDcAssdwo3Bw*|#4_vsvU zV?bzg@0Z@UHS@8XAf@Pz_CJ%wb_3r5=Ms}$gGG8zVAVG8qEMgqL=`vIw-N#<@vXZ* zmN`7BlJht{2a305Uy^sH>g(SY0ivOlC4@3qWiCDZ zZ#H#pgVT^{z_;WsuStDB%V6|wjJ6?dy}PS!Sx+>V)=l>`?@K2_;PL;0LIY`N8PH8G zx%;I0bh|6_>lowjU%(ShPk+K{GwgmNA+rbn!U6bHRH93iAYLhS&RN^Zhg#T`zuIo^ zk2bnZJ3Ag~ssN~E5;8yyX#WRk$+9|LRaC8WnM@&yq|5e+S$3wZd z{TzfOBfA)Dmg6v%!C)Bs8j{XI$r{6vm?$&WWZx^YXJ0BI5z64$_gxV+_GL;L493{r zdz@1}&v~BD`#$gI{k-=d^Vfaf^SgiBbzR@gZuF!uy^9@B}Q- zP$ZKOe0L#fwghqD7U{fr_mBYaf3;JEHv+VXl*xMeK2VdTUoi>jdX==^YHOpEOi=U( z)c1kf1RP;|$}^IIRWyup@Na(=iHH>xI@{mh?*RBovw(aRGmvYE5fnvhP&;S>6o0qL z=U)O-M8=P}BCbI#M5;s0&lxOJ>cjw5g!!6%1W=Uu0-%*6Y~~!n8@&jHq7^g{P}?G< zT;#PRY zZg&d+$Y3;Yx~04MU{uVzaU zuT?Z1k^vdLkhy1IU>gdeEK_Wk=MJ2zA3^QA%S>vf{R z#mFt1DN6NMdplq5p#)-Aun*@mYEqdsCi|1ZqM*phg1Y1qNmwxYCpNoD3>Urq_4G}H zkEn9cOA9Ly{H3xYynWj5b6J|wqM+RX%WacuUZ!`xe&FskzV={EZRZ24-6~qp>w<>$ zalg-NOe~X~R5->t#%V)YlcT9ZM6Ud)ZZorF12EJrJCd|%49FRfRX_czgkCf;F*Z zn0Dieu6c3?DmF;vcDPIPE{zX0IRmqv=C=r+=WTzEA~$9s$(4ZszbKjF)X2cG>o;}9 zud|&+$sTFw3Fx`%n2>xp4{nT*VKrW(b>vW^^k?C7>sT?1euGd5M~HRaj&1 zc1NaC-xE37yjM3DBdLNVu(V^=33JPfnx|JUi}G+p#^4Ar^WYgQpiBs{ZD!r~JsMgcFE5t6T`G^^f+B@b7tfArwgOy%3Xf0uQiIejm zcSB;w(h5+2C>z^;9%GMJq$zJYoSG?L>_EKD$Yo~4uGxV;ByYJ@ z7xtQ~V)NKgm`P4bdRw81-Tgzc168uhs&OOi{Om$f?pkEhncbmOp=%o@5M9j5=R52J zm6AOyMU}?ol9&req1-iiX8_P`-83O&9M&y(QX7pzT)2IEm03!B0VD#7Mq9#%HsjIqcKS zV(5+x#GU2~9Lh5gSpvN%itVC8rLauSb%GgcHCywmym&Bx^Zyk2sXpz@9*YXKYm|)4 zki~~8T-3;zFn_Ix2anPIc}UOQk`s{4fIg1_sm{T5ty5wTwW>iGVVfCkm@qsMb%BR| zp}ki<$1Y~R1~4}0Z0^$zyQ}Z3*b9%%{Q`7Z$Y`|F5LeYl&|%Y?DMD!5v94WrSrAeg zT@&u3tMX+DXC+#LqF5fYyu?!%Z_E(1?INn_)w+-9-(vL3Xaa3`-*`<3nb556qh|ND zgSnRZd$QKNXQ6sWDhFSs0K1CO#B5I{ngZf%UVZ3S7CEQIHEffM*J3Pw{UlkSIpY-` z2^IL=`*7$4SHO(tt>S!JWn_CJ+VkSK%?$jaZCA>aT8p^o@L120q$igLdr;3 zMA)jB)X89b3T;nq+ZZ(ns6{4fL4;GHdg82-%wiUQWh*jNvQxBe!Dg`xS$7(%GUvOT z*KyOa4#Tmjh%|To4Cz={oMKl4WVajQ9&)x@ZFIWx4U~xY_ohE^N_ZgQgjSK~i}Svq z9Y+-ky?EkjV?971MI-Yv%XSrG&NGPQxeDdFWV`QtUr0gRwOUZmyFt)C$r9(Jacrnh zLMUB3nD1#6oP5ZBz(Fp-t17}=1Z_$@SBHSo+?w{hliDwerz0=MyP5>M_asZ8{K@Vx zea*8le9{BDKp}EX^ulVGJ~SU1n3FHwua5WErhiII#aE3Yq{Z4C6rx64nZ_8jd`@i{ zOh0&2vSx_j4isFY(N!Bonj9cqobB$scSdn9%V_2#d%v^DX_Wuw&)j zt;2MQRRCf`CSA1K%k8~gas@36;JynXhWWC>&F~fOGM)Q0a3q7iraFFc@|-!(uyQz6 zcjPB~)ta?_PP9LO)yHJF_!9E9pGXzQ*Y#e`&E}y4`ZI1|w97CXub~~GQY=foY>~XQ zIH0(DOa7r57_GV^Vl*1%m@B040#vjaN(CGU5Y8q}D6PF6=t6AuMuH@a)rpLb&~a0o zyylOd^yRQnVLo$-SbyoX<3^rmM4jOgeF@Lu+~p+o1=FTvb?|@!(c5Ao5-V;J9HS0! z_fxeEEnq`Kx$=E%$1cqN{BV=`jC#~G(U|24o_c31G~lg(k(kL{Zymb_AU?xny>q$2 zIa+$6V&YllX>T8s2Z<)HIbeug70cw5uM~L=fvC_pA-RJnIvB*Yt#CataGh4YZFAk+(SEoK_V*v$EB%9(H)e-x~iY^P~CT*Sdo?sIEMm3~C{+VI&N#6IM6?|#I z)e{*f{WQQe#<b*FyB=wt8o%I<;|6%XfQx$bSgQT#NA-?SydaF8Qhi zLpcZmrUFLZ;HUidgwV~w?bBXE_6Bb2LdzdeeaYb-gcoZj4aXWL23r>8;wCvhK-u6& z6|BrW=W_0X1QTo=;OdKXup?XUn>y#hiHwG-%5x`#aT(A3{{x$%U|~&De>&}+b&p(69zf2xTm#l3K|XCHHocG_&=Z4b+XF1t67`zImMhbNkoQvL5?F(J$C!$uF(S1cJoPHtgDPF~~S#ZKd<(u-nQ zh<6ft$KIDgHV~S@Q&4iB1{~=01+pHHX8iA|pRH0ur8T2YmShCf8Ra5?Gl4e%*h~Oi!ZKOi((9R_19*Uzw|0Xc z-WO_N>WlGO-DTd~dIYqhGFJZrtON=~RlOl^K9v<%sN;N<|BC5HB;x$!`lDH#^$Hje z!{bjc{2AnLtS=RuEh-HG1}dj?t3>=`&VP#m{6E4A|I@48Zr%iR1}FjPBO;(=xk%Bc z2JA3I`k!H!5gZQO-bB%#1Mrj^PZ0Z+pdw_&l*KQTcZn#*GQ(?jXq`X(VDgbu>wCMszL->A?|4HNi-#mz? zGlkYh#Md)NJet}DB&A9}ymYduasLrsS8+@QNKf(KBL5nwsMEJ7PjwZ)37uv3L`w#2$0)U)7ZweJ*eaoh zCc?+#7;>6c-xA0+;orYaITWDKp?!VuDtvY{wE^6S{#aP0Gxe#~aCQ^8Js2}JG32v) zz4G^O?Ye%SLU5#cIE#xmPgA@4f)9SD`%yvbJF<5Bv+q5kNl`DM-QE5;PuBp50rlE3 zW_~`yQL6XtB_|~)Xz2o=Hu3}78lj@#DgXfd9rP38`wPeYhnoN$i~sN@0OdH?&6{tf zRi57oProKNx9U;kpwxsl0pFu6eZ3iF_`ZlH_6nS=93H@X}UhwWx4}rA6H2YNI zTr2DQX$J^|X*w_QyHZH(d=0*vosZc52j)OI;9K z>tHs~xa>+Q%PeVwj=nKgdB0s`eYGW-{r0SqX9llyx5pH~W1{e5T6Q0&hkH1RcK8?> zmA)`G`F|cGl=$}32k&)Z3HJSOhDP3jfp9at;^U>W8I^9 z1Ct#uo*oi_vjO>ajiNSLWlr@XH-eWkGGyFMv?nTeT#0Fou4p%mQAKs2Nne50I~;)8 zf17Iy$Aben^Qzl?j8*TxGBgFftrM3K25lbACn>i1cycws1GY0V#IH2L{N1e@5A3bp z+zv+Hn2X>l5B&lhZlu+AZ`sMV0eD58F_+!}pOznoM)1e`Av#Kf0-F=O2rJ!$+3t5Q z(7la>kDW`M)sgTfSl`tqv6~1C=9&#|it8=r@14D1ql16{d0PhslhSuR{5roDzP$~y z&8$_kfd*(<)wy}X-j8tT&hLKS>44j%*Gu^+ikXvp$IVV3C4cc<)p4JwSvElXu9vT{ z$I~Ai$?u&*-|Bw zGr_2xz9FK6x9bg+AeX&W$Zq)uluUqxBqORccJXV^zv&8MNd=hm=6-uj+Se|n-y)Wg zHxlyTE@f_~lTzNcgTUyEO_laY(p2QKt{Fc5c4X_gzA2+>6_Mu*m|4O2_xU?Xa{kES z0m5o^Kd>0G&pMd^1DCnee?~t+De|#@V)(l8oz$AakzoJfa>B)aGD)`0SVh9uviQel zf;rwAUq_9|k1Ix)T~inBh}6qE+-~(QxX9a(3H)8Ftu@Cc=}GlbMoqMp z9`?PRceDa+pQ|ragv+4m- z>nVDLhh&dY;0T_U2VCC60v)uJLaNkq)DQlWB4M6|qwEnITS+aCAX3qb)^D~n(DolD zWdeWI-dFp*V~NYcuWzqj>#Dy$*A6i0di`C(RdKfLT1Eh8Bt ze*T^jGT1$C5#W$vA4X4OYicAWH@?udWHR0f3r70{k@h9bN%;<}Ew$h@KH8VhwoH;WP*Mr`89WD0TNpI>}c^R?QE#j zUKRFC_?p^lf#SgF2?Yo3kFzW34XzYs&PYIO)0v)`{n&tMgYU^su8{eZ5pLxv^E8=CW;SiR39d;rLnYs zCe!kxFEhm}B<{Fmr~_Ql8CLeP7+J!{wc{yu&25OeZ8kZ=5! zg!%J%aU+1=@vqI$K%X+XfE4GC?a_xq1p;I$|AU(X_M~<3vH$R|6ul_g-P+FLaK>^! zv4HhAhREnwq0Q1@b@lzFAtkLZIN^UWw%ls|fh1z09Ybnd8px=HIP=oXZriVp_>?=Z z=vA4O!nelD0r@5%auv(C+RB8;YD15YuVrXFD9UMQ2+n{ literal 0 HcmV?d00001 diff --git a/docs/source/developers/img/conbench_ui.png b/docs/source/developers/img/conbench_ui.png new file mode 100644 index 0000000000000000000000000000000000000000..2f72df024b8011a2d947d3208da938804fba3449 GIT binary patch literal 70336 zcmeFZcTiJZ*Eg(mk&@6m0i-A(AV@C(LX#$n^dcZZR60lp=|P%E@6u5OM5uk0OuUrU9Qn1T4pl`Eubs;~!F zuHYfAT*0|PL;yVF#twRM<;troYA|_S53`L%_cx~cek&@THQkqO6K<#4Nj2U6U%GGU zkFQrfFDXfh)tgATtHxm)D8itYOWzkM0%>9TnvUyCfJmnlz7}``p&EGg22LzO^$9!Z z3K)SDahb}(y)>e&F?Vksy`+C=jZzalj+a#7-W3^xmZTanCNAuwCA11qlxFX7wE$ht7&lo zeF9V9ZFofT-p5;->PfdXT~_-0_zXYa*c>b85KGK_#uMzlF;-sbb-3m__cb7t(n570 zmP5rEeKPrQ3G&a~Jrt~#@jiONGHjV9YIkqc%-fDZ@G(@ob;uIqlESfSKbEl1-4 zO5Jw4&dqkDNXOH}>}b{)I%f$VIo)5u4vD23DxaeK84V&?;$X^UAb-8AS$5hVb76=# zUgKas(d_SURQ2o;gSg{knke`{=5&X^oAPHNx%cdErjs&?J`sAhyU<<|^5~_0xZ|kd zyxuyQ*pi3ZfYf$F-0<-aV~6kYf4^Z5>EhV&tiRl}a}WP&L7M%ui|Yv3S0UE`i?+~w zVVizeNk4m$LD4}P^znAq(Z;xoT6NERe39U`F!I9V8E=6MSzq+qcn3`!F#b^sA?wd^ z1#;;f=tQHpJ9^W>KP?8gnVeZNp2cUU#kGx8>`9GI*OO@FP=!tkq3DDJK|d9PS8w%Q z8T}O@mY$5tR!CFtH6x^EHj9jxh;|t7z{a(5?y2m=y_yFeOdP(QF{}fy2B+J2>~IPT zZmsmEapeDuvvSC;kT604Qg^mhKm2PbclZsP;$Al!l*clMsHd;{l99P%dAsS%zHYaJ zarkI+a+0|?OsnPe9vb@VDyU9D=f$l|fiv>WTsjsEB>|e)+IuC#9qyacc}^g1{~L|2z$Ram~U6I&T_`3h<&LlmTq zK0TIa2|BKJx^3m|N`MqNdQf`f>l^NaD_qTjabb5J2j2Uc(0o4N)L)@}XQ|^BZ)H&Q zxl3k(!F{Tz}YH}TjRZhAwU z$O7P~Jhjys@9nRQoaQdELa=eMQHEXgO3z~n^oz*{3p#M1<@VLT^tO1 z-odjn=x8qYA{1lu68yq(bb~?U>3tgL1mg;g&0zc&lhy||IN()<;>Xf4T3_?XFLQVy zb8qxvX-!Fnph57Y$H=02Y*JIisdVpBt8o=-XJxiS;OFo79)Mn0YRVsOPKwzuqIj+>^_T!WpPPN)D)ERP)c%5rA}r^-&VuE8YI^uXPCV-1Gy|^?(*qxgqa8S11gXCHcrd!wN5#84U5FOnw-dS=ktVD^;bgh_q%!lw@LU7ooAoj5| ziD^#h`$#3FVTjJM0giGe&Mi88$DduX5-e~EuxLUE*oNmLLII&p^q7rK=4_*)WNRnp zsgkS#%_!sO)7QVpr7MBl>PfY^g>)1YNzB;M6lo=tC@<8nLSoU-yOBMXRYRwBKJesP ztA#XEnyD}xE=8k9{7CpTZ^rcoD1-%y1_cZ4(omM^5-1(FE;|h#aCVo=;> zLaapvd8Ai00#sqY9}M7*=E$g&_!f^$r^8Cy;XS{W$kCJ~3Ug6Krq=F-`(K?}tx}9$ z*hk5=J}%2#7(+im#X^IBwEUzH30~?+G8ZXTS%+#vo1u(1qt9}CL5M-6bDE zpcTlvey?{I5^Y_6hJ~76_WI^&{(F1O9nC(y;k%B#Cl-9ht75`Wx-WaEl-LvVOzoYB zUgPL&xHFyIbh!-&g^o>ScBO|~Qe8>xH@CdOzykUy}nrlhnT zizBiK7nkVRxH0vZ4Rm*kgdNg?kVUXQ30%@`hwU-8M?@8YV?i&+D#KfI=XmBEW1tji ziu;G<3ltU4fU1UhR~>>1H735B7{GQZxENjw_CN>_Gb;utUVAc5?+tR}0>#S4Hz&IV zhe%y&rQG36x~9HuSz(9I{-+ME+6i7-pEIQKay$`CR>F0$cZ}< zrNi}$+~-@y{=ENWwcWTJ`fBQx74Rz_@&(1`2!R}{yz7{aQ6#~SCFzkgpjO_Z915nF zxRDx_0D8qUcE7$0!cLsVuXE)jMZyI>6Ukr^Y(<4<@6_f5$5C1?rbe7(b)c8i9G1n2 zghk&M1lsm8aYP{=@gxLy3LPorv?8wO3m!q%R3YqSaK$0<`v*H=#wZmAo|t6LC6d2) z-6zQ9hq*Z3_n*Kv^ zsaGiFPu|j1WDepc&naXo;>wTCi7y2*ik?K5CGS?=CbnLk<7xMiR4hj<9sDs4=SK|PvS|FK$giyp$|GH zBu7e5ybL^7>AUIEq;nJ5D0aui3N)4GkyN5T*0L2sF(A*e+Hg(_B3>e)6PsnSK+;xn z;jkhiB2+r&WFp%WHf<+;BiYkkI%7$OvbNYJZ#$`Pep^LIgmW8rv0Jk@D8%h=cK{{Mz& z|3ftQKYqE(cL0RF{WCrY1b{nzkH&4({<&*dYEtpr0^8Jo|NR&#fad-8??3*I%Yz^@2_CRLk(OwBBxJvB+UwfMiQ+eCTr>EN@$7tgA0UxTgmXQ%^M zU7OYa-070{AI!y`lm)!N{#kh{K0Et;I*LCX3)t2Kcn%!?i~@n>iA2GF+HV9x)haOY zVbu2cAK%P>b)uam zo1v8{J-%#`3!PB4ick9|giRQz5NjkjAcf9}s^`O$+S0Kd(adQ8m`=*{+52{IH0e-i z)fsK-92+Wv<@i?flg#chioU4*xvP(BJJI0jqP5taz&n{v`wu0RsTfcc4MF-G45=8B z)GZ|%qufSKlk6LIJFrAp-t)3X(P^LkJ}mubJCG@pg0HLY@rz>bdy`+;YT!)#_IKwee)obIWkb533_Bj4N4 zohal{5X0O8&p!u8rYIs5wPg-<*E1!{%g8F+K?u;Z8PAoBA2md5AAC;c!#q>IH~F3! z#n3K_dmTJor>Ig4BeSlNdb!cn*<&9;g`Y!f=E2q((OhC&1JC}z_am4!dULv7tPyj0 zcIq(BY|CaJturcpQvG85Eg6gS7?;dR2*5+GcsQv~#UAHCk$k+-I%LScNZw00KO*NRJopMd0* zV^}h_0J3GI%)HrkyYYCLkm!uA{Ug(q^ueH_q4a!T8%K=vTbmoxKm^#1eX=m*yA^S# zz4ltnlKsO%ElnUiHEL%4yz3UDrJR$A^z+2( zkSk><`)?BeK=ju-%H03raCCCBLX7lag0Bg3{76ULi^&*4r33kZO9Wx{*^#HY1G32UzdV(GUbI2x6htqE;5*Nc z6t$m9R-%I+LqJjV!d%z+h%^x#y&_ne-3u6*z1|y020DOur%3}nag=(}_2!55^ZVxv z21KwEn^4M2u>V*R!s-NA0$xX9S(<60RR!<;F_#i7c+8hJZJWt=gF(uEU>}eRmP$`Q zO0%!KIqkJxg5}XAVtN@~(`)1*>bu8caZd)gIW>{eG8f#PTFsdMSVfAVhk1@GW-#bo(AO5)wh78R3SYym4T%(pxDj`y+_zqs8c}zGEkI9B z&~!Q1=e1MrJ8&68z^l?6Mcb6O6_p1mmEp)wa& z{3`!OUc?d(ws0o9dR_iEx#h&MwAN9k3|&81Rw};R+8r_E22S4AUl8tk83CI2eSlHT3%bYjzfh5##Ed~2)=Sb$Rc~3 zTA?2E#kNFu>)p5ew=44L>Grkw=+klDK z2m-=1b0gGS#~i|ZLy5?1_|97Sy~#o*5w!Azu>hTYgqDXW_}sb<+g}-I{*{e%2xi4g zH>`kH6m@aRx|6(7ib?UEh5ZNp5TFHnl8V{r7TKA%86Lb882P$yyOTSRzne>2fXn(h zN@?=bb1u9baN9%l>TLc&j9hV)|H;`wdmH&JQ1}4^NZeOi0_g?$C>4q1p={eqY(a!q zD;z;CrC*j)&+&1aPC`HcjxETb!DpZP?0P)_(dhGm8UZre>7mj)61a6+Nb$-1eyY;$ zYr!!oA`2iDNl35+L*N2Ji(!2^ySRMxR{oqUFHSZ!AYKdM+|DO_ac#^!9qjg~c4whI zO}~?7Ac{D>|7eXU<7{nji6e-lFItyjZ)|^c0Q0l_plzGvWB9gzg+F>FD_Yb0Wz9je z_K$4LMFR971c=GD9H=gLmw@n-*WSMz`Ji+b%lFe&dMy&~BcKSr-hyz$yBqs$pM{Vd zM_iV`FF4>%%M>FWM+JyNp2e^YlD2<7^deeSkw~2H%yJp*#HQhUdAj1snhngDP0O`d zM04Zct!&%K}NTut%HXf;a4H8wq=v+&sw2jEs0DAbXNq!cwGjry0H-SeM=qouXdL`x2N z;eQ6)nNxn&4Rd+nIok)2wfY-n$5e-41Q`$+BG-V(z;gt1vXhVYO zkd!Q-T718-B0QqIwTd7MI4L8HBNI~OND_N9nCWpiO)Keqh4e=hZkUI`rLq?&4nZi# z$HAe2>YRarxtu7(wj@+i*?kelY%>susqf);&o3rA5X*2=Ami^*R2e+zxKxf6XpPVw z-NzTk3eFn-q?KsgkpfLSu@?*^ieJ6UVkRQ)}4{ItuFswn)z}gnZ zKtP?b&JIZt^^u885$nCvsyQTMPGNOKQW0W+6Z#dY!H5L0Kx*YH<)2vSk@d7HJYEp< zzUWOB;&4zkC6R*Omq)gc4jZ4!JR2!Wzby-NX9_3R=R?E|FFX7ce_m|~6)6vnZct(p zXgaY>1u9i08+9j9(EEyBV#^dJ3nvl}2$q^%iM?695f}W9#`6Z#xeiTv7-eS)eY+D% z(eUPP?4s%rD9dNgzm*`2cq!ITk|V(C1r1QmBh-PcT?GjOImi6y$9%xoAV_r83X2#U zBw>0l%}{hV<)lA4iA{P6q%uOfx?qy;HrOuxDMiHJNk1}|4n8EXH6cazGxlWHoT}21 z)m(%Z-KLqp_#1*nkzK4jD%M+~t!z-`KaTaLz{|0ira?5N$|I9{>8`ut;)QTw^KmXnhgF0!)fLnv$=|Dvh2t{)}F`-!Bxmht7_Nv@sUgc&Z-=Cq68by!-b zfR@Agvj<*_SzP9GSQ zy~_d!e$vkR>`I>Eq;F%=<`k9&&3JB&>@Sf?dotax)F?Q|C5Y{%pXvofo_0$8ftJl zMCGeLr$jVej;?+Hu5RbE=z!YxTW4idhoIMZ;k~d2|2^JbM2X4e5{Fn3+019R{lgLD z0ptU=Ly7qtCa54Lsh4~uMpd`?&6^rV`yh3n)ZE1I*^`pP4~YLxn-{19S-^p!ZwF=K&ppKHlZRXmBryncpsgH#*78 z&r>?RCGf*-?y(o3Zg;+!g&XbC)F|?{a6$3#=eIcOeWXx#LnEPJr<5e?>WgJjKI~$g z4)a?An)x@&y(w{KKpOnefkzCZz7Jf&i3;I$1x6@qCE>9%Ct{1rtF)i4t9;xND4icK zKo>uKj7P++VP#z+yei@P>%LKghtmw51Argx@DkLouFhqTHTz=@0BC1^;7B45M<1+W ztqQhhN1L?aV(DR6$A%1`J=y|x8CziFZGjefwNGrex0S-PgMa@151{U8KOZeE;lU{} zC}+>|2Ye7l7u3V2-MnQ#U1j%sg{d!o143POKA~K46V2HN0Yx}KG{{8b=#WEaYhW0{dGA?pUx+_g?CIb;pGzQJIB@8)7094}k@?IEE?o;=Z-Xi9z7KEY@g@15{>QE02JRP z{Oa-E&u^=MKSEt>%26@(?sL7D5l|7wxvwV6L~aQ{?rsZR+6SOxgX)!qV}N;Zk&pq< z<|<$}c_G~Pn=;EY`sMu@8ELrFThKg$Gc;I71e-p4Jx&ZPK(hcGoeZ|wUP*Z$Xx=*# zkB9eZ5S9f!0AIgXK30q%Bw)W5cnybmrJJ3sXJn_-p>?*}&e)Gbhf4UQXYUq_3Z+Yh z01CVhks#j^`^O^~^TI^AmCExIYxfiU1&)$nwTV+^a=HF_+Ieo6FbbCJKJBJY<9niM zwj=DmH6`sG)?C{rr=S(jwktHra|Bvos;Js(##}OyJH31aVZRN(OT8$7rp7WUtC7yl zsh2WVctvUubSh3PPHa~`z^o9`-F>D@Lx;tmu9xSh(r5WRAT|R?4#X00OUZajdriucg*$5#@V4opemKASV3OF~+zl8bp4BMV9STFf^F1 zt(ScPo=7I_0kTizGjtvbik!_$vv$f0690`Ue|++!yz}_$s|I~?@5?#S)}IC&{!|st z$xCfcdx0md4}@Fsy8~Q^q>J^-`YS?+V#01-0!(PuMVua>P#;JIbBSJfqgM+n{JR0H z2B-D~ejfLIRg08W0U~S(%c?hIgQ}+D6lIAF{QcE6hd+-71je42!$b!idjiZ<(?rcuh2qe*ZZ_JUk;9U*!bp!c{(|GTF1w2uo zU$sggNfJ%USI8=YK(D$Pg_H2v;k#4b+l_`M3c+lA=NWW@XTuH7015H5hr zy$4m6@}D)lc%lJoek+Rx%0lr()0KdyJ0DjV6d)7D045UwFd2A{Nr5PhmE*P}A6^bX zj4dyjvT4l`$m3ZAS)06=i!^9`!P2@ozYAF1tXr;G1gG2$RuO0W3PCwWYG$2gIfBg< z4iJv7)qBTYrJRUsog;1tNrgpaMtJkEQML0yf~ny@AjCq$T<}Q{mp60y7vU{cVSrTd zyFl|1u?6c8#FJwliX!BGN+c$ST%BN(J|orXU~>hF4!iBrsvm<8 z;^ah5wE_z1dG$P?dDh)1*S-4!0-BV+gPX+c<=#9sy6|9sHEg&hZ+k$L@I}(#xO$ft zF>HD|PZEz^>^;qwJs>erPW=R{CQ}E!0()bCIDHrNi-MgDhiJ=6o7In_01^+;z5nv7 z1bmySof?n*(uRhVoOywpC9&EmeMxWg8Z0j;-*!G`N7P{^J;|Fb88Wf6>fS2f#V&(T z`{(vvFSl% zk)QdIqy;I?nS`AH%a*1@3jg$pmRx~kM+!+Is!bM9#FGhNM=6r_OC2w$#IT~^P@0@SF`;}TQrtke4>xLO zoG-10wdJ}%!yB<7zuk7)eGL{kbqK(F<{ja&FW6u>Tc3P0x{IUZV;y;tYzy<~C1CeG z;bxfv8r$1t+wcKyly-_^!wGf8SQ-ycsCI@^rby>UZj=e195xzbpM@tOm#m*ObVV8} zS<6|MN(VfB3v^Wi(10Q`$2c1~7G(6^Xc-VDk&kIey;mWnf{cXni$HzrWi(tQ%*M;c zE$2*pF0>f3WO$-tvRVp-rKhVY`H9#?AS5nK$=UH|UnjglW z)-wPc031o3W3UmUOM@U12xgklB)ZotzOZ?H_l+1_XFk*!!h{-_q=P?+73(hI10@_& zwEF8nUJLZKqvV=C>-CM8|Nj2{@3z&NlML53GrN#k)KXb^k)>i_pJsvqC?(Mrs?W)fXYWA8iaywAq85AFuhvrEZz1%GgeZO z0RWz)i~TQN-2t`S+L`G|`#SawA@U2=Rdz{HWuNih*h;0c=Z%V*X_{ z?7zedzuO0Sb|s+bcA75c*#pT+Yi)IL;3lFX|L~4QZKI!0n7nl!12!FeKnTm?s}q#M zKi<$mat)4}xM*HR@U(3(!|B;{1u4S~m4m|(t+*kd0J1@@v&aj^m<%8I7VF2hskOgH2?L(qvAcrX69vwn8*(qCaiX$!GKDDzqHI?zNJ zcSn~gI5NqT#fBL^{25XiYCOSwc0-3Ifu&U^G+m_@2mTc3ZTe}af0uGsYINd3$SUjc z3_Ug^`v4X>13LC<&j~sOJ-<2dWPbju6A!UWys3{JZ=x0Tg%jTq=oU*SYiC47Sm0O< z_qh_2P+B3AULGeZ(s+t$8Vt{C=J`Oi_}V7~5xNuPK{>C?l)fUcYVnOuIJi4aiSjmP zk;XmVU1Tl?LIC#9|ICxc%>?HG7c9KN4)+Kp`hCOL57X4RJFnwg8-$;vGWN_RkB`JYrnG&w?ZZDJ%=dfo z-8l$JX*b{3@1??~yxVNVFz0V`7G2IFF!n^h1e@bQGlTvP+HVHkda{4woiW?IN5yn( z0=P18Z5xW(a(Z%>r+mHH>j$xh~SEf8l29Sl7T5ARdcI0?+?` z$Pk(L`2y`>nmd4SsexpT`|n@y)O+>H%rda@&N1ZKz28W>5$7LSI93wS?KCgVYwS%o zg%M3ZcyNZox2|D{fmQ0iPMwi()B2bB?lYd;fQ(ghh|348S1m>{`;QmG$D}tvC5+;7 z{OGMWqB!?$Dv4of$9Zwwz&Bu^|MwEF=%~C&`GK7d15A>DHGJZ;wK~=s?gfyP`*iOs z%8tx{h8RmKvi#>QI~4#lz#>LIVj<&SLFBTk1Je5%Si?92&@)T=oa}H?BSOC`LDrU8 zVCDef08AGH!@uwU8kr5$_WW|IPIxv?VseIzx9D|chsHxa7Vdkm$Hw`-!t`YqZN!B{swX2LZJC;^KIAyImcKNm%Mt zpb7-2R@e>~R9d#bt^~x~N&tkN)2qw6&}V)AY4OQ@d**z+2P;AHn>C1U&NNCzK+|V| z?G}C&U(r9stKHUr_(Zx8k}-ApjnCO1u6>t?)HnCNyE)s8VbA{KYhnn?Rf#FUPWbUs znju7Tp5=G+>150)4;1nOnQ++^Fe(<-Z`JYs-Eh%u&MWS+tYK9D(#~tAg|H$Gi+`tq z=4CZIjO89+x$Id12V!$~gQx_YNHat7F2H-Z8B!^{SUAQdmwvO=bhd?sH!BzbBIo?s zo`ZTdsBN=6D04g&2gt{R@DO><(kFmWgrj!~&;ZQtIzjE5gzXu_PJX_&os(SVi@#W~Q}8&GbuS1|drr&9h`M zvX>12LU`rr8xzmVG_TX%?p@)@0su@Q&}(CZa9NOmoFyHrt~QWF?Gr(JsjyKjt38Hn zL0jW-CML3hNSP%?n5AC+<66$Un(Z)KFXvJQy&;1Pk6S(YHhgzq$X{EKaX|7TH_(W^ zY6zt}Scnj)w!b0cFw-EtXww!->iH6+-ZS=)M>GXKK2}mQlbVDm?*t8NfdkkzX6{Tm zwvn!4aeNp#YriFkox%Ykjs*kKR<;1cBGK<$_dC9S?zO6$G+7>D$&#cF&6}VLXP|%^ z2B>zYt8$ZXt7dI@R)o|%TR6z98Rqt$wY4Qb;|=CD1uV^EZ$=F+tKfa9=W^tM^KD@( zoYvk)8{fRImHUwBtW14z@uUH(4}lT@NcuJ@fsLRqMVLE7&Rg_KLg?BX*KK-)Zw4U~jhM`rUga5Bl5q3^w2lA^AWV zx{$ihk4YvTX{DdsgD{*tlKEkfbdqu`mnX~$Ku<~_id*zMK%SPlPmiuy(tuk*m`*3r zOnCL+)^z=T2Wt1#oU{T+eQ>Oh2;-9y=_sK+_TpYtmSwt=+3zS}h%vD2xXknUqzn|k zqT+OE&zzo-vmj+|qmI(72FxYCXcA&Eq2Jf-64`>gy7+Kq4FYQX2-Q^qrU8%E^XD7wef8B?02t^Fr`!*F0*D9R2UzD0ko`4*W#7YEwfZ6r z(E)J%KAa#@TaGJk%8GvNFKoA|5cW;*d+7NZ!s_?M{U;XcrdsPT$l8mDxP|Qw3It7F zjJ6Mos}XhgKNrH)m3^=eq(dxM2x22HF>92>7Lsi-qa*!-*u>;bCxRjCLKd8%k!ulB zRETG6*I~u_`3F56AVj*nuH3&LgxB*8!0A3i8fq_hHy-Mq6b}uq3`8LhJ++}vK%_4^BbN#0|^aj}h9ByBxj1>QKdMdGe=s~gC>Q15z zzjZcrgnEq~q-mBaE3vO8`RS#K5&B|&^}7R^Mw#F9`cF}|vH%ccIw0j??~QfhlV)~&b~%^XH-F>d00iJs4eVw}2GJJ=*lrSN z?55-2he35`%aVb#kr0z@Z8G&JniW$ko2Khuh1{5{%Q&P&)Kw7b6hlRT9xUKp-yaPV zfjGF*izDGU2fajIyr$;8Jx-DXLWJG|QP)-i9EtyA)p48acCEx}77JsV-?YDp=K0!C zx$VR`rzgP!U;Lb*XC4nV*1pQIUaE}akE-Bdw}2uoVE|v%@qskcDt`h})}@_fg%!Z& zGyjeE_qYfJK#%C_`9{V9Gx)Oeeh**?3j5N;3IW-K7o&V*4WT`dp{cmii@B$JGTD88 zm0clZarQ(FJ?(&DFY$1Ah+f!tu|4@pJ51grA$V~z5LP)~)=U^Q$rRR%?tkZcC>O1L zcFe3*sFk6?Xw+(mwoyc(~8(7fieYqDCo_7d-5YF4UGqL-O%wYazFK-$N1dF1#7fElvl9-!G~dccXRFK9_OZ zX!*YtwVzD!+jekx_P(dt)>uC!tI|A)-w%Djnx$18kC_ir8RqvbEMWEZzMocCcRz_1 z`EJkYWXPSca5XL|Ul>?nZHzAqNByL^_)B!nXBiVi-f_;>@6`MF?TQf=mJFL|7s|>c z^_QBt%=5yRyl(xl4p~ge!YHB-4o6YpF6MayGmL|NGX2>sn6#ty-EjZ7{!Np?2eLL3 z@xI>iWrknq{;pfeS9Dm|h4n}NxCjNko&ol2W4(9>0LimF+sn0vursdClM2&P)xS7% z$S=O!-mYH4oDaKx-@dS@Tf>}IRIgmNxug8g4q5ZK%}kPCzL-tjsg2zxof%oBb5gL73yL{q4O|hfDe9gSc(EbU+k)G*RiK==Eh5DY6f=n-!P8N+Xi@ zJKYN{Fz+4KFszuHeHV+~lj&KfVeSQmeg_6djXz5&orXIq*Jr#@Gjf=U%40%*Pw<)F z&c(q*)}{YyQf@+(p-@LQLOsrnb+sj3r*X3)F=wGmyuOid|BpDQ^LEp zjX5b>{eZcU(=YAu$7~)f8na;Wu}1NXVVvKtJ1#2AFV{S+4t>Nrn{qjKjJbL74M5KZ zN&XWLl7d(=5=0%Q)Abt9y;9Fg&W1l?d~DL{Ql}pz#hiDV=Utnv0;&jng7tT$+7@O+Ad3qyJQwEKXJKWm#|(Q zlA&03Z{GnuYS&*?M}In-Z`adNNXjTybMx#TwyU-){qO1jda@EiYu1J91z1KRLjXjQ zC*iW3w>n%{h^>=YqaNV>awD?u&`eKfVU%P)6cBPeDRS*>BrnLqtYsPd#n`*B$l-=Y z-FsN5IWQl|Dkc#{Cwe=I&QB;;Sxc2%PDJo<-QE4tRF*tfMN?H6-rH-NdU^6wB!9^u z#Wt6#px&WQX4?7i(6%GF+rFSWSl9&}DN@gwE&*6aB{)6mJ0doyucEzn^ReFc8XSaE3Co%+Bg;PU?ykhjH)Z|n zIR@O{M@+{DDigg}rJXgT+$1EY$F`#nie*Rh0yu7sF8%7{HX(9Y{wM->UGmDgL?1N@ zO}lTmZT?sazV`5ToRN~|a z`JwnT-PF659(9+j{;_ZpM(iOA0pkw~@(e3h^2Ucdt4dhcoc6wo*!6O(qRlbBslras zw0&B-c=$l_2!+7X)LPv|IX1X$U(o<_Rp&~g+q*z7&0W{B!Dv^*0=k=ZUP4{ zpiW(A_fJ={@;8Cr#wOEG()YA|#jc;w=K6fP)M%o%oJ`is?V0ayXA(}`Pkjbc3lbgs zGoSa^@Y@<(qMxsRS;^8izdRBH-qG&ZUzu0d`$bz$Q$WBZS+qKS@^nDbUfai81<$rE$b=Y8QYAAlX>?SRUP% z1e~7{ljRa3owUw%@-e@`%*$4OqRvgd-p%lZ_fp-p{wk!3=@!P(UlMcEdi+P9lH2{# zESmmIxh$=!`0oew!l+JuyDUtP{q0iqR=&}J5HD%;Zn%gYI?@%bu~gST*Q}7DABg5P zuW%N^_Bf;pp2F(jSmm4vZ{4b`I`xgs2eP+OL`;&dMV%Aefx`gSDW2J>ld){R5&Us0 zFTvV%{YQy9uYT#qk1OdS9|7lu0PdQ76V|E?cnpS1d=eyHZWxq~?(J?t$7A(i?a!L* zM_J-&&&&YQpu*U*V;g-obu3!K9Jk~)k!Q>XA9tKpoY;Z0g zp=o1L-tJrh3HJp0*@%SLR=IN%snVy04g3Eou9zFQBM_Mwf+vaOoZofFS%rftuV;MJR1 z>ygg>CBh7ieyLG2D7m%ON~epFMZbHJ^}HQBNtGiUGQmR`2UKnE-I*PmE#%&N#W=0U z+Uyy{NRJAqdre8LQ%CE@05tEr-5j2Z(pb`80C-37FFj%i^H8sNI+W83X!>=&lw|}n z#9-FVRlvEbIh)uexAX~MrA%(eYrD_fa}C=fd%0*)16ZKX^@44GVNnY}F!eW&PrZ#g zsGGDcrfol`8CFDlNOfLBOByzMxiL1QgyL5K^}*#-Cpuw{We4zr{645dPJeC4sXqRV zO4AhO@qIrrh5AzNsfdMB^FGFhYccZ5yM8}1cbvblm0($)hXopHxP)XH+Q2CZN-Mdy zJr~chr;w<{apeOZfr4;K?gE=kPp%}w<1J@GR{w^(+aEU{mR9<1B?W_nk^nQi6e)|e zI4P^F^nhQ{ky)gF>#ff3EO`NF+X7j;T!m>TyYgM(&CqHmz?@>{Cijtc!xq^XKQ6cA z@?Bwk_na-g^hnt1yse5P_~FZk{CQ!|QylAr&t9+G@SFgQ2u<#8chm2UJcO{D_C~Xn zjNmrcztuyviibE`EkAP4%DX$Ze9>r#DssUsf8uoS%=Z-b;OL52*Cfm=$)j#K2~F znqNf?=iY!~;PXglParJozX$xfh3r%mK-Q0$zuPF4*&)XsPlG+6qYgMs#hH0u%+Cb_ z$s+0U5I5_u<1TTfVQ+44kvaAx(8^1CjvLE+9^V$*eLr+S4&!sX*^-gR1$#Wqqv;2HMT59X*MWx%g9cY2bt@~mF z;9$4*=H3-%U)EkO&LA6kkt~;Y-X#lKC_Kk-PYBIrHcJnH+IA>C^GFA2KG>cGU(5xO z7bKZ^-K_E53MZl4YCN8C1vV+Z}c-nH5<^QU%#9KVStzmylRs@85D!?Lrh(6gM`wunHZw%F2aKtWc3 zELgY!Zs`JJ`!Ag2A8xOF)mp=~1c_~8ee%yszi>q{ifz3mf;GI)x6<&=vAgSS%#I-` zs_o!Ox{6C+LKA>+MZOCS_y(Lk^2p5_QMnK%?U;p-ENJU_9M)pZWc3&Yc&w&u>rD=| z8}h{VdR9C+n%O`@IxA>8fMl2Td%vD1rbyqPB_~H748Fc_suO(l_4Fi5o;M?Z2BpIu zQ<+A$Gdy2UfzMuQu@=^xLHfp0kNU9BE#%4&0yu4C-cA~a#;w!kCW=1CH=RSE@=@@e zE$gY`h)wYcyQ!Y1xI?!;*qu#LrdX(*IOoZZmSoT!MS}%=e0Ka%a@?KI_5Fym zD+-hgI16NM57w&!z0t0{;;=C`bJKD4u!A|)fN+7X*fK!V_|Bx|ja`3yW|_SiQexMC znPz-bGGzxj`74dFVXBT|+V}&>EZKaeTGuuV^|+=`TxyhdSJ^0HOE;=H7K~O{wCfcP z6kl!hZ}7Zl0T!f=b^s!xSoPFLWn`^P?|3?&*R%M$Z{s<^RNnn#Fn(=p?x4F~SyhMe z%YoTv_&J|vpWQ?kw>#*c$6q{Jtl4Oy2`JjA@oD0a+*gtGcr4`j__0ELksf5BsVgm` zNdKJU$}Xtj;n$#KGMucIC23sRsZR$V8(Y-e_l%CrWeM33N`9*v%6?x&e<5{PYO{s) zj?4=xPS%v%d-7@ODD`z93t!~D-vJbz#JfIez&__W$UH!c7jiV=FQK{Q!aS7F;JRGM z5m3b8zd4kW)!$#UQf(w>jWHptB8H_addxo@Gm&!5lbTd7{2}sVt%<^9orK0xloa=k z!Lx3eh!v{NML#8}B~P`UbT#*5q7KnQCCOx6gaTx3wc{Fbt&yy?nZ%d z7wxeIns!R@#7A=B(YWbds0>_rlv#7O%=9tSzuzX#mZgB0SW_szNzT0TY*&t3D)WNF`Ab$*;6iBg6y0toDOk?hUw&N1X+boaaI@WztSomwR?LrfUzs8N|FHKS zUQKmPys&~&loE;}y-AbaTWHcdh=52hkzPXRgdj!fJ@lq@6p$tYQj|`BM4EJvKtOsg zq24_A-uJ!F{SUtNt-ID)S?eSx=bSxzX7(1#RQ@|9E^$Ww z>gb2_o~~zPOs)oVC(mVb-z-%$0{m#GG4Go<9s8nF@l~&-a3}=~7t8)&MAk{NENW9^ zPw&|mMM9e@OH?=dW`b16<|k>yllk#L?r-hh`TLR{o0#*@gPuLsiN$(@f4XNIsi+bn`tV^VoMZj% z5?n9>^TbJXylTwUitToC{t^)wcluJ`n|H?$SoJ1u%1_wx%KC9MPSaQ$TVTY{W#>9L z)wP5^`DJM774MU>XlJH+A*-b>EO3=_n}|ybdR%D6&@I^6l7VjMX6)@>j!8QIsZ&{>>kOMA$&9yg#MOG~PjBnvRkT@5>qv(D2_vkxCO`od zpCo4pdTqXM#MNT(cfKnV9kQ0_2mSb&5ayNL3y3hC_%oQswZ7tm7l&noN`$k+kfI}E zNU$)(P;~1(DQ}>%#zjB{``i6DfKH4zI#foXkNCKJBjY`9z~@~5mHS9mkez_r)R3f)X@#5++B{8h*ETNwxl1k{&P z`0B|F@=6AJ44+p&1*kFwHXrrD{hnhDn*P~j!I%xR?*?ws`)V;B_2Ck0m8vnZ20uyV zB75y_4p4Pk2cvKPmcWPL-_Sk%*y-rNzrQ|8CpC^57OQotN6=-gu$-wcC7DVpBn6i({!UhF z*^YMt{zevV3$kM|7N_ZG4f}*Y#472vY_kdaTygUrBI+M>B`G~!FIMgqr@%x8Z$|s5 zwX4FMjoQ-L^&6uG1cRkhtT)qSRu;n*t^o2_j-(XpLp@nGIk;p)YjftGOs`_57PXj% zw-z*!+U8utN!MgVDU5H z*u-Kx`I&`LUh#FR6(Y@*Ds+OXOR+7i8qRb|^8vP!t!nX48s-jAVmuLuE5X_f-->OT z9c@pQKnjm(%$1T-Nw^D*4M5)c?C06NmbW zkO_4(PLs{bAB@V!xhq_lPRP28?WWBeA*KRouW&GCS|Oz(=W(TS6QuaB5O4{l1%pM4 z;*ywC{d|a=q26dmKt}HoCwfJ2ip};#Vx!=0JDl*w2?dusWA5T4n^y}>I_+z}Wx_fnhVztdwm}~!Lwz9xB-?;*7bZ4Ii zCOFzmNOuOpRAsmyqKmBGDbzS8_BPaU11JbW2WeoMa+JZocFM7auNTyjmv(jENC`GqK~39o_{x1YuDr?{8t4T_opke>Ont7I_{q@ zE9nzv%00I;vPbtblN&K1dlLmD@5&^M@^oAFXaa>Tb0IyNT#4s)db9CLqH-y1 zy$bYneb`!3#wZdtHX_Z3>2JsDipC#--YLk)_)M@fs)qWIZCLaO%gB%Zydwkhd1^02 z3BqGo-+9h2DS!CdIn8Bk2tNmyrbfUy5jug7jXMOfP;5#X$j_e_;Z|ERpH*~%t07Lw zE(>M-kqUeenxcM{PKbnacsSAcjKv19E;;0pDkZ_9KJx}NvCbPY{S>J09Jsa(+AW_*T zm>)k4%}}D;<+Y?Ifj)oxr!~4MmY#sG*H2Zxxll!_kr^E;QUX!F;Qe|x5vK^n)UtZ| zuSL6^=%wbQAY**5Z zQc9@ICgZ}TllCgl)i2@F#pWj3w}zdgoUy(=iUQ~gY8sCrtYakueuG9R(cKf3V0K9+*x z@nsITt`ZS1zZSf6EWvup1; z%7=^+6ty2p6bAjYT;6)ZKc;p0CqZbL&hR7R|+-RF^(~ICn86X+lm47%}Rf`erO_z(Z{)h|PL6|K>hM zGlWInplp;Hek_bqC<)q^}0*_SM+JG^i9T`^b`roP9?>OyoYfo%(ayJ2|!?D zxy7uFOGW0qFd#57Rg$2RVFd_Qc~z2f_x15z4X&N=`WWx!nI|DnFSranfZDO3mwbeY z2f%Wj;(P-_Fshdnr1P5X+F}bhgupXwq#;P_bxrWGBDSYMyx$U+uvUADrYrG4c*sW7 zHWTpp@46lf%C?ff#5-%G9}7G0Z%DaxsowZ0w?lGEq&#o`yqWL&4pEnq53Bx!^jZ3D zv?JrFWf)PY1x=exnLvvpB(zGrI>iwYf4I^zmJVDg2fJ;c|)@*)d8P~(?*A^f9{9AEXN)uKEFjqQ7Dnbp-8BrSPZz>o2d+z(Yq9Ez!w zx#Z~{OKuI&+p5uRtO~u2ZX0ENrpNsB%cyV>J2IE$s^?DbN@V}u0JHP+tJtHG!uk&l zx&ekJdF5$z`MF3I3y;-NrH`?t|YO&`kjXyU6iH&;m_ZK=i(LaGZp4 zTO$a)LRSQoes6unl9ejRWI@tqm)spFGYuzZyXPFYGQojGw-#Tgb!xEts%P&Txb5*X z^xLT!G%68H_ij~C8m1PbPSri_ojeis9sE~D30 zdC_Pom{Vu^8x?BSonE@oSz**WDPfwC{ec9S`6KSn0}9K1@qnvHfWd|p?OBZmmv0Pn0db^Z zTc(%X_;2S1lB~F$Y@bxFuJ}tMxaoXh1!UHF-jUnra2OIPv{LXxQHG%P$DT%LqdA(r zbHdDw1-+;OwICcd@nXiHqzPJA!yxnO&skz;Bn(Z!JIR7INdr_K&3@=W6-HHBbW*mO z1^#MMu>OSjji$XLp5Z}(PFABZI=n(Z5z<*LHdMu60_X6mYVL{msxZHx>}^8b0*{K^ z?0L)^L%Ki))Zurrn{FRsm(WZqjlo?i+nhe##y8O)nbpVN?Xq@1rDC6lQIX zgNbr4@pj_YAM8pY0}E(N?gO3|AuGA&N15O5<5MuZXVu4T(G#Ves>%#1pkxs6{r!G9 z;TlGqQjAH1QDUA;_f^$4zp3Euifcb&3jwlo2Y*=T6D_>3-Pc}VdwZ~*bY97h2nUWz ztT1&+BMO@LY0l0o3W+jZ6M!#slvL)}Ix7-u=)`xNF)rxr~mDdWuic^#Okn|FQo(JOF?*F_}{58JdOv zBz5|YJn(Gyleq9=!QL%_#5jsy8ge|2`YqzLGpJ;x9>xT9?C!JIQ=3d7#JP zL2~&KGC~Lu>9qk=p~KP#7a2(46pW}+zciYi-$$fa^trmgqT+Lsq9LzDF=KWB1s5W? zB%MqYzr2F*5C!(wOws$$jUpA^4AV0{k0YZz`5}#nPZU|_rbpzj7)tmk;(!?ch!HGA z_U4+*>X)QJy{QtLiz3vUeV($(%PjVu6Xhnp>FHOLt-(5}3kN91#{_W+s@lHS<6h5V zRh}PZEpa1Gg7n0E-&b#D1%MM1p%eU!s8@6`*0WMO$&d+VMv5^*T*M`+xeH%lO!G3v zPeYc2VKdLm83L9|)|>Y~wpp`)b1YLhnq5^8%(L66%m@#xePr4wstFut!VD~NFBK`O zz*+=*-Zo0f-v2w%Ps!Sqpu~XX*h{fSj@Bi);?NF=&_O+d?p%?y^DKOSqAqH1)o#}o zL-Wki1U`PvH$-WEdW*E&sxm89Do|YPs1SDHEbuu%dBpcU^XUXDzlH3?(Jg8*|3=F{ z-_-6_A)G~tvV@|9g5@rr4_5*UGadGqkW9WGmxhdUzx$Iqj-edB!5q*?LsekyT3~HO z;A8Q$qO}-mp4bV+m@;m|dhS#WTxW8RaX_P|%Zo|7S49i9v3u6G~$dgen;&$pM& z9QQ0|XPU2Vtuj9ztFmaBl`0Ugz)KgUuhBZU;(M2NRG%+8h(1$9M>=n_J>wPiRRL;b z2O{TskFA~cE66ouRe2vrri&Oct!SmJCO}doo()&!cqI{+?Wu_ar&X?oC}nLw7XUwEwi2Z z7Lm$WYGHb&JZMB(f6n$H@%X0bhM#U|#tlWUIMd^hEJZkR=smxUVUf_~gdHJsl*zlV zv4=9Vy|zt)JLr+=JfBVdlAj?3(!P6`C2ZpBp4=DUOoWh86I#S`@SIa_cH_M3 zd23{*ywYvi65w&-#!z=6I9Cgtcr5&nX#1Lfy)pdHsdmgMw#Hdzc;-7))K z<|Mv$-4TGbZZAQK2$Wgo`P752GahB%aP^e{C7dB{oC@?c3J#{Z1*R)je#n{=o(mg4 zzbwVJ_%US$;r9SC!z^{u{ygQ5Va*ap*iut~1YU~ci=<6`g%Wv5QteBT_jH-V+VKs! z4}1_aS1NSpp?#EoU!AdBd>fi{9G?sb(*P2fF#6>IK!v4U*^%c~Q4)H9m2ECLPBQQ* z+M*yq@#%HJ_^*cJQHTEPn(Plfwx^X5lmaR2U|OlZ=GD=Xr`Ti#MvC>}x9RI`a+w~e zH_y>#iMje46IIv zR7OQ|T`#5*2~tU&aJ2{GXI-R~Ni9@%PAF{~%xHw}3L6l~>UHdqs2rW^#KsV3!!bH= zA{D39jm0{$_67eDh+Cv(ZCcfx!wfGNQ>u{Lmcc zA{9ieTp3g3puk~<@nXH4ZXlAplF8kNR!;Jie9Ze-MtsVqV-@cSO|Ixi%Ci8=EED=> zfrMRKSzKLuXYqpOQZrt-<(m7;Z6J1XG1VHy%n9IuoMs+lUuJT8+ZU$RPrqc*noF{* zy*;?f?QuLr%XoVntJk>WYVP$z|Ac}n^-7ho*&hO}uD0)!%W5mpB5^rXN@PZTHF2W6 z&G4^ny_^%*ZF>-F6-H+X`R2!tfc21SDEHR1;cV!|7h}+lr|*{pdj^!DL*u5>OyX&B zddhAcTJ=ILYyd)W&&Xqg3lp=Wc5Iz-L)0xzuV%R!JeFCyCqecQ1-av8IwM^ zXV9{Nqb+;M;x+38A+k#erR;62mzt$Osypy%iH%#ZMs{Xg<_uFP<3T$PUoVlz+M|UTU4S zy;Cf_m z(42$nZNsDJR7o(nV|7F&H$uq6KSuFs6`0=4gggOLs)NfOp%wc@@r3PnK%(es@X|Y( z7oK&gdQ+uoHa0IlR;gd;_491|-edEX5X^+r=e*6n%HjqccDg;(B4}%CH(qwi@R^U=qxtwG6qx8fk&!f0uzh0%IGa=Cm9T@v zJN=(tiVfIhZ1uf=CwvPMg=Y|P)(J0B?KQUO9I89cmfw}bH0y>D>Ds8R+iZ7RaJiIh zC$Rw%z~JPylN4f|yay-Ql18}?R5m{MSAVG45S2>Jlf`_2mafzeW{6mP?HyjFJ^8|E zIU}DZbs|`Nj+PBa;N(coTxNbf6dSF$Dou4ddyDXYT4X%y6_^P@_DB6}K4-Qcf#ZHxMBn1xqWOU9@*MRjn_ zxVjV9=C7>!VLx(q`oS5q3#z6dxUs_SRMS_k6OSqdtqx#uevY-P>r9bf9xxWCLlJY$ zHB1NG>tr8+y#68Yfy@}1Up=37U6u|%Du_MKAw=%UMR!rWKL%tU^Quexb8iSQZt$j$ z*UJ|J{i_rkIX0m$(xS{Sm_e4=BuhSg>&U0}_~ z6L6;cmNjxew`B58zUG_A&%S6XQ?#fCk?Z^?DNn*pDPD^S@V%k~(put^HVa@TjzWV7 z>9lepLA1U7k=Uob5+jD$d2T?0IhoPT9Ch^Ca-fHYAS8B3CGki#VE@}5)%&IfHx?5Q z@M8==63q?R(C(8|l}|EaL7XB=z&LSHpbyiye9FyP({i4CRp*G+n*kNIS1Q7rJbSZG zS*v>TubNHG5G!r4wU_Us7+)mtK-dgtc}9P?Z;(%IT<-dJUh-eahrGGR21<3;(VS`{ zlg{#=2lp2Zk7v|Bax36+PW3Dt91^K}6R&NZerV1Ks9UewJH+)E+J-WW_eekBj~RzF zHEgbU*Vf^(8Ce;Y7@q=`Eq**+w_tfuR>unlRH~crpUL2*c++q7EkUwx275o?=|znN~n@e%M4kS&pE5+vb`5LJse z&#~0+{2@RtyQG_Sr2MI&eGmdEySio!Bl_A2vi)ot=%Br84zk9H>Tt^op6`Y%FMe+d zgXGFIXH;0$p>c$hDPqHykqnU@+a?4QkNQPh8>Q_VYiHuYDs!WSwQ?A>JC8c`Z+Qh# zrO<>)NU(~Z*y(~Hk*_OXP-sHPB*SU*;kS`KaoqIwLqMj%@ ziT|ccO7gZ!xSTA$8Jq{Y`;L2!(a>72?o@P&a4}$o)*5Ck?@AA>B;ivinfGulhSM>kn!zRM?}_v=*B45exL+t z`Mdxxz&(cUcNv}`-F-qKs;7xHF?dm7Ij*XPdD9KJwo>W*oLz`OHG9Vo!a0q-Gm9@- z5cfEEAL~fyh#h7a%11wyp>S$6RFl`rB zh;}UIq{>Tniy_ioT-Bb&F{j&}#sW_vDF1^GNYnown{6)3`gewSSyb^}sya$@of`lO z9(+<*+wT+T%4G>wn@aHP=yjtC8~BlYuFKF@uhR}eG(6D50>t~=ee+cDYa98rx#|^A z@pj{29L^pzPLyn?{(>g-9_c~6V*n=Eb7%YqaL>MkW1TvN%kxYhS1#~C!c0j_X2L{b zSoI;+=y!2`)}IG^LSfwIRtvc2oT2xfFC#=oO(F|#6nq`VM86cLAaFodtD%!FBw$M0 zlgcjBq}@lI>mmNHmexH(3IlK&1?0-|DK5}Dd<@>?%;1H>;Y0PP6uFSOo#xT79j1Jn zMo1>EJqy!d7HPu9V<@5xmRRS(?;;L>m|njo4gk-?v?I7`lgdcHf5;C`-e?O{9Pd$h z+vuah4s~dqKXUyO?AIB>7*=AY;2ft$3jczGmEhAHV(V zJ7cas7I&T3#KSjJe$zfGhgaOsXWv+PJc|I$=}re-95sFh->2Aws9)*_#`9**go`Dd z28?~nu6%gn1{FqDn)-pMJGDZd%taP2P`YM*t&7N#4zKHe^|K(UwM$vSox#?8+dD7V zUCVs|I42c6aqNcB#q~S!OeZfCP-reDBir&^(_wTHy`WfGo=-B5+L-%F;lAsJWV9>1 zcz7KxYy(5dyd=B$OysC8tzS7W{}12F{Ap2yi|UFMfy4mx7Np_3l9hiNgZjkNxYA~% zdT=%u)+$Qq_Fe3Obi`M$UJmkN#gb)5?|zD$e#s??NBuk?+i<(CKUN1sX*mVfMbU~+ zEz;t37Imrnn&AylFNc*okRYiTf|x3RFzd>2G%s~I@+Iw7-ceJ0{$_lJFjbj5F>V!j zi_BH^m0t?_Bb})1Mm!k-bP(a!%Rj6KKAT!Kyub+SNg|hE#kcC^kt^KQ6a+v=MFnno ze4E*;GQ5;#9dD!19&S{8M21*j%&|pNRtwkX>K-osF8c9GotaEm@I7yhuj|Nnig3SF zAI`m@g7i1xmWy}3OU}7^Kl$uM^l+E4Fn_AF{&-%KBHH_o;CE*76J42J3Ui5Nj6o0f zn2}4=uW&u`CmBZj_19{*@%=J|gOU6PQ`M9jP>N_chpLK;95R((V=_4h-iKzI8*v~(~$0X<08aYBWvb*zACXRNDI4DfDHSMh=lg!H|UI0l1*)TvS=+f zu>um@)tV2}r0HD*oj&0Gjbf+Gl{U0P+uAwr#;r5I`Ku1%72-(~6j)f7E^2)57sVSRUKZ3pkUp)`%im>7C!PDv z^$c>I*WtM+@cW}95T(}0d~{e0*X%O@Rq3q>8vXtlFq6|^7&LKY*hPFYc_DUv&{DOf zdymaIRRW{dt)+JcA`P2{Tb7z_ zb*SE>jyH~ihaf{Z3m+{K#{H}!5k@PQS^9lISR@Sam~P*vh;|=6frS}`X;}W%=;N5> z`jzcAl1WYNc(Q?IxVdv8RY zO*0WT7D@px!-%f)uh0_r*^ZlJLUPyD=m!LUa(Lcjo0>V|u22imZrQv=E7~!H9;ha( zQNe{f4p<-u*-sjbH&Q)Nwe7%Spp;Z`$S`?m!MG@n>pI3c57&U09jxKii?cDg(~w!s z3waE@`a5nUhm|AiS5iC1b96hN7TH|#Y|(rEN-S)5N_SJan(|t^!I$1-f{a6VBXkIH>AS74+@{zb;IrQNaO{jHOu8#kaJJZFD1F!30_V_mE}K_CG@5WJ!Yrm z_M;oWKV%M)bPDyJimm@L&Eg$NqL+@}XgD4;YgcW~S^6d* z_gPS=aNIv>ryUsjkE!m$SF9Mx0My$Sp(Lsg6^p1P&LW$CI@~k$L+HIDrLqr2AyOWH zuN5z4Po2x6jk8onz>`O(gJd8Zpu$4w$52bMCiB>tx}Z}OOZCPCke&Qy@Jp=@Wr*V+ z(Z56b1(1B9lH%vc_(lgb3mJx05bT5#D7xOk zaz-CfL=6R__uD|u#g%*V2Rvz2^Ay&cWrsL_{;B8KAggveRaMn%k*vCeHs|3;Kk8_b zRtg1pre?g1=@}=A^Gi*P5sbZrL6zl2)VAatRLX!r65~Cs_?ZFTpJjqIT$f%=ilA`( z;hWegqCq>@(AK|u#jn4cl1QXoqyFk}+weXcc0^F8bUw`*}x9QTzae!LSbu$j#zU&`X^7cBW16n>Ab z3k)d;`W2NE>PeS)U|9Ge$>3D_js3XSuoC+=?OB5Cn*7lvSuvCr|Le@Ewsj!&+_P_} ze_}<6S+$LJ4$-Mb1flK5Ev*ye2TfKaw&tJ|&<=wHAPe zvwwkKMGDrGa1hc%xcjXmQ@H}iZFFr!>P{^mRWmwLMT9uYmNNq^uY>55{1|08vl)-UH(CV4Q)#FQ z0_7a$=-0M6eLqBSyb6>+`f|R&3Z!v`mNOYdxp~7cAab7xQ`M0#INQ*1ittDGcnWm@ zREb+FU%I1FJto;-V)YlN_obYpaR_LPL$h+6v6>M-ji9@Rg4?JDV(T0`{2SzL-nRoC zk_a6)Aq-FcVBxwhXXE`ds%V{{1$sk1d=MwAG`n^)MRdtz39qw?EpM?KbN1XX4HSLe z8iZrsxUpaj+M-&OQf!F zEZbY$q#f`tQBugZ+)9V=^#SJ#ZtUoqHZc$b2YXtx;9F7Liq zH1Uv7Zn<_`n16#S!M(0!+Q|)htLbXOU!DY?f^W4peX%0q5B!-X{LC+T$MvW!{+J|; z8$txCi&2EPVTSW9^ks)>O^ZV`3!xVZNBPs4b*ir_k3cx@_Nc1EMF2p{Zc@s_TQe?D z%5JYZHgLdPXefWL*l>QdqdxW>DUsv}=c4dP7A$7uDaip=);Ik^LVv<)9>qy1fpb`L zwF65~CNP{c%aahfF&z)y=Oz>>6fEmZ_FSoK&0cCCd)HkyVi<#0(J5Uuf~w@-!1sRt z=u%$GEY_%#$6DR2S1eVr#)62dM{Py1Nf)&zM^?iNs556HS?*@&zdGdxMO{EUa$BAl z;^hAEt;Ru+-s_2Cn&;{iQSDSQ6ZmwZV{kD{A;r1BWF(v+zZ zBQ!J+f~wBcH;F87IjOtQcWC)U%~bP0>UGIEJr^Py&KA;)+FB8GZ(bWIjgcViS5XVn z(IdUX`ONIv7QlR;jdW^G#TnJ}Q;xttpEQc)pqE(E>rx>x_IRi~KsVg*Nwx;&gSInu zp`_;68%h=^FSU>N3L+mo_DnB={2g?Dm#ZD2T>hBQ*(&NYw;VziZGtd=!oVG9#CarX zd2^bewuL2(wx{vfq1!k(*Ru&(S7G6%X%rUsDT}gb?fHk%ee1@j|ETWpG8Jsb&co_Ue=o)X-*1(^kT6ivu` ziZD!Ce$HRSl2q8^0(m7^#>{4$o}GexY1BCP6_8hVqtIf#DzQtIz_qL3>P;|OYJ(Gi z3ml3m@a`rQIhJ&p-!_inbN5iO=Sq?C7>3%;v1tjjd0z8s3T0(d?QrOl?hr|o> zV*1xkNbN5gEV(vl%7c%&5R2>69X*37DJH%=49g(bFOH3x8>W4l{=2zJycf>P8gUm) z?mv!dAgMEDm(xubE8Ig#z{?vUb$$>Z6X>IU4r=SLjpm^hbqT)%^bOTv#R76JLPVzp zxf7DT;|QvdqWU7V%8xf_Vem=~D?<3wPyTE%qU0KzM|NtCdhe2-@n}-pzXU-8Vk> z?`na6yc(GOssu)UlDiat9_X9=4$hLR)L(L^G`SGB#UYtz)3{Npb=fv_DTcHkyU``e zP$wF{TT8e&fGTvJNwp;6%&?2Os*;y}D^Ooc1iiJ!lQltUWtVZUUk{Z-N)LLU2zPdn z^|~^gvOTxvmQ&Z%o~^uDV(`xMMDFpU$Z?^FltWj%5@pLMv)7e|0U7Mo>{AEX1M4LC zQ(_v734cmH8IvzROtli5M!?|7)37%GC0lN2e$g1qd zq*trQfYyP1b^&i9sN_Jyhj=H(FVu_SufqntR7L`JJ$lT48Xu5V4WP(6)?^Wj1|4 zy`)@|0BYrrXwwwP*0$!7;fO+3MALu}(OcqsbEO1Fk=zhyB;xmvuWBx`*CPw*3(t69 z5$)^HPgB#s18!&5Oaq1vzkd>>bGHsbt-L;A=)0M;zfif>+mu9&%vo%z&Q3R8oV9sH ze)|Lr5hTs+DF<4ZhF&P4KEkx6M|O5t5_fDLO==$2K4U)W&@c2hSjxN!E%*l7$eFTr zd(Lcw{*+&Dk6F6_)a;^k^)~1 zAza+7<=3aur=%2;7meM$ja$1x?&`0$B|RtJ<&=$6rJQPR#e#FD%s!0+4l z<1XR3-gSp6*_%Vh&a)rIs$9r(r(v`6PrgVsAmXz=GOdR!i6nxWhnq~m>G_E|x9R-z z7oh7kC+jsSaBWYBf%yY5_TD>p-rmtvQF@s-_ZwIpRjL^Tam3q=}lOI-Jze-VOuoFG1H_Kd`?71UI zPfitXn&Mtd#nw(kwbfKDJy|FGynu*Neb97P<~wUF)#3UHtR^5!v4m;#s#{w$WZAHX z^(?L*8mahsp(gf-<<0hi_p~3R@F@6qY3=l*QRCo6F@56hJQbPMp}Y<5Wq7}KlS!CU zoWp9a7gVqR^Slt%wZ+uW3z)ogPE^A){u`P%$YrJcuR^*E4Zc6!6vP`IZ`m%;ZRQId>c~B+4es+n z%V1izXFycdk)jV&up9fn1^JK#M>6e8+bQzcty;&B!c&n@KcsJ$%yz2}(mjZZv)zrk zG%#&{?G%!n|0S)kyGC$m@6)Os5i?wGDyMYU^tMkI?>NNmms@t+pmvdW$E-=+V81@cw-qI0d8S8@~@wmYQP&%9n& zoew;q@QlGV9+s8st`i}9<=hQ6Gd5*US0XEx5O_G{e{`8zHf?p$&^_A?WF`dPRMkSr z(i_c|+qOH5f!x~>%}7X%`*=qaIK()}1WW>%X(z;`YBSv``PJz)M=d~J&H5*7Mg;U9 zG+j)@rWM~jqXu_rF$x|LiQ;DG<1xUSD99vk)wE&lj8ScaH4%Wn{hq}yp7+o1(qlj~ zuGULxvqfj_j}l&rPmq$mDhpVEH?%$|1a_-Lok3G~?@fr1wId8=OI_Yv`cq5jy9j(i zCge7+i_NB9f91}B*EsNFQz6aS=9qRFRt|%8yk8_|M+ZVC)E>{}KFB9{H&;KxciW52 zh=Ba`shR2aeBnadh~y^BHk05BIW1q$6iyt|+SnA1K8JI6hHrQ-c_gE(I*^p(i-<#5; z76A%}nb?8utfo)y52F-+eq1W4<|JvT%wZ*ZK2%)OH<{?;|^PLl{vS4-KP!sWar@48dM) z7C7@~Sk>yR7(cGRG?48rB?)S63WZn!;f7yg{G2md8ZdD;OX0sWgPlM&|L(8DTY&ZhlrWeBd} z*E(^x79+2X8;7^m<1pKCuT@p1dc46{&JAgo4kCI#nygNljAiIK=SNsZmcM~jiL|*w zQmRQvc0K!tSH^yDW7A#7hbrQBBw-#f&c8D8id=blaz1cq%Mzr^=AlXHNKzKJf7GMs z(|9^=I-t+v(_LAcU&}G4Wv^jb-KvuAS92xjJBH$lfk!+cV;d|l2>BcZcgrGv%fFEG zCStf(g*+O1j(hofzM-_C_?g8b(Q@i>Me^5d|=)HOv? z%Gk;);_&p2s6Sx%&g;37Ln$4WuHv#$hzZ%C=+DAV(N0Ik2mPnj+mzCuYRizvSj1wI z<@vlA!x^un^K#UycMiUDWy|pOMBF&xoTe2&U(2{CMhi254g(k_czP2{^kqm%upNv; z7djz@p37hOhU$SG#`6tfhNMp&|he%mu1*rZn)#;ciYFcK?A?r z8e-G~3YL!5Uq1J<(azd=Um-v&t4=}Mo;Q%uTU=oMvOv~^WftP{Vfkvd#!F{Kpu7S1 zN__tB7$tG6m1f{_l5EUovf$||zA$slX!2=}4wP=KQl{JG{RKF^xJDxWnI`E_+ot~~ zZue(3fyI6@Bj{z}+-rVcxDb#X)|G6pKAjHqq{Dsd-^B{0xR4+%2A<)7<}yGysr^TA z!<|$UCS{O+mjkGbkn7Lr6FVoh$yoiG>rutg4^%jK6Wx3pbVp{CzoU(qG8G*#$?4+Q zUg5dnwX;0lPl8zX7c)8AgkdG_M$5t=UtGgPfkV~v`bl?~bxZKSAM<=5#=kGV8q~yQ zR%d@QSf+O+p$Vwn?02E)l;I1>ja44=G*5GPZW=Q4Mboq+9Xr$zD~9Bh*1fA{*E4;f z-ib2v!=}OMjCAFucQa_NrQEWpSDJqyiPYc$tm;uPV!;7dG5inZ{yC^r$i7c1tK8Q+ zH-uTbKgslbtBu_I;zP`T2lAby`tiRK!^oU0GGZ%%xw~mDK5O$LB9(d?AK_TICHLQh!5hH()x;hRyF0TJdE_VQ$r9dSBu_~X8$Oc|23S$d&&5)-bbc0d6QGWU_;i^<{n2!KR2D_GEMhfBSMj-D` zBCG4KgK51FRQ~t;n`OxAA-_%1nryV)Uhi37xt-UXo8P6@9 z4AAQ*+?o4=smdiykkE)K+Ry#bt3WAUK>B~~x+}!Uv^~}Cwh(xfyVNGdz9x45`p|}c zOm+#2d}mNU6h&KDt*D~N3j3!4)i|NT`Oghf&wy^UfhTX7UaKP9*w93|&hYt~Kt8i^ z4KAqmpZjLW4n#jFJn!LpOsFI~F!Sa4L4#gI#eTJCh--=@Z?v269Oe<%93VPcd2{pu z{YjQ`%lcWtQ#A(fiW5hL%b9oQrB;%1e_H2BO<11&@9C?bSMu&~=V+)N{yEVDx14|7 z#1q42@=!Wr-S|t;q#Y*4x^)%<&Bwj2P~G8{s$kIQ@n{mv2V8NcNfTv|kG(HzKD2mJ zlC|GD_s4OH?jb|7s9~CG2 zBm0-5>w>_#zvb*r#Z47c{q-oKVm$dv2yWr~Uo3|EWMt=>6V#L*m^KqPnFDa^wH~=o z4yT)1wE(o-vKv0B$-9^4E`8Sq(uWOfvMMlprjKk{owcfV=yyNnYdSwj3Wc(gS^A8s z#sNJpc>ORQb$EvVOU0OFV8~Jek|CB=^L0t9Y!-ww3*WPMLB2vVG^3jN8SL@4JJ7aij+vcawj|20UObht_OI)9~b;J_}37Q$Ql4w6j@U&omL^qrm)MjslX~3rt=;_~ zFctRT8keAP83RQozsN+U$!G>zse)`yXqL4tFDC29S!^;MMgOx_{?Efd7P(S4*7tyP zE^Tc}pAJ%x%5Oqeo*~Gqw|2H8BKauVpaU~bD`!H!b0Mw0l?-tB&Q39YcLnxyTP!*g zv-j%%=mPJkf#tR6qjJ3neGY|tj};s}ITBE5oX<2m`R#Lmt8X5PRWUrr))gg8JLa;m zHhhlK!+Eg(m70uxz31VOs16)a@}#UsKQ60NMC-A$2bgKDW)7`Q;THCP(Eua8tZC`~ zBJ6JggK5jimvpYoe~oD9@D3A(q^KU~{4ZuS%OKpZa0H}dH0}PySU3vE;S9|G(BdD8 z{J-=6N00ws@xyu@vC6l3Z|(nkTB#K2$oL%@-JSye>C^mg>8a<_zvC>`nCDxYqUeKz z*;(MN|2I9nB{hf42lZA2eVEyq=Yaz&+Mww?SN*@PWzsDEIQh#>qBrG!Q4x3Se-E_J z2RuVVYSsT7y#>Yp@BBZ+z)|J@`}A0uAXe$bj(CL>UsgKal{yW2d_x|A? zpMD(J_fRJ9Q1LzvF)uH7I!wwK2e%xG9ooR*o9F$rPQXR~X-r-y@O``fAAqbv8uR~> z0bBr{bN{AmYUw-K4sEFiJ}AC>^;caXCwEpWj}B)gWbxKC+qJIcxR%j-wt+8t1!0EB zfNoMrK9beF+nc@hUeo_FSRU6j@zIWFXz&Ft(bPP&X`U|hV!Tjq>1s`qrf}=e1d)p5 zg3mbH%F8M1+lwg-CFN*M`|0e)=I1X3G}ixmp&v=@$79+w-|g057G_N8Vt06>qm%yU z|0jfbqj9aGO}d{I`6d+qm(`_UfcJa>etzjd5cW2dyioA-*)Hm_cr;JOa| zZ;ILZ1HuMO8j7Ylalp5{hX4BE8!ikWCGe8vnMu+A5s_Yxk9dY)3n?zV(SIcHpC<6* zIK+9GP9H1e-A{^*Ovp^d9!_MG<$ovu4m$X3eeJTGxd^-5InrLIcHG*+-81oF|1*H} z{Y!i1%jx57M}^>^`4uKfxeZPIoW=e3l6-o71f(nB4+XLWT2@nM()!~bsBA4u;d1(6 zxbEMAGHb@?i)S5@?N=B<>#JFd{}8Ho=Iw{Wg0nWqARLk9R`$)WVr}QZ+eHe{T2pv!>ybx=G`D=o-yZ=q*=ait zIEGC`$hl8acAOYvv>PD@%iUcg47`Sk2cC=Hm!KC6 z+%)W@2!Q3&%Lc6BnWvf?Q0R9Bgo{H`b*6snE z&^H)`Ogp}&KY|*ZTx_pGdTe6|C9Z6N|L0_#8XSOEibP(zVZz-sbX>&JHX z7d$drivbOd@ZymjDwL$Bk%PD-(|kDm?j7XL3_|6XphDx!c=rW*3eAGuH&sN{4? z$K94ns@=feYuPBgwujFnP63(X;@d@cT?#2I;5 zcicrL6zBO7ls$5|O`o(0d2i(qQ^@zn;>{YY72qllfl-)uHlP(6f35zh=&t4YpH7AM zt&Gnr;%Gf}X+4rCd3(6T@vO!~W3?E9bprNYLwId6eIJ9dPrL%Jyll(W81i75h0b-9sSkSHHZ*TXG&0wuTix%KZh|2Pks~D-v(B%ijD)iMS-X6B?7E zPY~!f0NM@6x!JJd5jkCR`%}sh%Wqlp`^~Sj*4ZCHy+8nq7|UG2#{)scbSq($J>u)r z?ZD1ucAeIf-Ej@+NR>N{)Zy`;)B*McypBmOw)NPQ?H*>dq{iziWJ6^zAx{i?k)Fg4 z1!ezrym13Hc#e)V?uV}AW7sorxiGIETvBnSTt>V8_F$MoxU@Fcf6d`P(D znTQe!OBv^VNO+-wNULtaOR(}oiyv0+AG)ZkKsN4eekFIb_X#IZ4DWh0Vn&S(o5sLP z+rWBkLNIgS307dG!1kxlwcNoN&wlw3zuBH>BA%_5!b`?c9y0BJ5t`uKr?IKaULQkj zx9xEmALnnycxHF-^p`hgyo7tWRPQISiV8)0bCL~hNuxizOue!I$rBHA|0z|8OuXQK zbi4y`i8eE-bNcIyyW0DG5RX0#rLfBQpts*ee&u%_Iv)9ToIjPeghj-I%{ek{>6J~< z3ZK!m#sgO>6xREoN{KVGzv9q7{}wV!jRS2e0$Gmhgy$t-f;&F=7@^v!Bf>c)-2RAJ z=w!Cc_huDKzVXEGZ2}g^`zXoD;#M2fdJ=e1V%~Og0k*o+hOjdw8Vcl%I2^kn6ZBr` zlgGTI@2V;R(`#|B&YoLurOXvM6e@yvUOWJm9`|rM4m~ZT#*kw-N$7($cG$Ai6qQ;lY#J?qgYNWQa{Uecg(%X&#^^6* zd9y!bTBmz&MXccYzf&?PS1;)$RqF+|9P|vh*^78ihsRnN4I(2Wzk^$O}e+r6Z3101pMo`kwl$YUpzYN?X|Q9M=vA32dy&0tK+AY zV^S`xa&})8NVl++*LC_c3%~wn=BkaE8H7&sN>993AJI1wDPMc|6SPBj+=#w_sg>~v zU@u!@@?ZNjj@Y8u9nD;SMAP7u(7EIB) zI)Z9>pqkD5Hrh2!0(}^WgCYz|HB@%Fi}Qi^mk808Dh zt4i5)DC)zeKYfV@Q#gmSQsj9=p{QK*{)Ul)@^|a@<9ol?b&~Xj2D(x^ue<@};B4a% zWVw=li#=wD@3R5C-jL;#G_Xfefa$9bXg*G$P}5C@@HD1^+B=+b3+`Z#f8C2-mtW2# zB^LDjSza?qNFTYvQyLXfv^a8QAmb`Vtq7Y+zlehcqxd5)bT~-Aiw*ZfqES)5z@3i2 zaee0epk3?-}mp zSUAd3nq6_>tuKIuG9(IpzSVp^AENiHM~(KPv0?g5%0u4V;~&MK7a7!@Ny{U79%ioo zY1Q`QT);RxBf7FXBPMYWHa*DO4z???uTM`Lu#Fa9S%zFaS4~{!kA!rjHv&87I+QR5 zZIA{C3_W5L7#P2%jRzn1@zam5tW8^s5-~EP5w8rRh0v@3vjrCVOfG(kg`*36GM-d* ziB*p`rmg~z%hSVl@m&71wp^h-1K|-PnR~Rk>eW(CD4jZb;*c?ODSwwV=}i`$WOVj?`fI?Z zrp>Ck2DEX&hLk2l0bMBGoId}(I)`rXtBha|Y|%&flzG3I1>X&&_a&Z(-HmdnuBlnF z{+i>$czTd?H<+`eQTI4jgJFuYlp*RRYup%%vCXHJ2edPDhTDd0L=Sss#N@hYs-9<& zY=|iI!2qp$btGGV;>6?g(=vjD#)5>IckXjizb01;2PIY4bs~zoYv07vRzCE|Wc4j4 zW}2t4P~VW*`_lZ6hM-N+TDdKJVo{1#bsoC%vm<&#`{abyTolCKFSD5Fz~XTVU2@r&hGNOrwmJ?d6mU$zxPR=XWVFcBSAjPEipJz zF;-hwKi`{^jvM?d=Q!z^f;Y=cKIsR3v@|6O=(y9kNe5xCpKs8$(A0N#B1ai-;GibWVwaWMKVFkYGyBT~6G9tv+{;O~LBl#mcd zDtr-I!?P(K_cj!f*S$6pU~s~dN)*ib5>bX!_V759S?_mpUF|cGC@*x@i+X%z8Z?GLJ*h;hha4atUZuNGKDDK5Y3?>xe+`Dm z#1C=wdQ`+%33ijUJ)g^DEXH2%uq!hm8(|!zE8p3DVkcBnnV)O{+WGER6^4klIgGrw zJ%CsbD}LXXlRf=B@*Tc1#wg*kl)A)ohJV|n9AmO#@cRUNDZ}D!B*eG1(-WPyR;2$5(950YG zz!l{I_#X@-ah3nsV#r#;fD2s!|3eiCU@`x*);#g<&1WTyy!Wht2jicM51s%g{r5Ng z_b31DdH`-n`lj&aAB(EW1UR9g_`mmm926?J17*$mpZ>bHf8d4he*y-$c)+pWN=D$n zogBKE>`QCP)s)&WOPqJA0@Ft6p8efP_z4;xm@ z#;$8}5Dt^O_g4G)I`_Z7XpHo}En+X^?${qH4f(C?W?s${r&JQbQ&6?>7cJm{!v0ov zZ_hu$c=;zGl+k2(Ml|{#YDBFst(CVNrT^_k0!n3fq{tg2XMDl@xRqp}ecy&^>6iZc z8=U_T6(Cse_J1J;;7X~2{{QFyW{&??=rL!=aI+^WpTpOBaI>3_3Zid<-R!QHfTZ4{ zAdAq3C;xEmJx8)X06gHRTMfQ6!|vYSaDVB6-S z^ql|gX3G1U_f#l!I#BE5V;$LlPPgofQFm*{=3tc-_~7XDx%}B-7?{iyc1T?WL^rlT zIvjQVZYcsK0=64gmXBl1(~#Gq=IMV>7yCskO95BwHy7f6#AFX28EB%$x#i@Sp%>=a zassCRVRYdA^$=zpWz;t770|K{@%!yGV&yui*R#gNsl)ze!g^K*bzvzd^7iU{act=v zMrbOCIJXV>>-36r1u?oS?{|G$9C*4kt{K<9I<;%wak9QE?=yE>1SxN}xcQyckp@CF zxZKqg2l)H;VYQ1Q0xJ+xFnZa|wrS~hShujnzoo@J-50z2&?Zr*K;Rf^Z<0P8d7@{Y ziCi%NS=@ve_*>jS$8R>BJ8oBDxz2YVT*f2#+CVqX^r-kgl15oVDZpX07p%EBvS>rC&a`freXdsS2>r6}el#@s^zt z(Q9*a?@8F4mzfx1Tii?MDsbAM2nDk^&Ou&4_{>UW&W7!lE&fpz@7c{o24G6#Ohu8! z_m}%COR@-a{x+){^X&8QVV^A_^MO~PahkKJb$awXThIwT(4c*)!*{LMw!E{$zw;P9 zZ#93V73BL1BSjl^E-Y6y2fD1idV9K9Swiw(iRwVDMgm4&e0lc6+OkCbyx%So za2A!LspC7o1$p*8@Ytt=6pXFVadqeCBzQ~F(RMlK1xW3W(}&aYpo55FdH*enCmpVr zgU$#mMjiwbA}0-ucx7oL?cXvM__2OaRiyx1Z2&p?u9~!qp?q)dyaL4#Iy$Hx*e(C3)O+me0 zw0I;_0)_n7A$C^*4cGqbtrAB6r_1%Z=M2r~W75U<(2li_w$D9^v)=+>iUf<{!*5wy zN*$A}o3!e6IIkk>t`by?$E5D>IBMR%?~Gy73^Ys{KFd7y1Jq1dH$C#iY$e6I=~l$X zA9cIZF?(h0({8lLGwizftN51z*HxH*E#S4ry1Hp!lHE)BAi4jJIcVn?d30O^v@mf* zdg#({Q;Pns;fP9z_#5UK)By>hnmNX#k;r2d&}mo)=vLGWaKgaf8$E#HXq{I;*59nF z$=^)EhRu<#NRO(J#)}yf3SgW744WoBZ1<+Is{>E7x0qC(x=bM7N>pjwOUYY^zJF7--eki=j6c<=6DR2JkW(g`M{?>Q5a2kM< z>#mWr9mos!drSjx`BNNq-g|JS?+->CSIPk4B)l5$S%})H(SmW0QggczzMWB=17HPL zB%ciKl}Fws0Exm|?2$fd+@PC{mUVC>?JvDYxNh(*f8C=Qy`~E*iP{5P2+v?sU$9%y>hZ(@!DIP_!lR0E0Nt6?iJ73^O);I;8SQPimZ(6BM+X7nZ0YUyK-c{e8~^-)fpq_e zY40F<#3}X4b)V85;M&m`1L`|1M=^0jq$r&IYSQa`Mw;xfoTS4!P27efxn(3 zhAlQMs@__;hU_lR=ZNPWxYUj}H8$Y`iRwokJ9&_zd%z7McofB5E>ZKsGlc=A5B=-4 ziY^U)Yzi`@n}(is{pAgL&c4(rkoJD_cmv#22(s?^J67>(eTfZ*)rA0xZVPn0IUv4% z?f}VBee>>W+b+M&p71=WdAKrGpF%;3Cie?3I`RO83Yz;QMU1a^5)? zkzmo^h|J%B7C&;GzX(h2Bna;YWkKjS2MZT(wcytFtI9BBB@#5wihIa4^3E?cr0Kf? z3;^j|d?!NaYAYq}=Ewl^;I=5Gb(3{^E5*G1+MM#<=`Gf_PF|6^Y(!Q}2sqtZjizii zKDG4!^e_x;FIRxwBTc?PAJGZOZZZ-Y)tYaF>!9Q}sK)2g)XUm;fWSEzQvd*!Tu~rq zSCEG*+5nswSaaPWA;&`s{)Ht?<2b#-)vIs@w)ni3`3B@_gf<*V_s+_aEe04i}v zuI2+EXm9GE`0(A>3V?ogQ=E5nFZ`B!4-}I+^_8Q)LRs+ljkPGifVJ(#NYHi?#&7Dt z=?lnd`L-Pu{&eISKg!jDNX=ydk`SbaMvOK67o*9L+|6VhpPc-~cOBY=??2f9KP4=b zUB<$9=NWx-rK0v?QOV5Q55-@&?4OVRMwLp;E`H2sskq!!YBEffKYnq8G8N99s3*vf zDdi@j;*KWF{K!;>YJ2Goz=bu}$fOGYEp1JPf=&}d6Pwp^_ox;HDINEVsEux*#6^AD zVcd?~3aw4XKCZHUmFzlkoS~)->ZStaYmuR>sKl7Nqvh;7o$HdNVd^$2I$bP}*cC1@ zlLtU(t$P@(c((1x<6qR?mA@;C`xtX>?O$rqa*v*;QOUVf%n4S_Tk`F`&Ap<(l5>$jrbb{ zLbrUkc*)N&;4oCdN(pt3V2xbi9~zDO&aB^h0njHTI@N0TTXpMHv3iQOrbM!9#}==% zk!G_bJ^)?~KOq$i(>-7#AOYXyxJCGb&JuEDYF`Hdl|r;L)LUZ*HW2_2vniWAZ{m4; zI4BOpV)UY>xMdF}3Kr(o?~!E++L?1e;O>$F;im4Vy%hYIdLi}D01^mKT=eVJ=~wEs z^tw^Yu3TudJtYpI(8uw-DJoU=025*?to^Vrsf<)8$THd$Sii^S-!FLsP`ghNy}-+x zKYNcI$Izu6Cl?J5$G!ff{uRfzaIUC(`|Ic}MY!X{hE*gsEPb}s<{p}DzW#L?P{BW~ zle&aG{nulrWD15UWZJmD4Y6vU!1!OseX^uN0!Sar%X@sdYHy(UHaZYF)`_^$fy3B* zvXKEjKn6$_Fc?5$jLl1;W z$Ll_NQ=j~-I9m7Xv4Co%a6lgXhu?@%XR4h;5~jsd^T@|vb6hGwF%o4uLIwiiaMXR6 zGSl*1m&Yx1-5XdC#?Xa(&*JI$-0pRN8JZ1imiOq&T+3-ZNnwD6*Ti(^&1Ycu6ee0Q@PD}entx%>K zMN5)B_uV?a>TO6}VZwC*yR4myydOE+$Fn&pwJuTa#Okmz4o@7Y3B!6Qq<^))O6>b6 zuH2!X;XUX&9!*Z;ei-Zan60vFGQ3XDQCt3bP3jE+nm~3 zXeRwM9UY?lH3XPlKrnv%9q&8vV@g3bqk$4@j>k_Fr90b42y4(&qfy#mq$YZjA(VZc z!Am9X-)|=KY(&sVG&}vx?H`f=n0&@(J`18inP;Zxq)YOMY5BJj{(Vy81+tXKQv^|Y z5{BqfJAY3o)gx@FKKa-mm+7axXz?zCl-~#i{#a>Fw+CIXHw*S&$2S$r^#&ZxQtUqJ zDb87q_Uqd-K%~0G!BpBe975nH_sCwDFcOGax*bU!#G01JS(jaCxBIb= zWRe46QEx-qG@CX}(I4`!k1GNo0=2pAbaWNZcyeWjNQOu}`+|w_D6>P9n~iXrBk$ zaWxs;7Fb`Ze3wW{6}Q&^_>GK~5E@cloB)&AIMa{rY_v5P%jY!*qH<;P)mC%TS@dgrcWBT%BHgI32hHuB1v7|j@l!VJT{n_! zcU(1d+tR6s#_{Vus^`s;k8V_agxQ8N+J`*9g00qHq@^x-*Z*rWOPPtPyex5VDGkE- zLR|QnppV6uDzO*6yOLY0tf=o}OrF$t_L)KrRnW=W)rCyN#!ow{x*8clmL+#-%s3$* zxbU?L1?FT7wR)U1Kg3uZ3^ev%&h7I#MsP=+$NX#FfL*k{jCWtWtF=NTq*>F`?pqatC+f8>6>w>Z86~oO!}L=bvgR@8QXoLPH#?`;r-$r`vz0l z&$*#kaPl3$iyD@|jM2QMoE5%kY-EHMsSlHTYOyt_e@r<0Y_?XoBoh3s??uV{I>k`_ zopIQv5C7Sy!F;VibjiWY3rSDS%(o4i^%j_>(tV%8dj1gwsJ5i4_a``K&Z|CC)6GZ67<3;E~kwtxy+EKbz# zXMOTpmhbt;TEDHPYS`A74?)CDH>gO^84_&ywKaWVgzLhQ1kqK*Er~{bQ{`fJv;-tU z1kLrhfcAUDNK@Ser+Uo>u_Ig`pZV-C_`ve*pY$5~%VB+zWWV_$9N|irgxP5IBexEy zkff-h%|_pwN~;{%YK`$YG~$oQ0?4!>S+A_izo%uYgVg(4QZmP_J$N8>qcG&MG)|Du z*NF^VdwlA^h=@Ecb$uKi**?5Bi*AX)E;gdBY0g^7;-74! z)#5IC(}30FmNIx7`d$Z2*N+=;&GaiWO(@RlTf7b$@Q>9>eMHs3L`2mR9kJ1hw>(@w z0{^)*t@KE&hky8m>E*U@=*x57%l-Bbm=0!!=;NQtY(XF7eikujqIK$}*Frq}Cmq+{ z;40&qsI3Pkh&ub!j8CwbNf3jD2HA*qKX0SyUJlso8e-%vhX?VX>0-XgariVB{B%=l zarhl=5ZJLt9by@Qw6*kkBu@c$|FmI6!$d^u{@ere8CpC}>wY%mIzbO(iV0!Fw3PIq zw>;!y%L{ZVC!lb2b>Xm8_nZZJxO&jB$V(rrtn-L?+|HV|Ds2MRuYXjW5d?-bs;zlcw$y^l2wM z;xN5=k^-Knm5X!Ypl!=87a$BKTOuC`)fkb~bO_<~=Io@4;rD5JfV43%BY2))Dbr2R zd8azJm&IFkdNN$W>f)qM0U^IOTxh2MR_>5arHa zGi$HM$aNuls16Nt1aOrX{OZ}LyM>+7O^#F-)!PSmB#t%4_E(p*sv*Vd{%nznqrbVJQgW_&)tKOFByn zQ}uqmr-ALi&S#UfY<_k{b0{O@hc*CSf?)w7Dj7}FjSp^L11JQqOtF$i8{`sq&bXBal*16>EY)!t33!re{Xg{1rDr4%EW&Vw{82ED1OP zp-`Bc@(*2xD9QslJMp4H2#)`ddZ19zFoY;TSzjFUhB_g~L3pD7Kzfc|`pspPmUiUv zdv2goU{JF#8_~6d7`%|e?deK}#9#9DvaY}8(%f@9m}dqe;G<1Xz!F{#anC*GXv#S* zG5QD);S|IRHC0-HWidT2aujHj{w?H;Dz{=0pc9((T9au zZPwfco<&)eT5cr=+XCU+1`7`7mst`c20%T!DKpr!JvI1H3g|@|%L@A|2B!6$(pQQP zZ39d_a~d2oKhUNOrl`j^)Zq4p4kALC2e^pIv$}&I%MWcic zq_PhleGqsezsw3kRTU{4wdk=C(`_v**Q7&f(yqGllUU5` zym!g=K&VJHaoZ1PSc8&ZeTkV@8N=-9o-5FEJ&@$y07}j#;xUW8sQq$un&KWuy~pe!c&SCXcF=>P%w+)0@5&d! zVpHNW4SSd`jN&;und^mJhvwnema^a3lk9>CdTCas_k$S zJyu@5jeruR81t%FEm82nSRfzSx1odBX?B{+{`t*2s*$;p24-Sf$pE%hOtD!W)Wuz6 z+s_Q*l`6Y;6+H|qE)NRAD%0GOTUlqCteTv3518gkf@DIzfW$(GX^IsJ0w+MVvo1_d zFU6ZS6>Nz%C2q}4rjzTYP0w2gF(xTv_!qoDxQ&10Yv8<6iA&e(E6)l zVe^hP$mjcI)?3Q$ksI&`@2F^<7ecd->o7nLwI#i4FJr6CZ8{F6knfBNwCD%QCC4_n z#{m8xi#}${wD!2`1F8U0^RXA^8(SQ&f~qIHb_okZ#>J0?A#PZnY4BxJ{4gL7yr&y; ze&Nbi`jb%hmMMc>EncWr4V74?p65YA;2`E(%LKr-uy>OepeLm;89mo8$2fC}O%hDU zCeIM*x6I$Re?qwl+7-hxy*gNOR4}VCmb@tJcv_Le0VKA6Uh|Ot^S&WBUvSQdvp}&} zaQB0Vm{>vn96BIM)Phu9Q%O77W)B%2UAbk76aS2QpYk(Obmv1!z6AP0?s1#le7%A@ z8IOHG8;-t}m3^|Gex2pv8#(1KqEzIXJ1*{z4dJMsC!FZ(XT3a5m&M8r|s z9(mE{;f_wE_g)W?F7(Svsibg^X_xY^w58P-d2+<`h)?{p9WYN`Y-@jiG9f)=@Ti3gkGb=$2x}e=su7dAO+T zu^f^Blv+2&L9I7_4f*bBw)pXH^q1^-=Mub5iZvA-U?vLl*aN$Zg1}Krs+on zEd6$A4+=Uaiz%;WJW}JH0f>a-8iSuW^xbh!ieA?c4Y03~cZ9&BZ_;Pgddx5C$~)fv z2LXW^Gn|R_?)4@#J~on^XM0C()4#Bx_CJM0GVOu4&isD`GWlO%H6KgSCV{XKIRJEu zB}dN#;0sFtqypjv{sS33Mw=WJUy%W>&}PL}4vlz!Wf4CPMESlsMRl~Df2#R0aMm$a zT2oVVn=_x5)hHcK6QAZE#NRdzx! zBydAf%}S?k8xy(DJO!a0jknFpcVY~Xt56_Aky zZtq2?whXSywNet6$~UF{BBk5`yPej&0gZ_&I=N5XO-|^#{{F49q^h}%y5N|YIwu?C zK5iHjENBD_fwUalk1?d`yHpHtpE|>_yP57Ld8O)NlN;1}mOK}0yKrBX$OUt<7ecYs z6^4h~`}pDJ#P`?5?n|6x7MOqLyCVpPk%-$1gkC5aWgK8KKHQTM11O;Vsi`MRvEV$4 zsUbvgEKMxaF@Hn0w(~rwyQv(V7~@orGi+V;C$X~B5n0p;+Hol}7zR9ogfcFcVXEr; zOAc4G5mw-*k4pSvC#4s9eC*W(=C~RTlS+9oi88K=w-WI48*m6yil2TN{5(G8E4uEr z!u|h4{vZC|eZd0eLvJbq-hugoh*OCiINXz$L>VA2E4T{_ICrHjPhvRxqOiI4^SDRY z7JqE0$!)1crc5cf(U(K909y|@0QWxyE56*%dnZeCmbH^QC64mHNB*hVp{ZMeXhAAH{WX0R+>Z2YeF#^@1gNzu&>UejsyDlT;EOR9OV+s!;KU*& zVPR$6s&*yARha?ihxi9@$bGpflF0CCla)h}OzwIfCvOu%(Q`M>wEOco|1s$t`m z>9GfYa1zDd0H0!yx0K&yrbDzeoBUX0JU_^kshe#)S@-q!3y4$BP+We5#!@Z4l}b*1 z^*1#o9iI%J0CFHIgi< zMBy8Wvi1|W^+Xbru+dj+N2>^iYFA;_L1ad}rpMKtT}V(P=St0ey`QN~VBebjG#H}qD>XOwQ<3-%`cL{TD35ONi^NHDZxEp5FU8{&Gj zBGm@z1IwTe#N>MnoL9{ArFaLi)7_6GdCBOTRDLOtaU@8=5qMe!(v ze%b#qHglLXPH$g)u}!ec2l1;-j4v}!eyC;k`S_2IVqcvjL&8D_$7`upt{QmWAXg1Iej}s!jS|p651_gfc0Hj!1gLYVRrq^hsiK#@zpm z;}68QPcBBwKpM{LQSD5%@l=YNo9a1KCWU6G-l?O0(dzvP9JrHWDJ-Y$?ae6eyVWIH zq2+;-gh*TAB=}~;62ir<)j?25wxrin8?DXiq?OXA{Fup3$~}oj$nuYd*vphMzqp~R zBFBF1EOkG6)2s;oC}F9oHSTiZ-SMZi&If02gJ4Dup1JhdVu+m%I^3^^HR4Ap-|eqV zJ&%1EKZ4dJsj67Co3)nS!RCQTBm7pML-CvIF2|Jp(2{86u-jR&B6PV5m3n6d^H~V^ zxp&?-g+V;lOys$ta^O(hg?;~p6so2*x&Kd&MN z@d`}meuzG__q)XUaG_4|+~sPv%!C34via&%kRj%n{ua-7@ab$h2Hul_R-g0D&A|kN z%#KI!SAnzgpE`DhxA*K>BQv#yypBvIJr4|%FE)|8)5X@4c`CbmBK9*p^R;#fnUdX^ zX+CsDL1KcKVopomqld%Tn=P(eLp1y$_1ubNbgS??9(3IoXDU>7i8&D~SzId3sv7NA z#lzF|BZ-Bo4{~L4G-a5b@s0Sd>a%!QTgH?97BrKebQ>5!2+`G7j21VMp05k8)oM1^ zVy6oZirskTzhvT1x|wppGTGzW#I>T*ZZ}WiwY*eE-zYh`re_Z0)imWWv}keNvl(?L zc?3Pw)wRDM)U?&>_@B(({<=64OW4d;r|$ittaz3W;(kvsnDEx-IcT~(n0`#eBa9jym%iuM0EXBkspUw-H z)=E(-aw1HXgL@)lYiX&9TXQ)M<^oNJ0>s$M%#ZA)RoXL~e#4BuIHE#mixzIqcs`)y zO|+^JE<1C(CHf*D0zj~!WxS6aoNMwc*!Mc$YT7O<(9fSO=*xIq=#(EWRBE7>cJ4w7 zzuQVb)868+LueMjzMHG7ii7l%11AjEZ@l93Uq=i__D*lEE=3Bp`>E|-Z=auaQIEe^ z8!9YRteM^X+4)Fvteu;ziAtMlm_V_|e~f8SXRc9l{?9l=T=Y~?OHO*S%Glmqx2to@ zVDcL^v$gLC za#v%HGjRuhQ_TrGJ1~G8^i4$3Mr_inhCdN=l^oh(j#u?e5%e9BH`Yo_!Q!nuTS4(m z4?H>NzslK@c^l3bt5rHd;e@zq{=FY>cMV?*o$WOKnrFGr+XeOp=!qxwxt3whik6+kgR zD(&1()fQrgUJ`*y@6e>~_gbN!T`oQ&60 z{ewiT(HeeO?j$ruE~>oM6S_7$@G+YDohK@Z}f zmG(>=mq(fU!gd(5q`i{G92W-Wsw{aY*R^a{wjia#Q|5U3d^55kScJ*PK+v@iTEvQB zE9`h>rO)LmWQ!g@l<`^k01*Dg$gCP&>C@!5(`!=i5N|V@KG<^H&W%MtGvJBTXY)OZ zKNer{XXZ7o{hTf9XY7ADZ#z}~RvNwZe>j5A%nG7&A++U;-LhM|Gh87@tUj z{+DRuC;TXps9(_@ex+T(TVsd3?1m*@4{u`!vs7PR2)*cmpFdpR_|7L>I6t9R^vkMc zAES!cZ?&}WvnjHu{4>|4ZJ9?fwg@`A9kYA}H&$Gh7(AyAt))k;=3vFtUh;hsnP4!CFs6Ty zd)rJPctDQkb+EHk^p(CAkU`nc}t7G+Vb zFKCtEqm09a;K-Kq3H1!MEAKdoK`m{34S+zz_JrTw~Ixc>^@qt@^H$>^#%~Z$xpKHt7!8Ij)O8g6Ieb`>LJ70b& zg*Ug*WC|#2k5e*2ZDJDehHVF%E;KfmX0}|tdugbq&!73^ThMZzJA{%26g9fuW zjA2!%Nv!#%rgM>jPR49hhy-nGK3V(;Z?)k&Kay*?EhZTs!l@a!Y?hw$5x$_aOS`&0 zoT^rMsHVfRATfsVUTet$KHKv8kN_^3u>J9eo{Dih6xcRWEZHAf`!2Jku3(KWGLBO= zDbbj)FZjw#Caa_&F>Jra*giTBFh)Wh+A^g@Sb|GzZ(!dMX*UgCg8j~C?j7Xwk2VCk zgRv2X(ArUblo4&lJDg+;69Vkehn8dk5AotmGIT!gsy4V-PRnTsJ5`Qg%M3qIe6!h| zCbZOJdnaJl7@_?je%%AbL}<%6lFC$09hgW(X9=7S;EuDXT^HYbJ7%Y=b&xun{lbc z!27&uV4d%?P4px&{hokIY^GIErg6jOr$y=u?AGBi~R=CLk$Ix-O9>&?6Va5vTD!)t+k;hU1(%A~VW zs*oUC8cK>Fq^$k^@Cqm();?N#VUIN$+_9KXS{^6QqeGo_;U?VP<}FzvL7Ng=}Gw>_bgXhbWh3poqY`0 z_&wVOpAt*y`%N*y4wIgJYYsbHGzcf8&*rPJ$AJG|+Q`z!VCz8c|l=;@(qs4VTfN zi)Z&a{`B%M(ZQl~JnPUyW9I84Ct3AvhXwkR7V+JWkFE_OzE{s;!>{_wQgn(Ps<%?H ztGyQ$;cCN?Gl-#zPcP+OJe*Nw);;(ZGU(Q>OHwtE%aa(1tf@E3`|cmtG_{&h*onjR z$n&rEtd?gtBXdl3w|rlQzT@Gm5w#%?g}%oseKu~$g{&Wx6Am$;8~p}U8`|fIF|IkI zb;$-nAoVY!MaT1_R!~gV0nW5dyuq2(ebyp6?BroAZ4PGPU~qk=whVv1P5Eee)BIOeOPrj*Um**%T?f%j)GtU^V~$ym)hPFq<%B4qPuE* zgG>#@48|%_d7=8_!U8C#RCW{gxWpMP^Q3Rh#IL@ag)0BzVs(2mx>aSWPbj|>deAY} zb3p!OchPkVGzpR@{E6^Xnxx0-#J=0D!JCysr)!+@O~`hKsi`WxCPuC{*{Bj#?N}&I zT8tQT&NwEMiJk=moo>-NIR}O8yX7?cgl$vy92eMBjWQ#D`wFg>9Gye>3np{pdc|JqKGuRBNVY>lYlrEET zI1PouA~ao-m3V&scOtmDwWDAaj@K(Xf z?T+7_^I!CiQVP*`(QCXEBWyf=&ilmrGcG4*P#B6^RrTFzdSdgt(4RYZ(>LxAGnJWl zGByzm@1I4gujmlX_cp^R5Ubhs3D#44Z#{VqUPv&;w42-YYZ(cHZm6gNPVX=@oQjB* z+30!{?S;iE(`}xWwD=T7Ls^(BsO&4!pwUS5k8YOF@E}_r(u5jq6%-nRZdpQ<=0qIP zCVd(Wk#QCu&`5lI;5tP zD;I4k*1QmHY`MR}{xq1wNdKv9l@p-*8c&wj6jbf9FD-b_0$Md6O2C);cq3Emwv-@j znB<1uWk3DOTSYTCpyKWF;Us!X+#FQ-c48${1@HLjc@O7wF@5X`1FL3^09LECO%{!O zKwo=?Ax41hkdy7rF4ei*vDkR5WYDPmA4_+UScZ2A335AfKKX|f((*juc#iyIahL=}w5J~XRCj74%Ysf7yg+0O&xs?U_xl>B$ znfiX;WunJI_$+I!;0JE8KJBjdD|od3bN;V|Ai zzCi*-5sR)+5!%V!*N?CWYOfe5Cw!_nx?UA(r0~r|0g9NY=UQAH&XJOKM}8}R$K8~1 zkHUM+EUDjS5s9C`dw<$mMjzpk^SrxGscjUs=~nLcNQj6gE27+;ipSL7ZmEaC54zS9MN*%5 zL?*jV@`vAkUid~uQQ1$}maWw5!d_InR$)#)lGtz=gxKJdj!g3C-OwG7g%xAF^0Xx(9 zMl0(rqVVOmbWAOR9rov#-^~PX#q!vobl-m{TugJ4o(D;@L65^J6~;mnY*D4Tn25F` z(!L{i@=~8;+88otA42a@+%O1zQJN^-w_`y;cHI_h!7AAw!AG#Qe8nrs<+C~8^-c)z z7%Z!6TDS0Q$7*3H!dzMOHb`F>!jEL1QE@#<{F&h46%}1_C*QAJEUNVSPDJY{cq{iq z@wHaTaQN|T>RB!#Qqe{?E3MJ6JXz)4O`f6dah&0FnITvA&p5&|+*F^XZ1~HdF?zJYYgvgcgO+oi`_jtLbF(QqrxgQqZ z73uvQj49)E*Rv%uq@qp^rcA|O`cem?d^8-aS9<1ebh@lYXfq;BHwARBfXRH@IvqN$ zdAqicq?>W7(ZeX?4WPjbmo3s=qNy1k9%3FurijFF9|tWsfdhMCU!}$JGsLq&}?WbG$(8UG7lLl9*O*nkwx&UZMs#O zCYyo@cLCSO|JsBcZh4H_f$7|jow6a88rjmCe$Q9l4QrEIo#*+SH?^(;qyu@y%qHw@ z>+TvpS>(3-!7FNUO6`e^s7#aV^J*7&r2)R)!3a=#7#4EDO!Yi{*i*eJ_L_QS^cGNH zv3$47=j$Uy!vA$h4DY%yw~vy~w&+h3FQ@zL1x_~Mm7?VFq6{&SF^Czx{xNqGANI=` zJlOW^~J5U^G4MNM>X~nn$$2~PH(TJ-@BS8^Dv#>og`LbOL6U_xzWN`C%MDh zMU`!2HZL)imFHIuY=vg38EM^yZQCY5vi>-?X)u?faIg)dw0_lltC9YhMnnZmFa&#) zgV7x0D(16G0K^Wh8VJ4{g?Od&IZjCH$Xo;keZ1MP*+2`nN~`5+oPq#1Vp*;6Jk?^Z zO~FH2_7MM)sh=j2skqW%=cJV4#C%ws>^lLzK%3EWD`jOx4Zj^!6_NFF^k;8~5yVPS zy`gVhIsUmx^E~p#Fz{oR13n=oS0z}-qB|mnh(=Hs3srqAbTzJR*p5=5kLHN}u-&`e z36fpMi3zkC+y3>u#f^tgtUw#RoMgn3{@``g^owYe%9J__wGwgFZ7DQP)LTcAUiHV z^X!LvlV$lB>lP>cSLLD|OYEey+X^BLw$95RmuZ>YRmJR94RectOq4Y}@k0{G?d*m0 z!HfGBg@96Q)bT}trZ*OAZh%~i3!CFQWea26!Mn$f<1Xu&3C7eRW-W+|aoY*w#ntFPRfntg|{-GrgT` zx_A6x)kS(~HGJWU;LKWNQN&BdDl)UAv`Bw}NhOqX-^1?A9823lQDzO^y~Hm>eI;+W z+opAq608u{In;lGEqo7FJ!DQ#J=L@ILAqfd2Sf>c z;Q83uJM-M@cv0=H3VHOA1wd*a8iARCBTJIV45Ib(zF||wIGT6lXD0T74Q6*r&LDy>c}Et3hR57+iV=(d7@rww2ig6x9lU|FJfWF{9E< ziz2~9`CfTHu$v+GlVpOlp_cd zGCH7J2!C%Hx{BnaiNzqpYa%MgODCs$L>AieYYn}=cs@PolJf{as?83Ph>x;YyMwqx z*z@^=04LAvq%LGhU0pNx$d6gY z!|fv699ZsQn$6~a}JUPDW6K(g*8xG(IoGJ8|eUg<^Lku7Ux6zCCqwYqzT*+M4BUpt1!^rEtlgmWwi=fPZXNR0=J zzUvD&QtH{n{ri|V^z)wyl*n3U-dNxTevDV3-`m{aM;4hTQJpUtj|&>WyX`+uA%QEN zNu;-X*}0{yD#iR^1fp-C!nnf=_N#O}O8vhpZ-otj!I?L-@=C^&$X<3z_SeQjLibJj zNR;Q-Rw-RHkZtwF!#zB&80h1k_i~LwCkYR&PDl_IAMx2|zO=76 zsttG=CmJVwn0mtO9_Q)*RE0!G5J`x+yc1GaASCW|b&M76&IU5lM;puBMLF7Laj&}U zqjdf4F?Npg}g}FBG-70&o3diYrMD5O;CFum=qHS8q7w1pY zD#JOv)Y9}mpylTcfRMMyNf*U0#8*SgFaYUAG_&@Z<9V$7FaeA?6VqzbL!}v=qNx`7 zy|4)D*DLu^5UY7y0O6Z?6W!?JECA{nrT$jgrXu;71vut|h&R7=gVDO#OsI;JHBs3+ zFQ;X)f=RFD*L$!jt}xLEJ?oR;59NN}xmfi+P~Mu#FN;x}G~BGKy0`VVPxn0qpF-;$ zC%!)4+Z++FGG#Y0Vqypv%)`!onihWtpd)*<)fjW;W@5Y!MO_EX8gm3xa>i9s!8vP^U-m z2R+$E@Aq51mgHnR{>5)DO3${B6rDx^wWiBr=B3BB3=<7x*8_i85!ZXa7>Bbdo?_h> z$+adl?Il?}Z-Z~tOTCC=I528EFRJL3Z;_NaIagnMeR5wihv;1Fh_qeY#I9yQL@}yO z&j%*E_#i*aNq!w!p`P6MPE<|ec+@UOYzu)VmHzDzpCIJ|$^Lw#Po2y#=rb?hbmyO| zg>!Q7tpVS zl}4~^1f~Smvg=zvsyeDc7POK;s=F>D{q-iAUJE#eO-KgB|Nbgll$4|$LmG1ov(OF6 zjH)HZ547JD1agpI1>f98S`oVic(#7Zj zTt@G2XcrP&yP;R)N&a%=XKtP^9;A0PxB#gaGhX;iC@7Grm!l>$d{Mc<36}Q4CU#%> zaBAG}Yx+kZ5AL>P|97HOHaLEEBT>zLJ=%AJ#B_P@ZdGP(Rs+o1M5Wbyg4S%7YiG&V zKCvS3`%C<7N}uwDiW_P2(({P2{uCA&NO~-m#3oP+z#{s?eQU>D3KOpDazJ{Zo%-mq zX_EcHM)WyH0DqwTb3*SJv+&S?D-8RolrHS6GJf2;xOc{~{S{X!wFjos@lPB06TdJ# zrPyo)>IJd+NTvTwgk(RIDoJBh{Dm{NWdp4^{Jv8qQn-|SnEdgK!x4;$hwuDE$i&m- zZuL2Kk`Odz&-!DaSdF8UJRpWjnD5es{bpRp9aNkV<8Vkw_Q zt9}Wy$K$&Wy~&<#IdEC3loJFV4CaM`Zr^_Ke$x2_JgpoWDF8jZ) zCikkhs-+yIUPA>e!d=q@PF97Ki@Wee@VBUnm)xdzaMCn;KUxfGw3vmP7YpEJa`(iB zI)xi|CX*RPCDwWLTRs2_ig(9 zg##KS^HY$WOIoolkzXFa1MauP&6!DkI7Zp<_7R$CLb&)=Ev1RGZ;2fw-1&I-K(roPtVt%t3R}Jc2$)1 z%^0^?+{egeF~@M}z}Yd@*u8|S)V3do+ag|!gz&B|D4rr;=t>;XrkFGx&5Hp`3CvYw zMCzUFg&alIDj2>x(fl;MxwkmCeTTW*@9#S(ifLoHr9+aEMA>Q>ONZ>G)7ls8f?$rO zjwW2vDAOrE0X{|%_tLsDoXWVe5J!A^Ep>89}S(tchyT zit7XX7IXn>5ub-o&)YImrafP@dyacT%zgF8LIAhllP& zEq1n{yJyW$xsw_3PqzBfi0!P5xAMv@UUUChJXb9+64lxf$Y*lfeG^MC5vH6gvSD@D ze406{V)9cNEL(b~JITWW{A1n-4T;YybHnT?35dkP#NDp*pA+JGVKnZsujVBu5 z+7L+VcUqU%gMpf2pygZD%p*||k=aBA7&`>0-U|lcx2O9V;6TJpx}ZgX ztT^d!APs=rC2$VYyiM*7)I2Oiw-PqC9F*_EA@k8Wa~G&hLEIm#*uK-BBE>wf$)ynn z`gF>DrzF}W${RL{H6qN61UhE~b81rsM1fGWLMKy%OVF~+DC?3fXtbmd6&r*4>^pG4 z7Bs6<4_e#Q$slI$vBRjQ+v@UjqE|qBm{ug?9VK)y-F?M;)4{Y<^~V5mIwJLnOZaGn1T~}1>Jv_q20sp;Q)AN z5)w%&3sM1Te0cX)6Ym;;=xTp*oO#bkM8gIJ=zVb6=CSrFP3~@r2>w~0WCVK>gw*yNZd6$dH7$pmiOrS-kD}lu3_uVPe-tv;-GMVOoo{d(v0mFG7{A#$C%=4O-{SH zpMJUYX5!Sa;HlQFhYnES(R{-;(+o^CHx1@e>42ccb4E!m%}c22j)%n zm)=~pCqzrw0*FM~rjc|9%ru_3^N*h5+vlep0_tMrK`FNg%DG5=ixIL46Mufph3G2) zUDZUCEt;SrxHCE0n5p||MOr~}Iw~>Q8rMcv@^9K5+Vqi?Rt!j)d>s3nrF45E+NV3J z;*pYVTnHpv(f4exT3BW>m-D4!-^L!nub)7>R0Rr-{XHib8xY z|8)alD6KT>0C7_FGk*=wx*&SO1Br}xf;?{{mEAUshUO@QSXF#(m1j8nqycx3sge1~ zEKv`^6M2O<8{wN`$EN-b`ZrN*^eXLm3PN8 zdjL3c!;8m_FBy}Uav~{}2|RSyAlGvtf`=@xE8DOGlLv-pdS4#AoZP_+&p{f!ovjx$ zjc=xG)}y{8Q;f#^yy{>6N!qw_4$5gdL)Z4KkJIvsMyOuDVC&h{$x9st@{6^XsUM8d z&skK+O3ktp^ZAwC4I~P(aG5<69GuF?F4K)f z(Y<#N?;xTgm8CVm(Tej}|3yZ6tow%`mODF^;@Lw)e4*Z}{;3j{36lb@1d}4Bj@g>n z9Ioe97E$1GKSOm`i+y(55|0d75<2QV?$bb7W^>ZXS~Fz|zXF&2G?M}wO`G_c7p%z} zdAdKV{a)O_IHTCe`2FvQ__`{TR|3Ey3~05-kE>CZF#v&q z9zjBw@W|01i1~L6XMkB)c`MVW8=&Yh#E}SdZ|A3fwDG~0OYS+^lRoYI6Tq_V`&YRKzn7a0y)GT^?Ns$|mCuNJ=srSXqZa_lDZ9d}qdPoO-)fxXYY>ENAA zvHUM}Bg}^S@GuoS#3G<8Jm$kF3*O=R_s?NOr4h@suNVO9_$I7@$<_xOdra>U40{dt{vHx_^qk1Q2#GxE)x(IL}nN^kNFo$cG<9z^+x>n^dZ zuPzy5i}DvVN@3HXB#YBbGsO0Z_LP{#Jh*HPP2I#GGB48)CpKV&C}?K zS1Kaic1zoK=SF?ooNqA}usLaVh+^P%+YX?lI~7mC#eB(BOB|7*(rYTKS6ljdQKetL zsh7%TFY2B-4PuCZ@jCphc2p#fXkz1tgciA74 zKr5XX)}wdoO}P1d5N-M&)|f)PD4&vdGMWGaqn*Jh+G=z(&^!U`V}_@b@ncKwwZJRn zx!((3N8!2})f_v^)4o|B%DgHZPIuRL-O67N^K_4&Ad5q&zUNA=4kj%FVrUA{9fBQu zZ;F!pPyX$KdFOjCL2B@q&zIeNu~D@=;3t&IW$a1|%XdG8rJl__6qL!OuYoBLatIM@F@fc9>XApc25A6seu?pWsJD*?;ERh3qx>1F7}>PN3cB%jh>(*W=hno*{SYyMK+J1a#o5Zck1(O>unAD+7E0?Awz zkFC~7nxZ_Hv2X)Aa3NOTWXM8X4{qsud3v8-ceka(gIGlDWQ63Ff`Qp;>o|~mmEpx;(Q&qF29b9MV%r_t-C9792!kDPLN{1t`H@sh2wU=hJHo<@tVcR z>cZs=@bx{L(0Tq+w)Jrokp@=1#5%*sXJ2L#AeWG!FzwuiGVO65(JAMCZYGsb$^LM0 zvTxn1Ag!9rNH^T=Btsvr$ zf@q;Rs0{t$SZKxdmMvq!%aGNP#f%C=cQotSU zj~x87=gsU@+Dt1ZIq5qQYUBxS*!yQ_jgl^jrnK+UJ<;G^?^z-LzK^6G&}uG`r4Yynxi~&i z4zNaa`j&m>1;zb9khHL1V30Yyl!d?cGCeEn-Ii$}!#Ym3)ivfX!4{EQ{y?eVKE3-^ z#mOB6mh!fXQqSQRgkjtCB6Y8Z{dsrMo2frabn{*zAOs(1Cb}8SQ_UlL+ji*RrfJz9 z{xSHlSQ&cUo9PjWHXYmdC3z4Zch$+)i#c|z@9`|+KtGJGJvs5Ni@a#vu{;g0`@}+! zXATo*X>ZwsYXGyr4g>#%-pQ4sn&wdX*yKWKzCC zf1Ttb^eU6>E{c%S=kL%woi?`|o)v%CR;kGS*lp`asmzTc#$iuO+Gy?vzOB)b^K>D^ zcr#-2&duo^n~B)&brLN6JU3H^>yT3Femlt9 zC<@V6NFJNf_%;O=e|{`6N*O^V=!eKSCERDF3xbT$i;M6ia?g4m8K!hVQf^HY1~|VS zqF3h_o>bbSAOP_JkL4jQlL)s6g~-z!ED>@50Zrt*fUa(za@}fUe~? zNmvbs>-({9J(?0UuAs-zQ~p^;mkmvw3z~(IghNYl&%E(NhvhB_@>F6B z4h?lt8OklzIX#XurqJl{rl*iSaIaqc^q?I(deD>>{J~o+4O-oAm`X^rC0Qa|D8#m>q zgKK2H^&)pt&qWfcT5km8$BDiLB1e`A(WrvcXYvC{@wo_U_5O8?(%Cbmt68gD%QI)K ztzc^_cxocqQdI#h^^D%<$Ze+Y&4*CRZa)snJfg&L8LxqJwINIIeFDCXI)d^B4~3Uu z_we(`JLyG4Rdqb{m7-Dt2b6`0;ALpJy0UC=5kAwF`6h9Z3*HE9+++ux5AqqOO&P=( z1fm9l?z_W2(a{6x*ji`$0-ic;od~yKUad*k{4-xl9&#ZpjFpTuj3G&BIHUm51IN$3 za7P?wR9a>uzmI!+gp@JJtaBx$Y^}Wn4-t#^APs`6)yb5SZl997$wtedG=&Z@0o3i0 zw#I&lW2>#O>bIw~6q4xWJX#n*k_}7!QsrRQnG}1pmmivzV`FG96L7Wzwne;lk)Yn~R19UEZq7|F!Dsu#OGCIHO3{qbDB z*tlXZ$LqGx256#fin-_@>+&IJ^HqAx;)Zv+I%vPfpIh<|seI?M}SJ0x+SKXA~=c*wXA ze`%ZaJ?iezL)$XHW|(TC&ueL-gJ$X{jqwiy@JZlz``h#LvhZDBuS|v$8J_S=Q!A{) z z_wzw`q3=%q;y7`w>5#i4mFvkZ;%30Zs;;{B#^39}qx9z9JDqDs&i%UAWXbfZ!FBQS zzWQ=ycDdzXdG?^?pg2=V+)8vIdFj6SOtNNgb60)KhxMM%8Ed@|-O>HGPAWjM=#*t? zv`3NLs3UQI!=y`TU@dn*St1a*@l0vvOG1u(AbWv^sI-y}Cz~9ng5rdd=CMuO7K^kV7kmR^}^&hGF1^lt7Hv`$hKIkax^ zIgHLZI1HLgWuLtMp{>8_(VcqCO^NH@H+}P%UkJX&_IU={Ko^8=127aF#ZTNnj1!Rl z)H{6PN_5yR{ALeXdmqr)_6+5Tw${Beb~hES2r(kS>Lkpo(@{dl!?FGV!S-594cY|& zfc-9}FgijM)6U_}Ch>0z9qjA~&%db)ywG9@YI<|k1vAx1p_ftt=$vN5kO*LQ ze7xnKa|8n-UlG4g=SP~DF`pKJ+?(ipk;ggjo65=bisR#dXeothKVxQjmXuIW^rOF> zUX`UpfqCZPjSEchQato~S zGZD0YST5Wv^2wK!eZ*yP&+mIl6KAy}D-LliwMTwv=R;l6u3tc)FvE_}ygD^EvVLx8{56uRtP0LO4M9@8exY zV#~9oSL`J?|Mb7w7b{8t_@5R^HUIh5J^DqKhbGQ|yT83Dr(916f>+U5;QyW2{4X1Q zH<}bK`$W?1i!|PuGQpgQ*dlZ#bH#C zO!3_0wbJWnjOvtJtahA9(OExrqPyralDh0N|mT=fk2H=5DZ016$fO)?e~1p+6+35^@T7GpqvsUx5wm0fNR4j(%SzgXW- zWXT;@c$~X`L#lPt#YFa|DLTH;ST{@#oFKhH@UkpFJNB@EOzZpKxMQa(rm-?}IF)Wo zyo#N+T(!?}edp;rcEZ^vNUDUFf%~c5Gu3C5F=(c{DgosHH%q@5bc0iZl{6;mKZpwJAgfOLo{4uTU4|*K0s>8(YTw#diEadU! zS{KWMOjLh(FhlIM`aDP1!F3!(-HBh$o{wM5PSg@tF*8F9t{i=gfiEKb_ISxY2Jhhv zC1xbn5)VC#jxW$PTy^g3ZqvkiaW60~qhY>9_ zIr`<$ZB5_zs&J(Oyo2~u`+7ZJ#K3dsImyG=TO2~Anet%E!CHoWuVrqNEDb=&e{Ml^ zWTgMxRGu46&f7>&t>sWH=LZ?}0lOMW=*8kgP!(>k0SzRLfQ5yyuON^DZ6909)pn zk_V^)jS=~W#rO|cjdB@K0L95tQ0Vx^ib8lVx(-kMglaGDv79f&uhu8$4w4!Uy;SaR zr{*>`P3}|YPs2_0IkO~h&m}jmkRAhG2`7CG37PS|cn)!Bg!|I|-NWU`1DoIN0Nq%% zk2TC?v&dJ=D63ZbKSMguln4OOWpglzjW98G>Hc23ZY`5!VE^Hx;{9=r=*EcO&G*|W zhX?i+e@Oiw*1A8Nsn1DIPsMA{_svMGQIcS-xxW_*@WOpKB=L*2XSgW6dIrnUFodDy zf6z1Zft>+I@c#4SI7dN9<&S)fo$bWvYS}i0n zC+DfM)|i}Lsbg8=_4S-{PdZ62x9=H)^LWdhzWx2i&yBoPM_$5cIJ#|rYTfjOZ1_E# zvTK3r~nW ze*8>%t=nOs6%*U?D(tcI`JP^?eDNoWFx&sFNETG%GNI)R{*=}y=`2MO^xMlKKp)T$ z85uOL#25{HS;77%BZ*<~f2bCk|K$pF&!bqu|Ia1-=l=tD)j;WDh%x$QWjJ~|!T|h2 zBMIF8i*lo<0Qz%N2xQ@*r-SEMyh3wK0LjX<|3kMSh69ei{tw&+P1}){Mn3m%`>HVy zMCTih6By8Wpm{X`9_OtA*xCW37doU$refvM*#CpcvAj4~&_4KsCid)`MpT6UoR``T z-0EqY0j#*5!2kFRlOUYlMg{^`EpRrKGgp0;6YFCL!ce z$-LCZR`Ks>JP;FV8b`wGyy&4EmuN~rig-n`03P`1FX3h0TK6}&Og;|u)r{u&SM+ab z1jC++fu9;drR}OJDyJrsAmqwm3V-52pIWUs5;Z9Bt~Le}xjkR!az8-*;vF#1ac`QJ z1@$7x5-slnafx7b5R~Je(XWi$03g$F=mh8*lSv()iKvbh-B;lcPBkgF5#mo;IA*#?*={u&lhxCX=InvF`Z4*3Ym83xQ$nxz&Q@t9QqG-OSk{FyXlaIQFb(8T9(MGgA^u0p@u z+Vih~m`@r+=<1` + Documentation of the Flight protocol, including how to use + Flight conceptually. + + :external+cookbook:doc:`Java Cookbook ` + Recipes for using Arrow Flight in Java. + +Writing a Flight Service +======================== + +Flight servers implement the `FlightProducer`_ interface. For convenience, +they can subclass `NoOpFlightProducer`_ instead, which offers default +implementations of all the RPC methods. + +.. code-block:: Java + + public class TutorialFlightProducer implements FlightProducer { + @Override + // Override methods or use NoOpFlightProducer for only methods needed + } + +Each RPC method always takes a ``CallContext`` for common parameters. To indicate +failure, pass an exception to the "listener" if present, or else raise an +exception. + +.. code-block:: Java + + // Server + @Override + public void listFlights(CallContext context, Criteria criteria, StreamListener listener) { + // ... + listener.onError( + CallStatus.UNAUTHENTICATED.withDescription( + "Custom UNAUTHENTICATED description message.").toRuntimeException()); + // ... + } + + // Client + try{ + Iterable flightInfosBefore = flightClient.listFlights(Criteria.ALL); + // ... + } catch (FlightRuntimeException e){ + // Catch UNAUTHENTICATED exception + } + +To start a server, create a `Location`_ to specify where to listen, and then create +a `FlightServer`_ with an instance of a producer. This will start the server, but +won't block the rest of the program. Call ``FlightServer.awaitTermination`` +to block until the server stops. + +.. code-block:: Java + + class TutorialFlightProducer implements FlightProducer { + @Override + // Override methods or use NoOpFlightProducer for only methods needed + } + + Location location = Location.forGrpcInsecure("0.0.0.0", 0); + try( + BufferAllocator allocator = new RootAllocator(); + FlightServer server = FlightServer.builder( + allocator, + location, + new TutorialFlightProducer() + ).build(); + ){ + server.start(); + System.out.println("Server listening on port " + server.getPort()); + server.awaitTermination(); + } catch (Exception e) { + e.printStackTrace(); + } + +.. code-block:: shell + + Server listening on port 58104 + +Using the Flight Client +======================= + +To connect to a Flight service, create a `FlightClient`_ with a location. + +.. code-block:: Java + + Location location = Location.forGrpcInsecure("0.0.0.0", 58104); + + try(BufferAllocator allocator = new RootAllocator(); + FlightClient client = FlightClient.builder(allocator, location).build()){ + // ... Consume operations exposed by Flight server + } catch (Exception e) { + e.printStackTrace(); + } + +Cancellation and Timeouts +========================= + +When making a call, clients can optionally provide ``CallOptions``. This allows +clients to set a timeout on calls. Also, some objects returned by client RPC calls +expose a cancel method which allows terminating a call early. + +.. code-block:: Java + + Location location = Location.forGrpcInsecure("0.0.0.0", 58609); + + try(BufferAllocator allocator = new RootAllocator(); + FlightClient tutorialFlightClient = FlightClient.builder(allocator, location).build()){ + + Iterator resultIterator = tutorialFlightClient.doAction( + new Action("test-timeout"), + CallOptions.timeout(2, TimeUnit.SECONDS) + ); + } catch (Exception e) { + e.printStackTrace(); + } + +On the server side, timeouts are transparent. For cancellation, the server needs to manually poll +``setOnCancelHandler`` or ``isCancelled`` to check if the client has cancelled the call, +and if so, break out of any processing the server is currently doing. + +.. code-block:: Java + + // Client + Location location = Location.forGrpcInsecure("0.0.0.0", 58609); + try(BufferAllocator allocator = new RootAllocator(); + FlightClient tutorialFlightClient = FlightClient.builder(allocator, location).build()){ + try(FlightStream flightStream = flightClient.getStream(new Ticket(new byte[]{}))) { + // ... + flightStream.cancel("tutorial-cancel", new Exception("Testing cancellation option!")); + } + } catch (Exception e) { + e.printStackTrace(); + } + // Server + @Override + public void getStream(CallContext context, Ticket ticket, ServerStreamListener listener) { + // ... + listener.setOnCancelHandler(()->{ + // Implement logic to handle cancellation option + }); + } + +Enabling TLS +============ + +TLS can be enabled when setting up a server by providing a +certificate and key pair to ``FlightServer.Builder.useTls``. + +On the client side, use ``Location.forGrpcTls`` to create the Location for the client. + +Enabling Authentication +======================= + +.. warning:: Authentication is insecure without enabling TLS. + +Handshake-based authentication can be enabled by implementing +``ServerAuthHandler``. Authentication consists of two parts: on +initial client connection, the server and client authentication +implementations can perform any negotiation needed. The client authentication +handler then provides a token that will be attached to future calls. + +The client send data to be validated through ``ClientAuthHandler.authenticate`` +The server validate data received through ``ServerAuthHandler.authenticate``. + +Custom Middleware +================= + +Servers and clients support custom middleware (or interceptors) that are called on every +request and can modify the request in a limited fashion. These can be implemented by implementing the +``FlightServerMiddleware`` and ``FlightClientMiddleware`` interfaces. + +Middleware are fairly limited, but they can add headers to a +request/response. On the server, they can inspect incoming headers and +fail the request; hence, they can be used to implement custom +authentication methods. + +Adding Services +=============== + +Servers can add other gRPC services. For example, to add the `Health Check service `_: + +.. code-block:: Java + + final HealthStatusManager statusManager = new HealthStatusManager(); + final Consumer consumer = (builder) -> { + builder.addService(statusManager.getHealthService()); + }; + final Location location = forGrpcInsecure(LOCALHOST, 5555); + try ( + BufferAllocator a = new RootAllocator(Long.MAX_VALUE); + Producer producer = new Producer(a); + FlightServer s = FlightServer.builder(a, location, producer) + .transportHint("grpc.builderConsumer", consumer).build().start(); + ) { + Channel channel = NettyChannelBuilder.forAddress(location.toSocketAddress()).usePlaintext().build(); + HealthCheckResponse response = HealthGrpc + .newBlockingStub(channel) + .check(HealthCheckRequest.getDefaultInstance()); + + System.out.println(response.getStatus()); + } + + +:external+arrow:ref:`Flight best practices ` +=================================================================== + +See the :external+arrow:ref:`best practices for C++ `. + + +.. _`FlightClient`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/flight/FlightClient.html +.. _`FlightProducer`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/flight/FlightProducer.html +.. _`FlightServer`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/flight/FlightServer.html +.. _`NoOpFlightProducer`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/flight/NoOpFlightProducer.html +.. _`Location`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/flight/Location.html diff --git a/docs/source/flight_sql.rst b/docs/source/flight_sql.rst new file mode 100644 index 0000000000..169a0e24bf --- /dev/null +++ b/docs/source/flight_sql.rst @@ -0,0 +1,32 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +================ +Arrow Flight SQL +================ + +Arrow Flight SQL is an RPC framework for efficient transfer of Arrow data +over the network. + +.. seealso:: + + :external+arrow:doc:`Flight SQL protocol documentation ` + Documentation of the Flight SQL protocol. + +For usage information, see the `API documentation`_. + +.. _API documentation: https://arrow.apache.org/docs/java/reference/org/apache/arrow/flight/sql/package-summary.html diff --git a/docs/source/flight_sql_jdbc_driver.rst b/docs/source/flight_sql_jdbc_driver.rst new file mode 100644 index 0000000000..1806930943 --- /dev/null +++ b/docs/source/flight_sql_jdbc_driver.rst @@ -0,0 +1,175 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +============================ +Arrow Flight SQL JDBC Driver +============================ + +The Flight SQL JDBC driver is a JDBC driver implementation that uses +the :external+arrow:doc:`Flight SQL protocol ` under +the hood. This driver can be used with any database that implements +Flight SQL. + +Installation and Requirements +============================= + +The driver is compatible with JDK 11+. Note that the following JVM +parameter is required: + +.. code-block:: shell + + java --add-opens=java.base/java.nio=ALL-UNNAMED ... + +To add a dependency via Maven, use a ``pom.xml`` like the following: + +.. code-block:: xml + + + + 4.0.0 + org.example + demo + 1.0-SNAPSHOT + + 18.1.0 + + + + org.apache.arrow + flight-sql-jdbc-driver + ${arrow.version} + + + + +Connecting to a Database +======================== + +The URI format is as follows:: + + jdbc:arrow-flight-sql://HOSTNAME:PORT[/?param1=val1¶m2=val2&...] + +For example, take this URI:: + + jdbc:arrow-flight-sql://localhost:12345/?username=admin&password=pass&useEncryption=1 + +This will connect to a Flight SQL service running on ``localhost`` on +port 12345. It will create a secure, encrypted connection, and +authenticate using the username ``admin`` and the password ``pass``. + +The components of the URI are as follows. + +* The URI scheme must be ``jdbc:arrow-flight-sql://``. +* **HOSTNAME** is the hostname of the Flight SQL service. +* **PORT** is the port of the Flight SQL service. + +Additional options can be passed as query parameters. Parameter names are +case-sensitive. The supported parameters are: + +.. list-table:: + :header-rows: 1 + + * - Parameter + - Default + - Description + + * - disableCertificateVerification + - false + - When TLS is enabled, whether to verify the server certificate + + * - password + - null + - The password for user/password authentication + + * - threadPoolSize + - 1 + - The size of an internal thread pool + + * - token + - null + - The token used for token authentication + + * - trustStore + - null + - When TLS is enabled, the path to the certificate store + + * - trustStorePassword + - null + - When TLS is enabled, the password for the certificate store + + * - tlsRootCerts + - null + - Path to PEM-encoded root certificates for TLS - use this as + an alternative to ``trustStore`` + + * - clientCertificate + - null + - Path to PEM-encoded client mTLS certificate when the Flight + SQL server requires client verification. + + * - clientKey + - null + - Path to PEM-encoded client mTLS key when the Flight + SQL server requires client verification. + + * - useEncryption + - true + - Whether to use TLS (the default is an encrypted connection) + + * - user + - null + - The username for user/password authentication + + * - useSystemTrustStore + - true + - When TLS is enabled, whether to use the system certificate store + + * - retainCookies + - true + - Whether to use cookies from the initial connection in subsequent + internal connections when retrieving streams from separate endpoints. + + * - retainAuth + - true + - Whether to use bearer tokens obtained from the initial connection + in subsequent internal connections used for retrieving streams + from separate endpoints. + +Note that URI values must be URI-encoded if they contain characters such +as !, @, $, etc. + +Any URI parameters that are not handled by the driver are passed to +the Flight SQL service as gRPC headers. For example, the following URI :: + + jdbc:arrow-flight-sql://localhost:12345/?useEncryption=0&database=mydb + +This will connect without authentication or encryption, to a Flight +SQL service running on ``localhost`` on port 12345. Each request will +also include a ``database=mydb`` gRPC header. + +Connection parameters may also be supplied using the Properties object +when using the JDBC Driver Manager to connect. When supplying using +the Properties object, values should *not* be URI-encoded. + +Parameters specified by the URI supercede parameters supplied by the +Properties object. When calling the `user/password overload of +DriverManager#getConnection() +`_, +the username and password supplied on the URI supercede the username and +password arguments to the function call. diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000000..5cdf41e197 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,48 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +.. _java: + +Java Implementation +=================== + +This is the documentation of the Java API of Apache Arrow. For more details +on the Arrow format and other language bindings see the :doc:`parent documentation <../index>`. + +.. toctree:: + :maxdepth: 2 + + quickstartguide + overview + install + developers/index + + memory + vector + vector_schema_root + table + ipc + algorithm + flight + flight_sql + flight_sql_jdbc_driver + dataset + substrait + cdata + jdbc + Reference (javadoc) + Cookbook diff --git a/docs/source/install.rst b/docs/source/install.rst new file mode 100644 index 0000000000..b2b1c7163f --- /dev/null +++ b/docs/source/install.rst @@ -0,0 +1,230 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +======================= +Installing Java Modules +======================= + +System Compatibility +==================== + +Java modules are regularly built and tested on macOS and Linux distributions. + +Java Compatibility +================== + +Java modules are compatible with JDK 11 and above. Currently, JDK versions +11, 17, 21, and latest are tested in CI. + +Note that some JDK internals must be exposed by +adding ``--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED`` to the ``java`` command: + +.. code-block:: shell + + # Directly on the command line + $ java --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED -jar ... + # Indirectly via environment variables + $ env JDK_JAVA_OPTIONS="--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED" java -jar ... + +Otherwise, you may see errors like ``module java.base does not "opens +java.nio" to unnamed module`` or ``module java.base does not "opens +java.nio" to org.apache.arrow.memory.core`` + +Note that the command has changed from Arrow 15 and earlier. If you are still using the flags from that version +(``--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED``) you will see the +``module java.base does not "opens java.nio" to org.apache.arrow.memory.core`` error. + +If you are using flight-core or dependent modules, you will need to mark that flight-core can read unnamed modules. +Modifying the command above for Flight: + +.. code-block:: shell + + # Directly on the command line + $ java --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED -jar ... + # Indirectly via environment variables + $ env JDK_JAVA_OPTIONS="--add-reads=org.apache.arrow.flight.core=ALL-UNNAMED --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED" java -jar ... + +Otherwise, you may see errors like ``java.lang.IllegalAccessError: superclass access check failed: class +org.apache.arrow.flight.ArrowMessage$ArrowBufRetainingCompositeByteBuf (in module org.apache.arrow.flight.core) +cannot access class io.netty.buffer.CompositeByteBuf (in unnamed module ...) because module +org.apache.arrow.flight.core does not read unnamed module ...`` + +Finally, if you are using arrow-dataset, you'll also need to report that JDK internals need to be exposed. +Modifying the command above for arrow-memory: + +.. code-block:: shell + + # Directly on the command line + $ java --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED -jar ... + # Indirectly via environment variables + $ env JDK_JAVA_OPTIONS="--add-opens=java.base/java.nio=org.apache.arrow.dataset,org.apache.arrow.memory.core,ALL-UNNAMED" java -jar ... + +Otherwise you may see errors such as ``java.lang.RuntimeException: java.lang.reflect.InaccessibleObjectException: +Unable to make static void java.nio.Bits.reserveMemory(long,long) accessible: module +java.base does not "opens java.nio" to module org.apache.arrow.dataset`` + +If using Maven and Surefire for unit testing, :ref:`this argument must +be added to Surefire as well `. + +Installing from Maven +===================== + +By default, Maven will download from the central repository: https://repo.maven.apache.org/maven2/org/apache/arrow/ + +Configure your pom.xml with the Java modules needed, for example: +arrow-vector, and arrow-memory-netty. + +.. code-block:: xml + + + + 4.0.0 + org.example + demo + 1.0-SNAPSHOT + + 9.0.0 + + + + org.apache.arrow + arrow-vector + ${arrow.version} + + + org.apache.arrow + arrow-memory-netty + ${arrow.version} + + + + +A bill of materials (BOM) module has been provided to simplify adding +Arrow modules. This eliminates the need to specify the version for +every module. An alternative to the above would be: + +.. code-block:: xml + + + + 4.0.0 + org.example + demo + 1.0-SNAPSHOT + + 15.0.0 + + + + org.apache.arrow + arrow-vector + + + org.apache.arrow + arrow-memory-netty + + + + + + org.apache.arrow + arrow-bom + ${arrow.version} + pom + import + + + + + +To use the Arrow Flight dependencies, also add the ``os-maven-plugin`` +plugin. This plugin generates useful platform-dependent properties +such as ``os.detected.name`` and ``os.detected.arch`` needed to resolve +transitive dependencies of Flight. + +.. code-block:: xml + + + + 4.0.0 + org.example + demo + 1.0-SNAPSHOT + + 9.0.0 + + + + org.apache.arrow + flight-core + ${arrow.version} + + + + + + kr.motd.maven + os-maven-plugin + 1.7.0 + + + + + +.. _java-install-maven-testing: + +The ``--add-opens`` flag must be added when running unit tests through Maven: + +.. code-block:: xml + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M6 + + --add-opens=java.base/java.nio=ALL-UNNAMED + + + + + +Or they can be added via environment variable, for example when executing your code: + +.. code-block:: + + JDK_JAVA_OPTIONS="--add-opens=java.base/java.nio=ALL-UNNAMED" mvn exec:java -Dexec.mainClass="YourMainCode" + +Installing from Source +====================== + +See :ref:`java-development`. + +IDE Configuration +================= + +Generally, no additional configuration should be needed. However, +ensure your Maven or other build configuration has the ``--add-opens`` +flag as described above, so that the IDE picks it up and runs tests +with that flag as well. diff --git a/docs/source/ipc.rst b/docs/source/ipc.rst new file mode 100644 index 0000000000..f593917917 --- /dev/null +++ b/docs/source/ipc.rst @@ -0,0 +1,202 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +=========================== +Reading/Writing IPC formats +=========================== +Arrow defines two types of binary formats for serializing record batches: + +* **Streaming format**: for sending an arbitrary number of record + batches. The format must be processed from start to end, and does not support + random access + +* **File or Random Access format**: for serializing a fixed number of record + batches. It supports random access, and thus is very useful when used with + memory maps + +Writing and Reading Streaming Format +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +First, let's populate a :class:`VectorSchemaRoot` with a small batch of records + +.. code-block:: Java + + BitVector bitVector = new BitVector("boolean", allocator); + VarCharVector varCharVector = new VarCharVector("varchar", allocator); + for (int i = 0; i < 10; i++) { + bitVector.setSafe(i, i % 2 == 0 ? 0 : 1); + varCharVector.setSafe(i, ("test" + i).getBytes(StandardCharsets.UTF_8)); + } + bitVector.setValueCount(10); + varCharVector.setValueCount(10); + + List fields = Arrays.asList(bitVector.getField(), varCharVector.getField()); + List vectors = Arrays.asList(bitVector, varCharVector); + VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors); + +Now, we can begin writing a stream containing some number of these batches. For this we use :class:`ArrowStreamWriter` +(DictionaryProvider used for any vectors that are dictionary encoded is optional and can be null)) + +.. code-block:: Java + + try ( + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ArrowStreamWriter writer = new ArrowStreamWriter(root, /*DictionaryProvider=*/null, Channels.newChannel(out)); + ) { + // ... do write into the ArrowStreamWriter + } + +Here we used an in-memory stream, but this could have been a socket or some other IO stream. Then we can do + +.. code-block:: Java + + writer.start(); + // write the first batch + writer.writeBatch(); + + // write another four batches. + for (int i = 0; i < 4; i++) { + // populate VectorSchemaRoot data and write the second batch + BitVector childVector1 = (BitVector)root.getVector(0); + VarCharVector childVector2 = (VarCharVector)root.getVector(1); + childVector1.reset(); + childVector2.reset(); + // ... do some populate work here, could be different for each batch + writer.writeBatch(); + } + + writer.end(); + +Note that, since the :class:`VectorSchemaRoot` in the writer is a container that can hold batches, batches flow through +:class:`VectorSchemaRoot` as part of a pipeline, so we need to populate data before ``writeBatch``, so that later batches +could overwrite previous ones. + +Now the :class:`ByteArrayOutputStream` contains the complete stream which contains 5 record batches. +We can read such a stream with :class:`ArrowStreamReader`. Note that the :class:`VectorSchemaRoot` within the reader +will be loaded with new values on every call to :class:`loadNextBatch()` + +.. code-block:: Java + + try (ArrowStreamReader reader = new ArrowStreamReader(new ByteArrayInputStream(out.toByteArray()), allocator)) { + // This will be loaded with new values on every call to loadNextBatch + VectorSchemaRoot readRoot = reader.getVectorSchemaRoot(); + Schema schema = readRoot.getSchema(); + for (int i = 0; i < 5; i++) { + reader.loadNextBatch(); + // ... do something with readRoot + } + } + +Here we also give a simple example with dictionary encoded vectors + +.. code-block:: Java + + // create provider + DictionaryProvider.MapDictionaryProvider provider = new DictionaryProvider.MapDictionaryProvider(); + + try ( + final VarCharVector dictVector = new VarCharVector("dict", allocator); + final VarCharVector vector = new VarCharVector("vector", allocator); + ) { + // create dictionary vector + dictVector.allocateNewSafe(); + dictVector.setSafe(0, "aa".getBytes()); + dictVector.setSafe(1, "bb".getBytes()); + dictVector.setSafe(2, "cc".getBytes()); + dictVector.setValueCount(3); + + // create dictionary + Dictionary dictionary = + new Dictionary(dictVector, new DictionaryEncoding(1L, false, /*indexType=*/null)); + provider.put(dictionary); + + // create original data vector + vector.allocateNewSafe(); + vector.setSafe(0, "bb".getBytes()); + vector.setSafe(1, "bb".getBytes()); + vector.setSafe(2, "cc".getBytes()); + vector.setSafe(3, "aa".getBytes()); + vector.setValueCount(4); + + // get the encoded vector + IntVector encodedVector = (IntVector) DictionaryEncoder.encode(vector, dictionary); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + + // create VectorSchemaRoot + List fields = Arrays.asList(encodedVector.getField()); + List vectors = Arrays.asList(encodedVector); + try (VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors)) { + + // write data + ArrowStreamWriter writer = new ArrowStreamWriter(root, provider, Channels.newChannel(out)); + writer.start(); + writer.writeBatch(); + writer.end(); + } + + // read data + try (ArrowStreamReader reader = new ArrowStreamReader(new ByteArrayInputStream(out.toByteArray()), allocator)) { + reader.loadNextBatch(); + VectorSchemaRoot readRoot = reader.getVectorSchemaRoot(); + // get the encoded vector + IntVector intVector = (IntVector) readRoot.getVector(0); + + // get dictionaries and decode the vector + Map dictionaryMap = reader.getDictionaryVectors(); + long dictionaryId = intVector.getField().getDictionary().getId(); + try (VarCharVector varCharVector = + (VarCharVector) DictionaryEncoder.decode(intVector, dictionaryMap.get(dictionaryId))) { + // ... use decoded vector + } + } + } + +Writing and Reading Random Access Files +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The :class:`ArrowFileWriter` has the same API as :class:`ArrowStreamWriter` + +.. code-block:: Java + + try ( + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ArrowFileWriter writer = new ArrowFileWriter(root, /*DictionaryProvider=*/null, Channels.newChannel(out)); + ) { + writer.start(); + // write the first batch + writer.writeBatch(); + // write another four batches. + for (int i = 0; i < 4; i++) { + // ... do populate work + writer.writeBatch(); + } + writer.end(); + } + +The difference between :class:`ArrowFileReader` and :class:`ArrowStreamReader` is that the input source +must have a ``seek`` method for random access. Because we have access to the entire payload, we know the +number of record batches in the file, and can read any at random + +.. code-block:: Java + + try (ArrowFileReader reader = new ArrowFileReader( + new ByteArrayReadableSeekableByteChannel(out.toByteArray()), allocator)) { + + // read the 4-th batch + ArrowBlock block = reader.getRecordBlocks().get(3); + reader.loadRecordBatch(block); + VectorSchemaRoot readBatch = reader.getVectorSchemaRoot(); + } diff --git a/docs/source/jdbc.rst b/docs/source/jdbc.rst new file mode 100644 index 0000000000..c0477cb06d --- /dev/null +++ b/docs/source/jdbc.rst @@ -0,0 +1,278 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +================== +Arrow JDBC Adapter +================== + +The Arrow JDBC Adapter assists with working with JDBC and Arrow +data. Currently, it supports reading JDBC ResultSets into Arrow +VectorSchemaRoots. + +ResultSet to VectorSchemaRoot Conversion +======================================== + +This can be accessed via the JdbcToArrow class. The resulting +ArrowVectorIterator will convert a ResultSet to Arrow data in batches +of rows. + +.. code-block:: java + + try (ArrowVectorIterator it = JdbcToArrow.sqlToArrowVectorIterator(resultSet, allocator)) { + while (it.hasNext()) { + VectorSchemaRoot root = it.next(); + // Consume the root… + } + } + +The batch size and type mapping can both be customized: + +.. code-block:: java + + JdbcToArrowConfig config = new JdbcToArrowConfigBuilder(allocator, /*calendar=*/null) + .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) + .setJdbcToArrowTypeConverter((jdbcFieldInfo -> { + switch (jdbcFieldInfo.getJdbcType()) { + case Types.BIGINT: + // Assume actual value range is SMALLINT + return new ArrowType.Int(16, true); + default: + return null; + } + })) + .build(); + try (ArrowVectorIterator iter = JdbcToArrow.sqlToArrowVectorIterator(rs, config)) { + while (iter.hasNext()) { + VectorSchemaRoot root = iter.next(); + // Consume the root… + } + } + +The JDBC type can be explicitly specified, which is useful since JDBC +drivers can give spurious type information. For example, the Postgres +driver has been observed to use Decimal types with scale and precision +0; these cases can be handled by specifying the type explicitly before +reading. Also, some JDBC drivers may return BigDecimal values with +inconsistent scale. A RoundingMode can be set to handle these cases: + +.. code-block:: java + + Map mapping = new HashMap<>(); + mapping.put(1, new JdbcFieldInfo(Types.DECIMAL, 20, 7)); + JdbcToArrowConfig config = new JdbcToArrowConfigBuilder(allocator, /*calendar=*/null) + .setBigDecimalRoundingMode(RoundingMode.UNNECESSARY) + .setExplicitTypesByColumnIndex(mapping) + .build(); + try (ArrowVectorIterator iter = JdbcToArrow.sqlToArrowVectorIterator(rs, config)) { + while (iter.hasNext()) { + VectorSchemaRoot root = iter.next(); + // Consume the root… + } + } + +The mapping from JDBC type to Arrow type can be overridden via the +``JdbcToArrowConfig``, but it is not possible to customize the +conversion from JDBC value to Arrow value itself, nor is it possible +to define a conversion for an unsupported type. + +Type Mapping +------------ + +The JDBC to Arrow type mapping can be obtained at runtime from +`JdbcToArrowUtils.getArrowTypeFromJdbcType`_. + +.. _JdbcToArrowUtils.getArrowTypeFromJdbcType: https://arrow.apache.org/docs/java/reference/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.html#getArrowTypeFromJdbcType-org.apache.arrow.adapter.jdbc.JdbcFieldInfo-java.util.Calendar- + ++--------------------+--------------------+-------+ +| JDBC Type | Arrow Type | Notes | ++====================+====================+=======+ +| ARRAY | List | \(1) | ++--------------------+--------------------+-------+ +| BIGINT | Int64 | | ++--------------------+--------------------+-------+ +| BINARY | Binary | | ++--------------------+--------------------+-------+ +| BIT | Bool | | ++--------------------+--------------------+-------+ +| BLOB | Binary | | ++--------------------+--------------------+-------+ +| BOOLEAN | Bool | | ++--------------------+--------------------+-------+ +| CHAR | Utf8 | | ++--------------------+--------------------+-------+ +| CLOB | Utf8 | | ++--------------------+--------------------+-------+ +| DATE | Date32 | | ++--------------------+--------------------+-------+ +| DECIMAL | Decimal128 | \(2) | ++--------------------+--------------------+-------+ +| DOUBLE | Double | | ++--------------------+--------------------+-------+ +| FLOAT | Float32 | | ++--------------------+--------------------+-------+ +| INTEGER | Int32 | | ++--------------------+--------------------+-------+ +| LONGVARBINARY | Binary | | ++--------------------+--------------------+-------+ +| LONGNVARCHAR | Utf8 | | ++--------------------+--------------------+-------+ +| LONGVARCHAR | Utf8 | | ++--------------------+--------------------+-------+ +| NCHAR | Utf8 | | ++--------------------+--------------------+-------+ +| NULL | Null | | ++--------------------+--------------------+-------+ +| NUMERIC | Decimal128 | | ++--------------------+--------------------+-------+ +| NVARCHAR | Utf8 | | ++--------------------+--------------------+-------+ +| REAL | Float32 | | ++--------------------+--------------------+-------+ +| SMALLINT | Int16 | | ++--------------------+--------------------+-------+ +| STRUCT | Struct | \(3) | ++--------------------+--------------------+-------+ +| TIME | Time32[ms] | | ++--------------------+--------------------+-------+ +| TIMESTAMP | Timestamp[ms] | \(4) | ++--------------------+--------------------+-------+ +| TINYINT | Int8 | | ++--------------------+--------------------+-------+ +| VARBINARY | Binary | | ++--------------------+--------------------+-------+ +| VARCHAR | Utf8 | | ++--------------------+--------------------+-------+ + +* \(1) The list value type must be explicitly configured and cannot be + inferred. Use `setArraySubTypeByColumnIndexMap`_ or + `setArraySubTypeByColumnNameMap`_. +* \(2) By default, the scale of decimal values must match the scale in + the type exactly; precision is allowed to be any value greater or + equal to the type precision. If there is a mismatch, by default, an + exception will be thrown. This can be configured by setting a + different RoundingMode with setBigDecimalRoundingMode. +* \(3) Not fully supported: while the type conversion is defined, the + value conversion is not. See ARROW-17006_. +* \(4) If a Calendar is provided, then the timestamp will have the + timezone of the calendar, else it will be a timestamp without + timezone. + +.. _setArraySubTypeByColumnIndexMap: https://arrow.apache.org/docs/java/reference/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigBuilder.html#setArraySubTypeByColumnIndexMap-java.util.Map- +.. _setArraySubTypeByColumnNameMap: https://arrow.apache.org/docs/java/reference/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigBuilder.html#setArraySubTypeByColumnNameMap-java.util.Map- +.. _ARROW-17006: https://issues.apache.org/jira/browse/ARROW-17006 + +VectorSchemaRoot to PreparedStatement Parameter Conversion +========================================================== + +The adapter can bind rows of Arrow data from a VectorSchemaRoot to +parameters of a JDBC PreparedStatement. This can be accessed via the +JdbcParameterBinder class. Each call to next() will bind parameters +from the next row of data, and then the application can execute the +statement, call addBatch(), etc. as desired. Null values will lead to +a setNull call with an appropriate JDBC type code (listed below). + +.. code-block:: java + + final JdbcParameterBinder binder = + JdbcParameterBinder.builder(statement, root).bindAll().build(); + while (binder.next()) { + statement.executeUpdate(); + } + // Use a VectorLoader to update the root + binder.reset(); + while (binder.next()) { + statement.executeUpdate(); + } + +The mapping of vectors to parameters, the JDBC type code used by the +converters, and the type conversions themselves can all be customized: + +.. code-block:: java + + final JdbcParameterBinder binder = + JdbcParameterBinder.builder(statement, root) + .bind(/*parameterIndex*/2, /*columnIndex*/0) + .bind(/*parameterIndex*/1, customColumnBinderInstance) + .build(); + +Type Mapping +------------ + +The Arrow to JDBC type mapping can be obtained at runtime via +a method on ColumnBinder. + ++----------------------------+----------------------------+-------+ +| Arrow Type | JDBC Type | Notes | ++============================+============================+=======+ +| Binary | VARBINARY (setBytes) | | ++----------------------------+----------------------------+-------+ +| Bool | BOOLEAN (setBoolean) | | ++----------------------------+----------------------------+-------+ +| Date32 | DATE (setDate) | | ++----------------------------+----------------------------+-------+ +| Date64 | DATE (setDate) | | ++----------------------------+----------------------------+-------+ +| Decimal128 | DECIMAL (setBigDecimal) | | ++----------------------------+----------------------------+-------+ +| Decimal256 | DECIMAL (setBigDecimal) | | ++----------------------------+----------------------------+-------+ +| FixedSizeBinary | BINARY (setBytes) | | ++----------------------------+----------------------------+-------+ +| Float32 | REAL (setFloat) | | ++----------------------------+----------------------------+-------+ +| Int8 | TINYINT (setByte) | | ++----------------------------+----------------------------+-------+ +| Int16 | SMALLINT (setShort) | | ++----------------------------+----------------------------+-------+ +| Int32 | INTEGER (setInt) | | ++----------------------------+----------------------------+-------+ +| Int64 | BIGINT (setLong) | | ++----------------------------+----------------------------+-------+ +| LargeBinary | LONGVARBINARY (setBytes) | | ++----------------------------+----------------------------+-------+ +| LargeUtf8 | LONGVARCHAR (setString) | \(1) | ++----------------------------+----------------------------+-------+ +| Time[s] | TIME (setTime) | | ++----------------------------+----------------------------+-------+ +| Time[ms] | TIME (setTime) | | ++----------------------------+----------------------------+-------+ +| Time[us] | TIME (setTime) | | ++----------------------------+----------------------------+-------+ +| Time[ns] | TIME (setTime) | | ++----------------------------+----------------------------+-------+ +| Timestamp[s] | TIMESTAMP (setTimestamp) | \(2) | ++----------------------------+----------------------------+-------+ +| Timestamp[ms] | TIMESTAMP (setTimestamp) | \(2) | ++----------------------------+----------------------------+-------+ +| Timestamp[us] | TIMESTAMP (setTimestamp) | \(2) | ++----------------------------+----------------------------+-------+ +| Timestamp[ns] | TIMESTAMP (setTimestamp) | \(2) | ++----------------------------+----------------------------+-------+ +| Utf8 | VARCHAR (setString) | | ++----------------------------+----------------------------+-------+ + +* \(1) Strings longer than Integer.MAX_VALUE bytes (the maximum length + of a Java ``byte[]``) will cause a runtime exception. +* \(2) If the timestamp has a timezone, the JDBC type defaults to + TIMESTAMP_WITH_TIMEZONE. If the timestamp has no timezone, + technically there is not a correct conversion from Arrow value to + JDBC value, because a JDBC Timestamp is in UTC, and we have no + timezone information. In this case, the default binder will call + `setTimestamp(int, Timestamp) + `_, + which will lead to the driver using the "default timezone" (that of + the Java VM). diff --git a/docs/source/memory.rst b/docs/source/memory.rst new file mode 100644 index 0000000000..58ef382dc9 --- /dev/null +++ b/docs/source/memory.rst @@ -0,0 +1,499 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +================= +Memory Management +================= + +The memory modules contain all the functionality that Arrow uses to allocate and deallocate memory. This document is divided in two parts: +The first part, *Memory Basics*, provides a high-level introduction. The following section, *Arrow Memory In-Depth*, fills in the details. + +Memory Basics +============= +This section will introduce you to the major concepts in Java’s memory management: + +* `ArrowBuf`_ +* `BufferAllocator`_ +* Reference counting + +It also provides some guidelines for working with memory in Arrow, and describes how to debug memory issues when they arise. + +Getting Started +--------------- + +Arrow's memory management is built around the needs of the columnar format and using off-heap memory. +Arrow Java has its own independent implementation. It does not wrap the C++ implementation, although the framework is flexible enough +to be used with memory allocated in C++ that is used by Java code. + +Arrow provides multiple modules: the core interfaces, and implementations of the interfaces. +Users need the core interfaces, and exactly one of the implementations. + +* ``memory-core``: Provides the interfaces used by the Arrow libraries and applications. +* ``memory-netty``: An implementation of the memory interfaces based on the `Netty`_ library. +* ``memory-unsafe``: An implementation of the memory interfaces based on the `sun.misc.Unsafe`_ library. + + +ArrowBuf +-------- + +ArrowBuf represents a single, contiguous region of `direct memory`_. It consists of an address and a length, +and provides low-level interfaces for working with the contents, similar to ByteBuffer. + +Unlike (Direct)ByteBuffer, it has reference counting built in, as discussed later. + +Why Arrow Uses Direct Memory +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* The JVM can optimize I/O operations when using direct memory/direct buffers; it will attempt to avoid copying buffer contents to/from an intermediate buffer. This can speed up IPC in Arrow. +* Since Arrow always uses direct memory, JNI modules can directly wrap native memory addresses instead of copying data. We use this in modules like the C Data Interface. +* Conversely, on the C++ side of the JNI boundary, we can directly access the memory in ArrowBuf without copying data. + +BufferAllocator +--------------- + +The `BufferAllocator`_ is primarily an arena or nursery used for accounting of buffers (ArrowBuf instances). +As the name suggests, it can allocate new buffers associated with itself, but it can also +handle the accounting for buffers allocated elsewhere. For example, it handles the Java-side accounting for +memory allocated in C++ and shared with Java using the C-Data Interface. In the code below it performs an allocation: + +.. code-block:: Java + + import org.apache.arrow.memory.ArrowBuf; + import org.apache.arrow.memory.BufferAllocator; + import org.apache.arrow.memory.RootAllocator; + + try(BufferAllocator bufferAllocator = new RootAllocator(8 * 1024)){ + ArrowBuf arrowBuf = bufferAllocator.buffer(4 * 1024); + System.out.println(arrowBuf); + arrowBuf.close(); + } + +.. code-block:: shell + + ArrowBuf[2], address:140363641651200, length:4096 + +The concrete implementation of the BufferAllocator interface is `RootAllocator`_. Applications should generally create +one RootAllocator at the start of the program, and use it through the BufferAllocator interface. Allocators implement +AutoCloseable and must be closed after the application is done with them; this will check that all outstanding memory +has been freed (see the next section). + +Arrow provides a tree-based model for memory allocation. The RootAllocator is created first, then more allocators +are created as children of an existing allocator via `newChildAllocator`_. When creating a RootAllocator or a child +allocator, a memory limit is provided, and when allocating memory, the limit is checked. Furthermore, when allocating +memory from a child allocator, those allocations are also reflected in all parent allocators. Hence, the RootAllocator +effectively sets the program-wide memory limit, and serves as the master bookkeeper for all memory allocations. + +Child allocators are not strictly required, but can help better organize code. For instance, a lower memory limit can +be set for a particular section of code. The child allocator can be closed when that section completes, +at which point it checks that that section didn't leak any memory. +Child allocators can also be named, which makes it easier to tell where an ArrowBuf came from during debugging. + +Reference counting +------------------ + +Because direct memory is expensive to allocate and deallocate, allocators may share direct buffers. To manage shared buffers +deterministically, we use manual reference counting instead of the garbage collector. +This simply means that each buffer has a counter keeping track of the number of references to +the buffer, and the user is responsible for properly incrementing/decrementing the counter as the buffer is used. + +In Arrow, each ArrowBuf has an associated `ReferenceManager`_ that tracks the reference count. You can retrieve +it with ArrowBuf.getReferenceManager(). The reference count is updated using `ReferenceManager.release`_ to decrement the count, +and `ReferenceManager.retain`_ to increment it. + +Of course, this is tedious and error-prone, so instead of directly working with buffers, we typically use +higher-level APIs like ValueVector. Such classes generally implement Closeable/AutoCloseable and will automatically +decrement the reference count when closed. + +Allocators implement AutoCloseable as well. In this case, closing the allocator will check that all buffers +obtained from the allocator are closed. If not, ``close()`` method will raise an exception; this helps track +memory leaks from unclosed buffers. + +Reference counting needs to be handled carefully. To ensure that an +independent section of code has fully cleaned up all allocated buffers, use a new child allocator. + +Development Guidelines +---------------------- + +Applications should generally: + +* Use the BufferAllocator interface in APIs instead of RootAllocator. +* Create one RootAllocator at the start of the program and explicitly pass it when needed. +* ``close()`` allocators after use (whether they are child allocators or the RootAllocator), either manually or preferably via a try-with-resources statement. + + +Debugging Memory Leaks/Allocation +--------------------------------- + +In ``DEBUG`` mode, the allocator and supporting classes will record additional +debug tracking information to better track down memory leaks and issues. To +enable DEBUG mode pass the following system property to the VM when starting +``-Darrow.memory.debug.allocator=true``. + +When DEBUG is enabled, a log will be kept of allocations. Configure SLF4J to see these logs (e.g. via Logback/Apache Log4j). +Consider the following example to see how it helps us with the tracking of allocators: + +.. code-block:: Java + + import org.apache.arrow.memory.ArrowBuf; + import org.apache.arrow.memory.BufferAllocator; + import org.apache.arrow.memory.RootAllocator; + + try (BufferAllocator bufferAllocator = new RootAllocator(8 * 1024)) { + ArrowBuf arrowBuf = bufferAllocator.buffer(4 * 1024); + System.out.println(arrowBuf); + } + +Without the debug mode enabled, when we close the allocator, we get this: + +.. code-block:: shell + + 11:56:48.944 [main] INFO o.apache.arrow.memory.BaseAllocator - Debug mode disabled. + ArrowBuf[2], address:140508391276544, length:4096 + 16:28:08.847 [main] ERROR o.apache.arrow.memory.BaseAllocator - Memory was leaked by query. Memory leaked: (4096) + Allocator(ROOT) 0/4096/4096/8192 (res/actual/peak/limit) + +Enabling the debug mode, we get more details: + +.. code-block:: shell + + 11:56:48.944 [main] INFO o.apache.arrow.memory.BaseAllocator - Debug mode enabled. + ArrowBuf[2], address:140437894463488, length:4096 + Exception in thread "main" java.lang.IllegalStateException: Allocator[ROOT] closed with outstanding buffers allocated (1). + Allocator(ROOT) 0/4096/4096/8192 (res/actual/peak/limit) + child allocators: 0 + ledgers: 1 + ledger[1] allocator: ROOT), isOwning: , size: , references: 1, life: 261438177096661..0, allocatorManager: [, life: ] holds 1 buffers. + ArrowBuf[2], address:140437894463488, length:4096 + reservations: 0 + +Additionally, in debug mode, `ArrowBuf.print()`_ can be used to obtain a debug string. +This will include information about allocation operations on the buffer with stack traces, such as when/where the buffer was allocated. + +.. code-block:: java + + import org.apache.arrow.memory.ArrowBuf; + import org.apache.arrow.memory.BufferAllocator; + import org.apache.arrow.memory.RootAllocator; + + try (final BufferAllocator allocator = new RootAllocator()) { + try (final ArrowBuf buf = allocator.buffer(1024)) { + final StringBuilder sb = new StringBuilder(); + buf.print(sb, /*indent*/ 0); + System.out.println(sb.toString()); + } + } + +.. code-block:: text + + ArrowBuf[2], address:140433199984656, length:1024 + event log for: ArrowBuf[2] + 675959093395667 create() + at org.apache.arrow.memory.util.HistoricalLog$Event.(HistoricalLog.java:175) + at org.apache.arrow.memory.util.HistoricalLog.recordEvent(HistoricalLog.java:83) + at org.apache.arrow.memory.ArrowBuf.(ArrowBuf.java:96) + at org.apache.arrow.memory.BufferLedger.newArrowBuf(BufferLedger.java:271) + at org.apache.arrow.memory.BaseAllocator.bufferWithoutReservation(BaseAllocator.java:300) + at org.apache.arrow.memory.BaseAllocator.buffer(BaseAllocator.java:276) + at org.apache.arrow.memory.RootAllocator.buffer(RootAllocator.java:29) + at org.apache.arrow.memory.BaseAllocator.buffer(BaseAllocator.java:240) + at org.apache.arrow.memory.RootAllocator.buffer(RootAllocator.java:29) + at REPL.$JShell$14.do_it$($JShell$14.java:10) + at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(NativeMethodAccessorImpl.java:-2) + at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:566) + at jdk.jshell.execution.DirectExecutionControl.invoke(DirectExecutionControl.java:209) + at jdk.jshell.execution.RemoteExecutionControl.invoke(RemoteExecutionControl.java:116) + at jdk.jshell.execution.DirectExecutionControl.invoke(DirectExecutionControl.java:119) + at jdk.jshell.execution.ExecutionControlForwarder.processCommand(ExecutionControlForwarder.java:144) + at jdk.jshell.execution.ExecutionControlForwarder.commandLoop(ExecutionControlForwarder.java:262) + at jdk.jshell.execution.Util.forwardExecutionControl(Util.java:76) + at jdk.jshell.execution.Util.forwardExecutionControlAndIO(Util.java:137) + at jdk.jshell.execution.RemoteExecutionControl.main(RemoteExecutionControl.java:70) + +The BufferAllocator also provides a ``BufferAllocator.toVerboseString()`` which can be used in +``DEBUG`` mode to get extensive stacktrace information and events associated with various Allocator behaviors. + +Finally, enabling the ``TRACE`` logging level will automatically provide this stack trace when the allocator is closed: + +.. code-block:: java + + // Assumes use of Logback; adjust for Log4j, etc. as appropriate + import ch.qos.logback.classic.Level; + import ch.qos.logback.classic.Logger; + import org.apache.arrow.memory.ArrowBuf; + import org.apache.arrow.memory.BufferAllocator; + import org.apache.arrow.memory.RootAllocator; + import org.slf4j.LoggerFactory; + + // Set log level to TRACE to get tracebacks + ((Logger) LoggerFactory.getLogger("org.apache.arrow")).setLevel(Level.TRACE); + try (final BufferAllocator allocator = new RootAllocator()) { + // Leak buffer + allocator.buffer(1024); + } + +.. code-block:: text + + | Exception java.lang.IllegalStateException: Allocator[ROOT] closed with outstanding buffers allocated (1). + Allocator(ROOT) 0/1024/1024/9223372036854775807 (res/actual/peak/limit) + child allocators: 0 + ledgers: 1 + ledger[1] allocator: ROOT), isOwning: , size: , references: 1, life: 712040870231544..0, allocatorManager: [, life: ] holds 1 buffers. + ArrowBuf[2], address:139926571810832, length:1024 + event log for: ArrowBuf[2] + 712040888650134 create() + at org.apache.arrow.memory.util.StackTrace.(StackTrace.java:34) + at org.apache.arrow.memory.util.HistoricalLog$Event.(HistoricalLog.java:175) + at org.apache.arrow.memory.util.HistoricalLog.recordEvent(HistoricalLog.java:83) + at org.apache.arrow.memory.ArrowBuf.(ArrowBuf.java:96) + at org.apache.arrow.memory.BufferLedger.newArrowBuf(BufferLedger.java:271) + at org.apache.arrow.memory.BaseAllocator.bufferWithoutReservation(BaseAllocator.java:300) + at org.apache.arrow.memory.BaseAllocator.buffer(BaseAllocator.java:276) + at org.apache.arrow.memory.RootAllocator.buffer(RootAllocator.java:29) + at org.apache.arrow.memory.BaseAllocator.buffer(BaseAllocator.java:240) + at org.apache.arrow.memory.RootAllocator.buffer(RootAllocator.java:29) + at REPL.$JShell$18.do_it$($JShell$18.java:13) + at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(NativeMethodAccessorImpl.java:-2) + at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:566) + at jdk.jshell.execution.DirectExecutionControl.invoke(DirectExecutionControl.java:209) + at jdk.jshell.execution.RemoteExecutionControl.invoke(RemoteExecutionControl.java:116) + at jdk.jshell.execution.DirectExecutionControl.invoke(DirectExecutionControl.java:119) + at jdk.jshell.execution.ExecutionControlForwarder.processCommand(ExecutionControlForwarder.java:144) + at jdk.jshell.execution.ExecutionControlForwarder.commandLoop(ExecutionControlForwarder.java:262) + at jdk.jshell.execution.Util.forwardExecutionControl(Util.java:76) + at jdk.jshell.execution.Util.forwardExecutionControlAndIO(Util.java:137) + + reservations: 0 + + | at BaseAllocator.close (BaseAllocator.java:405) + | at RootAllocator.close (RootAllocator.java:29) + | at (#8:1) + +Sometimes, explicitly passing allocators around is difficult. For example, it +can be hard to pass around extra state, like an allocator, through layers of +existing application or framework code. A global or singleton allocator instance +can be useful here, though it should not be your first choice. + +How this works: + +1. Set up a global allocator in a singleton class. +2. Provide methods to create child allocators from the global allocator. +3. Give child allocators proper names to make it easier to figure out where + allocations occurred in case of errors. +4. Ensure that resources are properly closed. +5. Check that the global allocator is empty at some suitable point, such as + right before program shutdown. +6. If it is not empty, review the above allocation bugs. + +.. code-block:: java + + //1 + private static final BufferAllocator allocator = new RootAllocator(); + private static final AtomicInteger childNumber = new AtomicInteger(0); + ... + //2 + public static BufferAllocator getChildAllocator() { + return allocator.newChildAllocator(nextChildName(), 0, Long.MAX_VALUE); + } + ... + //3 + private static String nextChildName() { + return "Allocator-Child-" + childNumber.incrementAndGet(); + } + ... + //4: Business code + try (BufferAllocator allocator = GlobalAllocator.getChildAllocator()) { + ... + } + ... + //5 + public static void checkGlobalCleanUpResources() { + ... + if (!allocator.getChildAllocators().isEmpty()) { + throw new IllegalStateException(...); + } else if (allocator.getAllocatedMemory() != 0) { + throw new IllegalStateException(...); + } + } + +.. _`ArrowBuf`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/ArrowBuf.html +.. _`ArrowBuf.print()`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/ArrowBuf.html#print-java.lang.StringBuilder-int-org.apache.arrow.memory.BaseAllocator.Verbosity- +.. _`BufferAllocator`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/BufferAllocator.html +.. _`BufferLedger`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/BufferLedger.html +.. _`RootAllocator`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/RootAllocator.html +.. _`newChildAllocator`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/RootAllocator.html#newChildAllocator-java.lang.String-org.apache.arrow.memory.AllocationListener-long-long- +.. _`Netty`: https://netty.io/wiki/ +.. _`sun.misc.unsafe`: https://web.archive.org/web/20210929024401/http://www.docjar.com/html/api/sun/misc/Unsafe.java.html +.. _`Direct Memory`: https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/nio/ByteBuffer.html +.. _`ReferenceManager`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/ReferenceManager.html +.. _`ReferenceManager.release`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/ReferenceManager.html#release-- +.. _`ReferenceManager.retain`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/ReferenceManager.html#retain-- + +Arrow Memory In-Depth +===================== + +Design Principles +----------------- +Arrow’s memory model is based on the following basic concepts: + +- Memory can be allocated up to some limit. That limit could be a real + limit (OS/JVM) or a locally imposed limit. +- Allocation operates in two phases: accounting then actual allocation. + Allocation could fail at either point. +- Allocation failure should be recoverable. In all cases, the Allocator + infrastructure should expose memory allocation failures (OS or + internal limit-based) as ``OutOfMemoryException``\ s. +- Any allocator can reserve memory when created. This memory shall be + held such that this allocator will always be able to allocate that + amount of memory. +- A particular application component should work to use a local + allocator to understand local memory usage and better debug memory + leaks. +- The same physical memory can be shared by multiple allocators and the + allocator must provide an accounting paradigm for this purpose. + +Reserving Memory +---------------- + +Arrow provides two different ways to reserve memory: + +- BufferAllocator accounting reservations: When a new allocator (other + than the ``RootAllocator``) is initialized, it can set aside memory + that it will keep locally for its lifetime. This is memory that will + never be released back to its parent allocator until the allocator is + closed. +- ``AllocationReservation`` via BufferAllocator.newReservation(): + Allows a short-term preallocation strategy so that a particular + subsystem can ensure future memory is available to support a + particular request. + +Reference Counting Details +-------------------------- + +Typically, the ReferenceManager implementation used is an instance of `BufferLedger`_. +A BufferLedger is a ReferenceManager that also maintains the relationship between an ``AllocationManager``, +a ``BufferAllocator`` and one or more individual ``ArrowBuf``\ s + +All ArrowBufs (direct or sliced) related to a single BufferLedger/BufferAllocator combination +share the same reference count and either all will be valid or all will be invalid. +For simplicity of accounting, we treat that memory as being used by one +of the BufferAllocators associated with the memory. When that allocator +releases its claim on that memory, the memory ownership is then moved to +another BufferLedger belonging to the same AllocationManager. + +Allocation Details +------------------ + +There are several Allocator types in Arrow Java: + +- ``BufferAllocator`` - The public interface application users should be leveraging +- ``BaseAllocator`` - The base implementation of memory allocation, contains the meat of the Arrow allocator implementation +- ``RootAllocator`` - The root allocator. Typically only one created for a JVM. It serves as the parent/ancestor for child allocators +- ``ChildAllocator`` - A child allocator that derives from the root allocator + +Many BufferAllocators can reference the same piece of physical memory at the same +time. It is the AllocationManager’s responsibility to ensure that in this situation, +all memory is accurately accounted for from the Root’s perspective +and also to ensure that the memory is correctly released once all +BufferAllocators have stopped using that memory. + +For simplicity of accounting, we treat that memory as being used by one +of the BufferAllocators associated with the memory. When that allocator +releases its claim on that memory, the memory ownership is then moved to +another BufferLedger belonging to the same AllocationManager. Note that +because a ArrowBuf.release() is what actually causes memory ownership +transfer to occur, we always proceed with ownership transfer (even if +that violates an allocator limit). It is the responsibility of the +application owning a particular allocator to frequently confirm whether +the allocator is over its memory limit (BufferAllocator.isOverLimit()) +and if so, attempt to aggressively release memory to ameliorate the +situation. + + +Object Hierarchy +---------------- + +There are two main ways that someone can look at the object hierarchy +for Arrow’s memory management scheme. The first is a memory based +perspective as below: + +Memory Perspective +~~~~~~~~~~~~~~~~~~ + +.. code-block:: none + + + AllocationManager + | + |-- UnsignedDirectLittleEndian (One per AllocationManager) + | + |-+ BufferLedger 1 ==> Allocator A (owning) + | ` - ArrowBuf 1 + |-+ BufferLedger 2 ==> Allocator B (non-owning) + | ` - ArrowBuf 2 + |-+ BufferLedger 3 ==> Allocator C (non-owning) + | - ArrowBuf 3 + | - ArrowBuf 4 + ` - ArrowBuf 5 + +In this picture, a piece of memory is owned by an allocator manager. An +allocator manager is responsible for that piece of memory no matter +which allocator(s) it is working with. An allocator manager will have +relationships with a piece of raw memory (via its reference to +UnsignedDirectLittleEndian) as well as references to each +BufferAllocator it has a relationship to. + +Allocator Perspective +~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: none + + + RootAllocator + |-+ ChildAllocator 1 + | | - ChildAllocator 1.1 + | ` ... + | + |-+ ChildAllocator 2 + |-+ ChildAllocator 3 + | | + | |-+ BufferLedger 1 ==> AllocationManager 1 (owning) ==> UDLE + | | `- ArrowBuf 1 + | `-+ BufferLedger 2 ==> AllocationManager 2 (non-owning)==> UDLE + | `- ArrowBuf 2 + | + |-+ BufferLedger 3 ==> AllocationManager 1 (non-owning)==> UDLE + | ` - ArrowBuf 3 + |-+ BufferLedger 4 ==> AllocationManager 2 (owning) ==> UDLE + | - ArrowBuf 4 + | - ArrowBuf 5 + ` - ArrowBuf 6 + +In this picture, a RootAllocator owns three ChildAllocators. The first +ChildAllocator (ChildAllocator 1) owns a subsequent ChildAllocator. +ChildAllocator has two BufferLedgers/AllocationManager references. +Coincidentally, each of these AllocationManager’s is also associated +with the RootAllocator. In this case, one of the these +AllocationManagers is owned by ChildAllocator 3 (AllocationManager 1) +while the other AllocationManager (AllocationManager 2) is +owned/accounted for by the RootAllocator. Note that in this scenario, +ArrowBuf 1 is sharing the underlying memory as ArrowBuf 3. However the +subset of that memory (e.g. through slicing) might be different. Also +note that ArrowBuf 2 and ArrowBuf 4, 5 and 6 are also sharing the same +underlying memory. Also note that ArrowBuf 4, 5 and 6 all share the same +reference count and fate. diff --git a/docs/source/overview.rst b/docs/source/overview.rst new file mode 100644 index 0000000000..be579c1495 --- /dev/null +++ b/docs/source/overview.rst @@ -0,0 +1,90 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +=================== +High-Level Overview +=================== + +The Apache Arrow Java modules implement various specifications including the +columnar format and IPC. Most modules are native Java implementations, +but some modules are JNI bindings to the C++ library. + +.. list-table:: Arrow Java Modules + :widths: 25 50 25 + :header-rows: 1 + + * - Module + - Description + - Implementation + * - arrow-format + - Generated Java files from the IPC Flatbuffer definitions. + - Native + * - arrow-memory-core + - Core off-heap memory management libraries for Arrow ValueVectors. + - Native + * - arrow-memory-unsafe + - Memory management implementation based on sun.misc.Unsafe. + - Native + * - arrow-memory-netty + - Memory management implementation based on Netty. + - Native + * - arrow-vector + - An off-heap reference implementation for Arrow columnar data format. + - Native + * - arrow-tools + - Java applications for working with Arrow ValueVectors. + - Native + * - arrow-jdbc + - (Experimental) A library for converting JDBC data to Arrow data. + - Native + * - flight-core + - An RPC mechanism for transferring ValueVectors. + - Native + * - flight-sql + - Contains utility classes to expose Flight SQL semantics for clients and servers over Arrow Flight. + - Native + * - flight-integration-tests + - Integration tests for Flight RPC. + - Native + * - arrow-performance + - JMH benchmarks for the Arrow libraries. + - Native + * - arrow-algorithm + - (Experimental) A collection of algorithms for working with ValueVectors. + - Native + * - arrow-avro + - (Experimental) A library for converting Avro data to Arrow data. + - Native + * - arrow-compression + - (Experimental) A library for working with compression/decompression of Arrow data. + - Native + * - arrow-c-data + - Java implementation of `C Data Interface`_ + - JNI + * - arrow-orc + - (Experimental) A JNI wrapper for the C++ ORC reader implementation. + - JNI + * - arrow-gandiva + - Java wrappers around the native Gandiva SQL expression compiler. + - JNI + * - arrow-dataset + - Java bindings to the Arrow Datasets library. + - JNI + +Arrow Java modules support working with data (1) in-memory, (2) at rest, and (3) on-the-wire. + +.. _`C Data Interface`: https://arrow.apache.org/docs/format/CDataInterface.html diff --git a/docs/source/quickstartguide.rst b/docs/source/quickstartguide.rst new file mode 100644 index 0000000000..adb07d7002 --- /dev/null +++ b/docs/source/quickstartguide.rst @@ -0,0 +1,314 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +================= +Quick Start Guide +================= + +Arrow Java provides several building blocks. Data types describe the types of values; +ValueVectors are sequences of typed values; fields describe the types of columns in +tabular data; schemas describe a sequence of columns in tabular data, and +VectorSchemaRoot represents tabular data. Arrow also provides readers and +writers for loading data from and persisting data to storage. + +Create a ValueVector +******************** + +**ValueVectors** represent a sequence of values of the same type. +They are also known as "arrays" in the columnar format. + +Example: create a vector of 32-bit integers representing ``[1, null, 2]``: + +.. code-block:: Java + + import org.apache.arrow.memory.BufferAllocator; + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.IntVector; + + try( + BufferAllocator allocator = new RootAllocator(); + IntVector intVector = new IntVector("fixed-size-primitive-layout", allocator); + ){ + intVector.allocateNew(3); + intVector.set(0,1); + intVector.setNull(1); + intVector.set(2,2); + intVector.setValueCount(3); + System.out.println("Vector created in memory: " + intVector); + } + +.. code-block:: shell + + Vector created in memory: [1, null, 2] + + +Example: create a vector of UTF-8 encoded strings representing ``["one", "two", "three"]``: + +.. code-block:: Java + + import org.apache.arrow.memory.BufferAllocator; + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.VarCharVector; + + try( + BufferAllocator allocator = new RootAllocator(); + VarCharVector varCharVector = new VarCharVector("variable-size-primitive-layout", allocator); + ){ + varCharVector.allocateNew(3); + varCharVector.set(0, "one".getBytes()); + varCharVector.set(1, "two".getBytes()); + varCharVector.set(2, "three".getBytes()); + varCharVector.setValueCount(3); + System.out.println("Vector created in memory: " + varCharVector); + } + +.. code-block:: shell + + Vector created in memory: [one, two, three] + +Create a Field +************** + +**Fields** are used to denote the particular columns of tabular data. +They consist of a name, a data type, a flag indicating whether the column can have null values, +and optional key-value metadata. + +Example: create a field named "document" of string type: + +.. code-block:: Java + + import org.apache.arrow.vector.types.pojo.ArrowType; + import org.apache.arrow.vector.types.pojo.Field; + import org.apache.arrow.vector.types.pojo.FieldType; + import java.util.HashMap; + import java.util.Map; + + Map metadata = new HashMap<>(); + metadata.put("A", "Id card"); + metadata.put("B", "Passport"); + metadata.put("C", "Visa"); + Field document = new Field("document", + new FieldType(true, new ArrowType.Utf8(), /*dictionary*/ null, metadata), + /*children*/ null); + System.out.println("Field created: " + document + ", Metadata: " + document.getMetadata()); + +.. code-block:: shell + + Field created: document: Utf8, Metadata: {A=Id card, B=Passport, C=Visa} + +Create a Schema +*************** + +**Schemas** hold a sequence of fields together with some optional metadata. + +Example: Create a schema describing datasets with two columns: +an int32 column "A" and a UTF8-encoded string column "B" + +.. code-block:: Java + + import org.apache.arrow.vector.types.pojo.ArrowType; + import org.apache.arrow.vector.types.pojo.Field; + import org.apache.arrow.vector.types.pojo.FieldType; + import org.apache.arrow.vector.types.pojo.Schema; + import java.util.HashMap; + import java.util.Map; + import static java.util.Arrays.asList; + + Map metadata = new HashMap<>(); + metadata.put("K1", "V1"); + metadata.put("K2", "V2"); + Field a = new Field("A", FieldType.nullable(new ArrowType.Int(32, true)), /*children*/ null); + Field b = new Field("B", FieldType.nullable(new ArrowType.Utf8()), /*children*/ null); + Schema schema = new Schema(asList(a, b), metadata); + System.out.println("Schema created: " + schema); + +.. code-block:: shell + + Schema created: Schema(metadata: {K1=V1, K2=V2}) + +Create a VectorSchemaRoot +************************* + +A **VectorSchemaRoot** combines ValueVectors with a Schema to represent tabular data. + +Example: Create a dataset of names (strings) and ages (32-bit signed integers). + +.. code-block:: Java + + import org.apache.arrow.memory.BufferAllocator; + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VarCharVector; + import org.apache.arrow.vector.VectorSchemaRoot; + import org.apache.arrow.vector.types.pojo.ArrowType; + import org.apache.arrow.vector.types.pojo.Field; + import org.apache.arrow.vector.types.pojo.FieldType; + import org.apache.arrow.vector.types.pojo.Schema; + import java.nio.charset.StandardCharsets; + import java.util.HashMap; + import java.util.Map; + import static java.util.Arrays.asList; + + Field age = new Field("age", + FieldType.nullable(new ArrowType.Int(32, true)), + /*children*/null + ); + Field name = new Field("name", + FieldType.nullable(new ArrowType.Utf8()), + /*children*/null + ); + Schema schema = new Schema(asList(age, name), /*metadata*/ null); + try( + BufferAllocator allocator = new RootAllocator(); + VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator); + IntVector ageVector = (IntVector) root.getVector("age"); + VarCharVector nameVector = (VarCharVector) root.getVector("name"); + ){ + ageVector.allocateNew(3); + ageVector.set(0, 10); + ageVector.set(1, 20); + ageVector.set(2, 30); + nameVector.allocateNew(3); + nameVector.set(0, "Dave".getBytes(StandardCharsets.UTF_8)); + nameVector.set(1, "Peter".getBytes(StandardCharsets.UTF_8)); + nameVector.set(2, "Mary".getBytes(StandardCharsets.UTF_8)); + root.setRowCount(3); + System.out.println("VectorSchemaRoot created: \n" + root.contentToTSVString()); + } + +.. code-block:: shell + + VectorSchemaRoot created: + age name + 10 Dave + 20 Peter + 30 Mary + + +Interprocess Communication (IPC) +******************************** + +Arrow data can be written to and read from disk, and both of these can be done in +a streaming and/or random-access fashion depending on application requirements. + +**Write data to an arrow file** + +Example: Write the dataset from the previous example to an Arrow IPC file (random-access). + +.. code-block:: Java + + import org.apache.arrow.memory.BufferAllocator; + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VarCharVector; + import org.apache.arrow.vector.VectorSchemaRoot; + import org.apache.arrow.vector.ipc.ArrowFileWriter; + import org.apache.arrow.vector.types.pojo.ArrowType; + import org.apache.arrow.vector.types.pojo.Field; + import org.apache.arrow.vector.types.pojo.FieldType; + import org.apache.arrow.vector.types.pojo.Schema; + import java.io.File; + import java.io.FileOutputStream; + import java.io.IOException; + import java.nio.charset.StandardCharsets; + import java.util.HashMap; + import java.util.Map; + import static java.util.Arrays.asList; + + Field age = new Field("age", + FieldType.nullable(new ArrowType.Int(32, true)), + /*children*/ null); + Field name = new Field("name", + FieldType.nullable(new ArrowType.Utf8()), + /*children*/ null); + Schema schema = new Schema(asList(age, name)); + try( + BufferAllocator allocator = new RootAllocator(); + VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator); + IntVector ageVector = (IntVector) root.getVector("age"); + VarCharVector nameVector = (VarCharVector) root.getVector("name"); + ){ + ageVector.allocateNew(3); + ageVector.set(0, 10); + ageVector.set(1, 20); + ageVector.set(2, 30); + nameVector.allocateNew(3); + nameVector.set(0, "Dave".getBytes(StandardCharsets.UTF_8)); + nameVector.set(1, "Peter".getBytes(StandardCharsets.UTF_8)); + nameVector.set(2, "Mary".getBytes(StandardCharsets.UTF_8)); + root.setRowCount(3); + File file = new File("random_access_file.arrow"); + try ( + FileOutputStream fileOutputStream = new FileOutputStream(file); + ArrowFileWriter writer = new ArrowFileWriter(root, /*provider*/ null, fileOutputStream.getChannel()); + ) { + writer.start(); + writer.writeBatch(); + writer.end(); + System.out.println("Record batches written: " + writer.getRecordBlocks().size() + + ". Number of rows written: " + root.getRowCount()); + } catch (IOException e) { + e.printStackTrace(); + } + } + +.. code-block:: shell + + Record batches written: 1. Number of rows written: 3 + +**Read data from an arrow file** + +Example: Read the dataset from the previous example from an Arrow IPC file (random-access). + +.. code-block:: Java + + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.ipc.ArrowFileReader; + import org.apache.arrow.vector.ipc.message.ArrowBlock; + import org.apache.arrow.vector.VectorSchemaRoot; + import java.io.File; + import java.io.FileInputStream; + import java.io.FileOutputStream; + import java.io.IOException; + + try( + BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + FileInputStream fileInputStream = new FileInputStream(new File("random_access_file.arrow")); + ArrowFileReader reader = new ArrowFileReader(fileInputStream.getChannel(), allocator); + ){ + System.out.println("Record batches in file: " + reader.getRecordBlocks().size()); + for (ArrowBlock arrowBlock : reader.getRecordBlocks()) { + reader.loadRecordBatch(arrowBlock); + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + System.out.println("VectorSchemaRoot read: \n" + root.contentToTSVString()); + } + } catch (IOException e) { + e.printStackTrace(); + } + +.. code-block:: shell + + Record batches in file: 1 + VectorSchemaRoot read: + age name + 10 Dave + 20 Peter + 30 Mary + +More examples available at `Arrow Java Cookbook`_. + +.. _`Arrow Java Cookbook`: https://arrow.apache.org/cookbook/java diff --git a/docs/source/reference/index.rst b/docs/source/reference/index.rst new file mode 100644 index 0000000000..523ac0c7f7 --- /dev/null +++ b/docs/source/reference/index.rst @@ -0,0 +1,21 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Java Reference (javadoc) +======================== + +Stub page for the Java reference docs; actual source is located in the java/ directory. diff --git a/docs/source/substrait.rst b/docs/source/substrait.rst new file mode 100644 index 0000000000..b3678ac815 --- /dev/null +++ b/docs/source/substrait.rst @@ -0,0 +1,201 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +========= +Substrait +========= + +The ``arrow-dataset`` module can execute Substrait_ plans via the :external+arrow:doc:`Acero ` +query engine. + +Executing Queries Using Substrait Plans +======================================= + +Plans can reference data in files via URIs, or "named tables" that must be provided along with the plan. + +Here is an example of a Java program that queries a Parquet file using Java Substrait +(this example use `Substrait Java`_ project to compile a SQL query to a Substrait plan): + +.. code-block:: Java + + import com.google.common.collect.ImmutableList; + import io.substrait.isthmus.SqlToSubstrait; + import io.substrait.proto.Plan; + import org.apache.arrow.dataset.file.FileFormat; + import org.apache.arrow.dataset.file.FileSystemDatasetFactory; + import org.apache.arrow.dataset.jni.NativeMemoryPool; + import org.apache.arrow.dataset.scanner.ScanOptions; + import org.apache.arrow.dataset.scanner.Scanner; + import org.apache.arrow.dataset.source.Dataset; + import org.apache.arrow.dataset.source.DatasetFactory; + import org.apache.arrow.dataset.substrait.AceroSubstraitConsumer; + import org.apache.arrow.memory.BufferAllocator; + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.ipc.ArrowReader; + import org.apache.calcite.sql.parser.SqlParseException; + + import java.nio.ByteBuffer; + import java.util.HashMap; + import java.util.Map; + + public class ClientSubstrait { + public static void main(String[] args) { + String uri = "file:///data/tpch_parquet/nation.parquet"; + ScanOptions options = new ScanOptions(/*batchSize*/ 32768); + try ( + BufferAllocator allocator = new RootAllocator(); + DatasetFactory datasetFactory = new FileSystemDatasetFactory(allocator, NativeMemoryPool.getDefault(), + FileFormat.PARQUET, uri); + Dataset dataset = datasetFactory.finish(); + Scanner scanner = dataset.newScan(options); + ArrowReader reader = scanner.scanBatches() + ) { + // map table to reader + Map mapTableToArrowReader = new HashMap<>(); + mapTableToArrowReader.put("NATION", reader); + // get binary plan + Plan plan = getPlan(); + ByteBuffer substraitPlan = ByteBuffer.allocateDirect(plan.toByteArray().length); + substraitPlan.put(plan.toByteArray()); + // run query + try (ArrowReader arrowReader = new AceroSubstraitConsumer(allocator).runQuery( + substraitPlan, + mapTableToArrowReader + )) { + while (arrowReader.loadNextBatch()) { + System.out.println(arrowReader.getVectorSchemaRoot().contentToTSVString()); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + static Plan getPlan() throws SqlParseException { + String sql = "SELECT * from nation"; + String nation = "CREATE TABLE NATION (N_NATIONKEY BIGINT NOT NULL, N_NAME CHAR(25), " + + "N_REGIONKEY BIGINT NOT NULL, N_COMMENT VARCHAR(152))"; + SqlToSubstrait sqlToSubstrait = new SqlToSubstrait(); + Plan plan = sqlToSubstrait.execute(sql, ImmutableList.of(nation)); + return plan; + } + } + +.. code-block:: text + + // Results example: + FieldPath(0) FieldPath(1) FieldPath(2) FieldPath(3) + 0 ALGERIA 0 haggle. carefully final deposits detect slyly agai + 1 ARGENTINA 1 al foxes promise slyly according to the regular accounts. bold requests alon + +Executing Projections and Filters Using Extended Expressions +============================================================ + +Dataset also supports projections and filters with Substrait's `Extended Expression`_. +This requires the substrait-java library. + +This Java program: + +- Loads a Parquet file containing the "nation" table from the TPC-H benchmark. +- Applies a filter: + - ``N_NATIONKEY > 18`` +- Projects two new columns: + - ``N_REGIONKEY + 10`` + - ``N_NAME || ' - ' || N_COMMENT`` + + + +.. code-block:: Java + + import com.google.common.collect.ImmutableList; + import io.substrait.isthmus.SqlExpressionToSubstrait; + import io.substrait.proto.ExtendedExpression; + import org.apache.arrow.dataset.file.FileFormat; + import org.apache.arrow.dataset.file.FileSystemDatasetFactory; + import org.apache.arrow.dataset.jni.NativeMemoryPool; + import org.apache.arrow.dataset.scanner.ScanOptions; + import org.apache.arrow.dataset.scanner.Scanner; + import org.apache.arrow.dataset.source.Dataset; + import org.apache.arrow.dataset.source.DatasetFactory; + import org.apache.arrow.memory.BufferAllocator; + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.ipc.ArrowReader; + import org.apache.calcite.sql.parser.SqlParseException; + + import java.nio.ByteBuffer; + import java.util.Base64; + import java.util.Optional; + + public class ClientSubstraitExtendedExpressionsCookbook { + + public static void main(String[] args) throws SqlParseException { + projectAndFilterDataset(); + } + + private static void projectAndFilterDataset() throws SqlParseException { + String uri = "file:///Users/data/tpch_parquet/nation.parquet"; + ScanOptions options = + new ScanOptions.Builder(/*batchSize*/ 32768) + .columns(Optional.empty()) + .substraitFilter(getByteBuffer(new String[]{"N_NATIONKEY > 18"})) + .substraitProjection(getByteBuffer(new String[]{"N_REGIONKEY + 10", + "N_NAME || CAST(' - ' as VARCHAR) || N_COMMENT"})) + .build(); + try (BufferAllocator allocator = new RootAllocator(); + DatasetFactory datasetFactory = + new FileSystemDatasetFactory( + allocator, NativeMemoryPool.getDefault(), FileFormat.PARQUET, uri); + Dataset dataset = datasetFactory.finish(); + Scanner scanner = dataset.newScan(options); + ArrowReader reader = scanner.scanBatches()) { + while (reader.loadNextBatch()) { + System.out.println(reader.getVectorSchemaRoot().contentToTSVString()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static ByteBuffer getByteBuffer(String[] sqlExpression) throws SqlParseException { + String schema = + "CREATE TABLE NATION (N_NATIONKEY INT NOT NULL, N_NAME VARCHAR, " + + "N_REGIONKEY INT NOT NULL, N_COMMENT VARCHAR)"; + SqlExpressionToSubstrait expressionToSubstrait = new SqlExpressionToSubstrait(); + ExtendedExpression expression = + expressionToSubstrait.convert(sqlExpression, ImmutableList.of(schema)); + byte[] expressionToByte = + Base64.getDecoder().decode(Base64.getEncoder().encodeToString(expression.toByteArray())); + ByteBuffer byteBuffer = ByteBuffer.allocateDirect(expressionToByte.length); + byteBuffer.put(expressionToByte); + return byteBuffer; + } + } + +.. code-block:: text + + column-1 column-2 + 13 ROMANIA - ular asymptotes are about the furious multipliers. express dependencies nag above the ironically ironic account + 14 SAUDI ARABIA - ts. silent requests haggle. closely express packages sleep across the blithely + 12 VIETNAM - hely enticingly express accounts. even, final + 13 RUSSIA - requests against the platelets use never according to the quickly regular pint + 13 UNITED KINGDOM - eans boost carefully special requests. accounts are. carefull + 11 UNITED STATES - y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be + +.. _`Substrait`: https://substrait.io/ +.. _`Substrait Java`: https://github.com/substrait-io/substrait-java +.. _`Acero`: https://arrow.apache.org/docs/cpp/streaming_execution.html +.. _`Extended Expression`: https://github.com/substrait-io/substrait/blob/main/site/docs/expressions/extended_expression.md diff --git a/docs/source/table.rst b/docs/source/table.rst new file mode 100644 index 0000000000..5aa95e153c --- /dev/null +++ b/docs/source/table.rst @@ -0,0 +1,378 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +===== +Table +===== + +**NOTE**: The Table API is experimental and subject to change. See the list of limitations below. + +`Table`_ is an immutable tabular data structure based on `FieldVector`_. Like `VectorSchemaRoot`_, ``Table`` is a columnar data structure backed by Arrow arrays, or more specifically, by ``FieldVector`` objects. It differs from ``VectorSchemaRoot`` mainly in that it is fully immutable and lacks support for batch operations. Anyone processing batches of tabular data in a pipeline should continue to use ``VectorSchemaRoot``. Finally, the ``Table`` API is mainly row-oriented, so in some ways it's more like the JDBC API than the ``VectorSchemaRoot`` API, but you can still use ``FieldReaders`` to work with data in a columnar fashion. + +Mutation in Table and VectorSchemaRoot +====================================== + +``VectorSchemaRoot`` provides a thin wrapper on the vectors that hold its data. Individual vectors can be retrieved from a vector schema root. These vectors have *setters* for modifying their elements, making ``VectorSchemaRoot`` immutable only by convention. The protocol for mutating a vector is documented in the `ValueVector`_ interface: + +- values need to be written in order (e.g. index 0, 1, 2, 5) +- null vectors start with all values as null before writing anything +- for variable width types, the offset vector should be all zeros before writing +- you must call setValueCount before a vector can be read +- you should never write to a vector once it has been read. + +The rules aren't enforced by the API so the programmer is responsible for ensuring that they are followed. Failure to do so could lead to runtime exceptions. + +``Table``, on the other hand, is immutable. The underlying vectors are not exposed. When a table is created from existing vectors, their memory is transferred to new vectors, so subsequent changes to the original vectors can't impact the new table's values. + +Features and limitations +====================================== + +A basic set of table functionality is currently available: + +- Create a table from vectors or ``VectorSchemaRoot`` +- Iterate tables by row, or set the current row index directly +- Access vector values as primitives, objects, and/or nullable `ValueHolder`_ instances (depending on type) +- Get a ``FieldReader`` for any vector +- Add and remove vectors, creating new tables +- Encode and decode a table's vectors using dictionary encoding +- Export table data for use by native code +- Print representative data to TSV strings +- Get a table's schema +- Slice tables +- Convert table to ``VectorSchemaRoot`` + +Limitations in the 11.0.0 release: + +- No support ``ChunkedArray`` or any form of row-group. Support for chunked arrays or row groups will be considered for a future release. +- No support for the C-Stream API. Support for the streaming API is contingent on chunked array support +- No support for creating tables directly from Java POJOs. All data held by a table must be imported via a ``VectorSchemaRoot``, or from collections or arrays of vectors. + +The Table API +============= + +Like ``VectorSchemaRoot``, a table contains a `Schema`_ and an ordered collection of ``FieldVector`` objects, but it is designed to be accessed via a row-oriented interface. + +Creating a Table from a VectorSchemaRoot +**************************************** + +Tables are created from a ``VectorSchemaRoot`` as shown below. The memory buffers holding the data are transferred from the vector schema root to new vectors in the new table, clearing the source vectors in the process. This ensures that the data in your new table is never changed. Since the buffers are transferred rather than copied, this is a very low overhead operation. + +.. code-block:: Java + + Table t = new Table(someVectorSchemaRoot); + +If you now update the vectors held by the ``VectorSchemaRoot`` (using some version of ``ValueVector#setSafe()``), it would reflect those changes, but the values in table *t* are unchanged. + +Creating a Table from FieldVectors +********************************** + +Tables can be created from ``FieldVectors`` as shown below, using 'var-arg' array arguments: + +.. code-block:: Java + + IntVector myVector = createMyIntVector(); + VectorSchemaRoot vsr1 = new VectorSchemaRoot(myVector); + +or by passing a collection: + +.. code-block:: Java + + IntVector myVector = createMyIntVector(); + List fvList = List.of(myVector); + VectorSchemaRoot vsr1 = new VectorSchemaRoot(fvList); + +It is rarely a good idea to share vectors between multiple vector schema roots, and it would not be a good idea to share them between vector schema roots and tables. Creating a ``VectorSchemaRoot`` from a list of vectors does not cause the reference counts for the vectors to be incremented. Unless you manage the counts manually, the code below would lead to more references than reference counts, and that could lead to trouble. There is an implicit assumption that the vectors were created for use by *one* ``VectorSchemaRoot`` that this code violates. + +*Don't do this:* + +.. code-block:: Java + + IntVector myVector = createMyIntVector(); // Reference count for myVector = 1 + VectorSchemaRoot vsr1 = new VectorSchemaRoot(myVector); // Still one reference + VectorSchemaRoot vsr2 = new VectorSchemaRoot(myVector); + // Ref count is still one, but there are two VSRs with a reference to myVector + vsr2.clear(); // Reference count for myVector is 0. + +What is happening is that the reference counter works at a lower level than the ``VectorSchemaRoot`` interface. A reference counter counts references to `ArrowBuf`_ instances that control memory buffers. It doesn't count references to the vectors that hold those ArrowBufs. In the example above, each ``ArrowBuf`` is held by one vector, so there is only one reference. This distinction is blurred when you call the ``VectorSchemaRoot``'s clear() method, which frees the memory held by each of the vectors it references even though another instance references the same vectors. + +When you create tables from vectors, it's assumed that there are no external references to those vectors. To be certain, the buffers underlying these vectors are transferred to new vectors in the new table, and the original vectors are cleared. + +*Don't do this either, but note the difference from above:* + +.. code-block:: Java + + IntVector myVector = createMyIntVector(); // Reference count for myVector = 1 + Table t1 = new Table(myVector); + // myVector is cleared; Table t1 has a new hidden vector with the data from myVector + Table t2 = new Table(myVector); + // t2 has no rows because myVector was just cleared + // t1 continues to have the data from the original vector + t2.clear(); + // no change because t2 is already empty and t1 is independent + +With tables, memory is explicitly transferred on instantiation so the buffers held by a table are held by *only* that table. + +Creating Tables with dictionary-encoded vectors +*********************************************** + +Another point of difference is that ``VectorSchemaRoot`` is uninformed about any dictionary-encoding of its vectors, while tables hold an optional `DictionaryProvider`_ instance. If any vectors in the source data are encoded, a DictionaryProvider must be set to un-encode the values. + +.. code-block:: Java + + VectorSchemaRoot vsr = myVsr(); + DictionaryProvider provider = myProvider(); + Table t = new Table(vsr, provider); + +In ``Table``, dictionaries are used like they are with vectors. To decode a vector, the user provides the name of the vector to decode and the dictionary id: + +.. code-block:: Java + + Table t = new Table(vsr, provider); + ValueVector decodedName = t.decode("name", 1L); + +To encode a vector from a table, a similar approach is used: + +.. code-block:: Java + + Table t = new Table(vsr, provider); + ValueVector encodedName = t.encode("name", 1L); + +Freeing memory explicitly +************************* + +Tables use off-heap memory that must be freed when it is no longer needed. ``Table`` implements ``AutoCloseable`` so the best way to create one is in a try-with-resources block: + +.. code-block:: Java + + try (VectorSchemaRoot vsr = myMethodForGettingVsrs(); + Table t = new Table(vsr)) { + // do useful things. + } + +If you don't use a try-with-resources block, you must close the table manually: + +.. code-block:: Java + + try { + VectorSchemaRoot vsr = myMethodForGettingVsrs(); + Table t = new Table(vsr); + // do useful things. + } finally { + vsr.close(); + t.close(); + } + +Manual closing should be performed in a finally block. + +Getting the schema +****************** + +You get the table's schema just as you would with a vector schema root: + +.. code-block:: Java + + Schema s = table.getSchema(); + +Adding and removing vectors +*************************** + +``Table`` provides facilities for adding and removing vectors modeled on the same functionality in ``VectorSchemaRoot``. These operations return new instances rather than modifying the original instance in-place. + +.. code-block:: Java + + try (Table t = new Table(vectorList)) { + IntVector v3 = new IntVector("3", intFieldType, allocator); + Table t2 = t.addVector(2, v3); + Table t3 = t2.removeVector(1); + // don't forget to close t2 and t3 + } + +Slicing tables +************** + +``Table`` supports *slice()* operations, where a slice of a source table is a second Table that refers to a single, contiguous range of rows in the source. + +.. code-block:: Java + + try (Table t = new Table(vectorList)) { + Table t2 = t.slice(100, 200); // creates a slice referencing the values in range (100, 200] + ... + } + +This raises the question: If you create a slice with *all* the values in the source table (as shown below), how would that differ from a new Table constructed with the same vectors as the source? + +.. code-block:: Java + + try (Table t = new Table(vectorList)) { + Table t2 = t.slice(0, t.getRowCount()); // creates a slice referencing all the values in t + // ... + } + +The difference is that when you *construct* a new table, the buffers are transferred from the source vectors to new vectors in the destination. With a slice, both tables share the same underlying vectors. That's OK, though, since both tables are immutable. + +Using FieldReaders +****************** + +You can get a `FieldReader`_ for any vector in the Table passing either the `Field`_, vector index, or vector name as an argument. The signatures are the same as in ``VectorSchemaRoot``. + +.. code-block:: Java + + FieldReader nameReader = table.getReader("user_name"); + +Row operations +************** + +Row-based access is supported by the `Row`_ object. ``Row`` provides *get()* methods by both vector name and vector position, but no *set()* operations. + +It is important to recognize that rows are NOT reified as objects, but rather operate like a cursor where the data from numerous logical rows in the table can be viewed (one at a time) using the same ``Row`` instance. See "Moving from row-to-row" below for information about navigating through the table. + +Getting a row +************* + +Calling ``immutableRow()`` on any table instance returns a new ``Row`` instance. + +.. code-block:: Java + + Row r = table.immutableRow(); + +Moving from row-to-row +********************** + +Since rows are iterable, you can traverse a table using a standard while loop: + +.. code-block:: Java + + Row r = table.immutableRow(); + while (r.hasNext()) { + r.next(); + // do something useful here + } + +``Table`` implements ``Iterable`` so you can access rows directly from a table in an enhanced *for* loop: + +.. code-block:: Java + + for (Row row: table) { + int age = row.getInt("age"); + boolean nameIsNull = row.isNull("name"); + ... + } + +Finally, while rows are usually iterated in the order of the underlying data vectors, but they are also positionable using the ``Row#setPosition()`` method, so you can skip to a specific row. Row numbers are 0-based. + +.. code-block:: Java + + Row r = table.immutableRow(); + int age101 = r.setPosition(101); // change position directly to 101 + +Any changes to position are applied to all the columns in the table. + +Note that you must call ``next()``, or ``setPosition()`` before accessing values via a row. Failure to do so results in a runtime exception. + +Read operations using rows +************************** + +Methods are available for getting values by vector name and vector index, where index is the 0-based position of the vector in the table. For example, assuming 'age' is the 13th vector in 'table', the following two gets are equivalent: + +.. code-block:: Java + + Row r = table.immutableRow(); + r.next(); // position the row at the first value + int age1 = r.get("age"); // gets the value of vector named 'age' in the table at row 0 + int age2 = r.get(12); // gets the value of the 13th vector in the table at row 0 + +You can also get value using a nullable ``ValueHolder``. For example: + +.. code-block:: Java + + NullableIntHolder holder = new NullableIntHolder(); + int b = row.getInt("age", holder); + +This can be used to retrieve values without creating a new Object for each. + +In addition to getting values, you can check if a value is null using ``isNull()``. This is important if the vector contains any nulls, as asking for a value from a vector can cause NullPointerExceptions in some cases. + +.. code-block:: Java + + boolean name0isNull = row.isNull("name"); + +You can also get the current row number: + +.. code-block:: Java + + int row = row.getRowNumber(); + +Reading values as Objects +************************* + +For any given vector type, the basic *get()* method returns a primitive value wherever possible. For example, *getTimeStampMicro()* returns a long value that encodes the timestamp. To get the LocalDateTime object representing that timestamp in Java, another method with 'Obj' appended to the name is provided. For example: + +.. code-block:: Java + + long ts = row.getTimeStampMicro(); + LocalDateTime tsObject = row.getTimeStampMicroObj(); + +The exception to this naming scheme is for complex vector types (List, Map, Schema, Union, DenseUnion, and ExtensionType). These always return objects rather than primitives so no "Obj" extension is required. It is expected that some users may subclass ``Row`` to add getters that are more specific to their needs. + +Reading VarChars and LargeVarChars +********************************** + +Strings in arrow are represented as byte arrays encoded with the UTF-8 charset. You can get either a String result or the actual byte array. + +.. code-block:: Java + + byte[] b = row.getVarChar("first_name"); + String s = row.getVarCharObj("first_name"); // uses the default encoding (UTF-8) + +Converting a Table to a VectorSchemaRoot +**************************************** + +Tables can be converted to vector schema roots using the *toVectorSchemaRoot()* method. Buffers are transferred to the vector schema root and the source table is cleared. + +.. code-block:: Java + + VectorSchemaRoot root = myTable.toVectorSchemaRoot(); + +Working with the C-Data interface +********************************* + +The ability to work with native code is required for many Arrow features. This section describes how tables can be be exported for use with native code + +Exporting works by converting the data to a ``VectorSchemaRoot`` instance and using the existing facilities to transfer the data. You could do it yourself, but that isn't ideal because conversion to a vector schema root breaks the immutability guarantees. Using the ``exportTable()`` methods in the `Data`_ class avoids this concern. + +.. code-block:: Java + + Data.exportTable(bufferAllocator, table, dictionaryProvider, outArrowArray); + +If the table contains dictionary-encoded vectors and was constructed with a ``DictionaryProvider``, the provider argument to ``exportTable()`` can be omitted and the table's provider attribute will be used: + +.. code-block:: Java + + Data.exportTable(bufferAllocator, table, outArrowArray); + +.. _`ArrowBuf`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/ArrowBuf.html +.. _`Data`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/c/Data.html +.. _`DictionaryProvider`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/dictionary/DictionaryProvider.html +.. _`Field`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/types/pojo/Field.html +.. _`FieldReader`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/complex/reader/FieldReader.html +.. _`FieldVector`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/FieldVector.html +.. _`Row`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/table/Row.html +.. _`Schema`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/types/pojo/Schema.html +.. _`Table`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/table/Table.html +.. _`ValueHolder`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/holders/ValueHolder.html +.. _`ValueVector`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/ValueVector.html +.. _`VectorSchemaRoot`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/VectorSchemaRoot.html diff --git a/docs/source/vector.rst b/docs/source/vector.rst new file mode 100644 index 0000000000..1996277444 --- /dev/null +++ b/docs/source/vector.rst @@ -0,0 +1,366 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +=========== +ValueVector +=========== + +:class:`ValueVector` interface (which called Array in C++ implementation and +the :external+arrow:doc:`the specification `) is an abstraction that is used to store a +sequence of values having the same type in an individual column. Internally, those values are +represented by one or several buffers, the number and meaning of which depend on the vector’s data type. + +There are concrete subclasses of :class:`ValueVector` for each primitive data type +and nested type described in the specification. There are a few differences in naming +with the type names described in the specification: +Table with non-intuitive names (BigInt = 64 bit integer, etc). + +It is important that vector is allocated before attempting to read or write, +:class:`ValueVector` "should" strive to guarantee this order of operation: +create > allocate > mutate > set value count > access > clear (or allocate to start the process over). +We will go through a concrete example to demonstrate each operation in the next section. + +Vector Life Cycle +================= + +As discussed above, each vector goes through several steps in its life cycle, +and each step is triggered by a vector operation. In particular, we have the following vector operations: + +1. **Vector creation**: we create a new vector object by, for example, the vector constructor. +The following code creates a new ``IntVector`` by the constructor: + +.. code-block:: Java + + RootAllocator allocator = new RootAllocator(Long.MAX_VALUE); + ... + IntVector vector = new IntVector("int vector", allocator); + +By now, a vector object is created. However, no underlying memory has been allocated, so we need the +following step. + +2. **Vector allocation**: in this step, we allocate memory for the vector. For most vectors, we +have two options: 1) if we know the maximum vector capacity, we can specify it by calling the +``allocateNew(int)`` method; 2) otherwise, we should call the ``allocateNew()`` method, and a default +capacity will be allocated for it. For our running example, we assume that the vector capacity never +exceeds 10: + +.. code-block:: Java + + vector.allocateNew(10); + +3. **Vector mutation**: now we can populate the vector with values we desire. For all vectors, we can populate +vector values through vector writers (An example will be given in the next section). For primitive types, +we can also mutate the vector by the set methods. There are two classes of set methods: 1) if we can +be sure the vector has enough capacity, we can call the ``set(index, value)`` method. 2) if we are not sure +about the vector capacity, we should call the ``setSafe(index, value)`` method, which will automatically +take care of vector reallocation, if the capacity is not sufficient. For our running example, we know the +vector has enough capacity, so we can call + +.. code-block:: Java + + vector.set(/*index*/5, /*value*/25); + +4. **Set value count**: for this step, we set the value count of the vector by calling the +``setValueCount(int)`` method: + +.. code-block:: Java + + vector.setValueCount(10); + +After this step, the vector enters an immutable state. In other words, we should no longer mutate it. +(Unless we reuse the vector by allocating it again. This will be discussed shortly.) + +5. **Vector access**: it is time to access vector values. Similarly, we have two options to access values: +1) get methods and 2) vector reader. Vector reader works for all types of vectors, while get methods are +only available for primitive vectors. A concrete example for vector reader will be given in the next section. +Below is an example of vector access by get method: + +.. code-block:: Java + + int value = vector.get(5); // value == 25 + +6. **Vector clear**: when we are done with the vector, we should clear it to release its memory. This is done by +calling the ``close()`` method: + +.. code-block:: Java + + vector.close(); + +Some points to note about the steps above: + +* The steps are not necessarily performed in a linear sequence. Instead, they can be in a loop. For example, + when a vector enters the access step, we can also go back to the vector mutation step, and then set value + count, access vector, and so on. + +* We should try to make sure the above steps are carried out in order. Otherwise, the vector + may be in an undefined state, and some unexpected behavior may occur. However, this restriction + is not strict. That means it is possible that we violates the order above, but still get + correct results. + +* When mutating vector values through set methods, we should prefer ``set(index, value)`` methods to + ``setSafe(index, value)`` methods whenever possible, to avoid unnecessary performance overhead of handling + vector capacity. + +* All vectors implement the ``AutoCloseable`` interface. So they must be closed explicitly when they are + no longer used, to avoid resource leak. To make sure of this, it is recommended to place vector related operations + into a try-with-resources block. + +* For fixed width vectors (e.g. IntVector), we can set values at different indices in arbitrary orders. + For variable width vectors (e.g. VarCharVector), however, we must set values in non-decreasing order of the + indices. Otherwise, the values after the set position will become invalid. For example, suppose we use the + following statements to populate a variable width vector: + +.. code-block:: Java + + VarCharVector vector = new VarCharVector("vector", allocator); + vector.allocateNew(); + vector.setSafe(0, "zero"); + vector.setSafe(1, "one"); + ... + vector.setSafe(9, "nine"); + +Then we set the value at position 5 again: + +.. code-block:: Java + + vector.setSafe(5, "5"); + +After that, the values at positions 6, 7, 8, and 9 of the vector will become invalid. + +Building ValueVector +==================== + +Note that the current implementation doesn't enforce the rule that Arrow objects are immutable. +:class:`ValueVector` instances could be created directly by using new keyword, there are +set/setSafe APIs and concrete subclasses of FieldWriter for populating values. + +For example, the code below shows how to build a :class:`BigIntVector`, in this case, we build a +vector of the range 0 to 7 where the element that should hold the fourth value is nulled + +.. code-block:: Java + + try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + BigIntVector vector = new BigIntVector("vector", allocator)) { + vector.allocateNew(8); + vector.set(0, 1); + vector.set(1, 2); + vector.set(2, 3); + vector.setNull(3); + vector.set(4, 5); + vector.set(5, 6); + vector.set(6, 7); + vector.set(7, 8); + vector.setValueCount(8); // this will finalizes the vector by convention. + ... + } + +The :class:`BigIntVector` holds two ArrowBufs. The first buffer holds the null bitmap, which consists +here of a single byte with the bits 1|1|1|1|0|1|1|1 (the bit is 1 if the value is non-null). +The second buffer contains all the above values. As the fourth entry is null, the value at that position +in the buffer is undefined. Note compared with set API, setSafe API would check value capacity before setting +values and reallocate buffers if necessary. + +Here is how to build a vector using writer + +.. code-block:: Java + + try (BigIntVector vector = new BigIntVector("vector", allocator); + BigIntWriter writer = new BigIntWriterImpl(vector)) { + writer.setPosition(0); + writer.writeBigInt(1); + writer.setPosition(1); + writer.writeBigInt(2); + writer.setPosition(2); + writer.writeBigInt(3); + // writer.setPosition(3) is not called which means the fourth value is null. + writer.setPosition(4); + writer.writeBigInt(5); + writer.setPosition(5); + writer.writeBigInt(6); + writer.setPosition(6); + writer.writeBigInt(7); + writer.setPosition(7); + writer.writeBigInt(8); + } + +There are get API and concrete subclasses of :class:`FieldReader` for accessing vector values, what needs +to be declared is that writer/reader is not as efficient as direct access + +.. code-block:: Java + + // access via get API + for (int i = 0; i < vector.getValueCount(); i++) { + if (!vector.isNull(i)) { + System.out.println(vector.get(i)); + } + } + + // access via reader + BigIntReader reader = vector.getReader(); + for (int i = 0; i < vector.getValueCount(); i++) { + reader.setPosition(i); + if (reader.isSet()) { + System.out.println(reader.readLong()); + } + } + +Building ListVector +=================== + +A :class:`ListVector` is a vector that holds a list of values for each index. Working with one you need to handle the same steps as mentioned above (create > allocate > mutate > set value count > access > clear), but the details of how you accomplish this are slightly different since you need to both create the vector and set the list of values for each index. + +For example, the code below shows how to build a :class:`ListVector` of int's using the writer :class:`UnionListWriter`. We build a vector from 0 to 9 and each index contains a list with values [[0, 0, 0, 0, 0], [0, 1, 2, 3, 4], [0, 2, 4, 6, 8], …, [0, 9, 18, 27, 36]]. List values can be added in any order so writing a list such as [3, 1, 2] would be just as valid. + +.. code-block:: Java + + try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + ListVector listVector = ListVector.empty("vector", allocator)) { + UnionListWriter writer = listVector.getWriter(); + for (int i = 0; i < 10; i++) { + writer.startList(); + writer.setPosition(i); + for (int j = 0; j < 5; j++) { + writer.writeInt(j * i); + } + writer.setValueCount(5); + writer.endList(); + } + listVector.setValueCount(10); + } + +:class:`ListVector` values can be accessed either through the get API or through the reader class :class:`UnionListReader`. To read all the values, first enumerate through the indexes, and then enumerate through the inner list values. + +.. code-block:: Java + + // access via get API + for (int i = 0; i < listVector.getValueCount(); i++) { + if (!listVector.isNull(i)) { + ArrayList elements = (ArrayList) listVector.getObject(i); + for (Integer element : elements) { + System.out.println(element); + } + } + } + + // access via reader + UnionListReader reader = listVector.getReader(); + for (int i = 0; i < listVector.getValueCount(); i++) { + reader.setPosition(i); + while (reader.next()) { + IntReader intReader = reader.reader(); + if (intReader.isSet()) { + System.out.println(intReader.readInteger()); + } + } + } + +Dictionary Encoding +=================== + +Dictionary encoding is a form of compression where values of one type are replaced by values of a smaller type: an array of ints replacing an array of strings is a common example. The mapping between the original values and the replacements is held in a 'dictionary'. Since the dictionary needs only one copy of each of the longer values, the combination of the dictionary and the array of smaller values may use less memory. The more repetitive the original data, the greater the savings. + +A ``FieldVector`` can be dictionary encoded for performance or improved memory efficiency. Nearly any type of vector might be encoded if there are many values, but few unique values. + +There are a few steps involved in the encoding process: + +1. Create a regular, un-encoded vector and populate it +2. Create a dictionary vector of the same type as the un-encoded vector. This vector must have the same values, but each unique value in the un-encoded vector need appear here only once. +3. Create a ``Dictionary``. It will contain the dictionary vector, plus a ``DictionaryEncoding`` object that holds the encoding's metadata and settings values. +4. Create a ``DictionaryEncoder``. +5. Call the encode() method on the ``DictionaryEncoder`` to produce an encoded version of the original vector. +6. (Optional) Call the decode() method on the encoded vector to re-create the original values. + +The encoded values will be integers. Depending on how many unique values you have, you can use ``TinyIntVector``, ``SmallIntVector``, ``IntVector``, or ``BigIntVector`` to hold them. You specify the type when you create your ``DictionaryEncoding`` instance. You might wonder where those integers come from: the dictionary vector is a regular vector, so the value's index position in that vector is used as its encoded value. + +Another critical attribute in ``DictionaryEncoding`` is the id. It's important to understand how the id is used, so we cover that later in this section. + +This result will be a new vector (for example, an ``IntVector``) that can act in place of the original vector (for example, a ``VarCharVector``). When you write the data in arrow format, it is both the new ``IntVector`` plus the dictionary that is written: you will need the dictionary later to retrieve the original values. + +.. code-block:: Java + + // 1. create a vector for the un-encoded data and populate it + VarCharVector unencoded = new VarCharVector("unencoded", allocator); + // now put some data in it before continuing + + // 2. create a vector to hold the dictionary and populate it + VarCharVector dictionaryVector = new VarCharVector("dictionary", allocator); + + // 3. create a dictionary object + Dictionary dictionary = new Dictionary(dictionaryVector, new DictionaryEncoding(1L, false, null)); + + // 4. create a dictionary encoder + DictionaryEncoder encoder = new DictionaryEncoder.encode(dictionary, allocator); + + // 5. encode the data + IntVector encoded = (IntVector) encoder.encode(unencoded); + + // 6. re-create an un-encoded version from the encoded vector + VarCharVector decoded = (VarCharVector) encoder.decode(encoded); + +One thing we haven't discussed is how to create the dictionary vector from the original un-encoded values. That is left to the library user since a custom method will likely be more efficient than a general utility. Since the dictionary vector is just a normal vector, you can populate its values with the standard APIs. + +Finally, you can package a number of dictionaries together, which is useful if you're working with a ``VectorSchemaRoot`` with several dictionary-encoded vectors. This is done using an object called a ``DictionaryProvider``. as shown in the example below. Note that we don't put the dictionary vectors in the same ``VectorSchemaRoot`` as the data vectors, as they will generally have fewer values. + + +.. code-block:: Java + + DictionaryProvider.MapDictionaryProvider provider = + new DictionaryProvider.MapDictionaryProvider(); + + provider.put(dictionary); + +The ``DictionaryProvider`` is simply a map of identifiers to ``Dictionary`` objects, where each identifier is a long value. In the above code you will see it as the first argument to the ``DictionaryEncoding`` constructor. + +This is where the ``DictionaryEncoding``'s 'id' attribute comes in. This value is used to connect dictionaries to instances of ``VectorSchemaRoot``, using a ``DictionaryProvider``. Here's how that works: + +* The ``VectorSchemaRoot`` has a ``Schema`` object containing a list of ``Field`` objects. +* The field has an attribute called 'dictionary', but it holds a ``DictionaryEncoding`` rather than a ``Dictionary`` +* As mentioned, the ``DictionaryProvider`` holds dictionaries indexed by a long value. This value is the id from your ``DictionaryEncoding``. +* To retrieve the dictionary for a vector in a ``VectorSchemaRoot``, you get the field associated with the vector, get its dictionary attribute, and use that object's id to look up the correct dictionary in the provider. + +.. code-block:: Java + + // create the encoded vector, the Dictionary and DictionaryProvider as discussed above + + // Create a VectorSchemaRoot with one encoded vector + VectorSchemaRoot vsr = new VectorSchemaRoot(List.of(encoded)); + + // now we want to decode our vector, so we retrieve its dictionary from the provider + Field f = vsr.getField(encoded.getName()); + DictionaryEncoding encoding = f.getDictionary(); + Dictionary dictionary = provider.lookup(encoding.getId()); + +As you can see, a ``DictionaryProvider`` is handy for managing the dictionaries associated with a ``VectorSchemaRoot``. More importantly, it helps package the dictionaries for a ``VectorSchemaRoot`` when it's written. The classes ``ArrowFileWriter`` and ``ArrowStreamWriter`` both accept an optional ``DictionaryProvider`` argument for that purpose. You can find example code for writing dictionaries in the documentation for (:doc:`ipc`). ``ArrowReader`` and its subclasses also implement the ``DictionaryProvider`` interface, so you can retrieve the actual dictionaries when reading a file. + +Slicing +======= + +Similar with C++ implementation, it is possible to make zero-copy slices of vectors to obtain a vector +referring to some logical sub-sequence of the data through :class:`TransferPair` + +.. code-block:: Java + + IntVector vector = new IntVector("intVector", allocator); + for (int i = 0; i < 10; i++) { + vector.setSafe(i, i); + } + vector.setValueCount(10); + + TransferPair tp = vector.getTransferPair(allocator); + tp.splitAndTransfer(0, 5); + IntVector sliced = (IntVector) tp.getTo(); + // In this case, the vector values are [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] and the sliceVector values are [0, 1, 2, 3, 4]. diff --git a/docs/source/vector_schema_root.rst b/docs/source/vector_schema_root.rst new file mode 100644 index 0000000000..3119122d9a --- /dev/null +++ b/docs/source/vector_schema_root.rst @@ -0,0 +1,163 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +============ +Tabular Data +============ + +While arrays (aka: :doc:`ValueVector <./vector>`) represent a one-dimensional sequence of +homogeneous values, data often comes in the form of two-dimensional sets of +heterogeneous data (such as database tables, CSV files...). Arrow provides +several abstractions to handle such data conveniently and efficiently. + +Fields +====== + +Fields are used to denote the particular columns of tabular data. +A field, i.e. an instance of `Field`_, holds together a field name, a data +type, and some optional key-value metadata. + +.. code-block:: Java + + // Create a column "document" of string type with metadata + import org.apache.arrow.vector.types.pojo.ArrowType; + import org.apache.arrow.vector.types.pojo.Field; + import org.apache.arrow.vector.types.pojo.FieldType; + + Map metadata = new HashMap<>(); + metadata.put("A", "Id card"); + metadata.put("B", "Passport"); + metadata.put("C", "Visa"); + Field document = new Field("document", new FieldType(true, new ArrowType.Utf8(), /*dictionary*/ null, metadata), /*children*/ null); + +Schemas +======= + +A `Schema`_ describes the overall structure consisting of any number of columns. It holds a sequence of fields together +with some optional schema-wide metadata (in addition to per-field metadata). + +.. code-block:: Java + + // Create a schema describing datasets with two columns: + // a int32 column "A" and a utf8-encoded string column "B" + import org.apache.arrow.vector.types.pojo.ArrowType; + import org.apache.arrow.vector.types.pojo.Field; + import org.apache.arrow.vector.types.pojo.FieldType; + import org.apache.arrow.vector.types.pojo.Schema; + import static java.util.Arrays.asList; + + Map metadata = new HashMap<>(); + metadata.put("K1", "V1"); + metadata.put("K2", "V2"); + Field a = new Field("A", FieldType.nullable(new ArrowType.Int(32, true)), null); + Field b = new Field("B", FieldType.nullable(new ArrowType.Utf8()), null); + Schema schema = new Schema(asList(a, b), metadata); + +VectorSchemaRoot +================ + +A `VectorSchemaRoot`_ is a container for batches of data. Batches flow through +VectorSchemaRoot as part of a pipeline. + +.. note:: + + VectorSchemaRoot is somewhat analogous to tables or record batches in the + other Arrow implementations in that they all are 2D datasets, but their + usage is different. + +The recommended usage is to create a single VectorSchemaRoot based on a known +schema and populate data over and over into that root in a stream of batches, +rather than creating a new instance each time (see `Flight`_ or +``ArrowFileWriter`` as examples). Thus at any one point, a VectorSchemaRoot may +have data or may have no data (say it was transferred downstream or not yet +populated). + +Here is an example of creating a VectorSchemaRoot: + +.. code-block:: Java + + BitVector bitVector = new BitVector("boolean", allocator); + VarCharVector varCharVector = new VarCharVector("varchar", allocator); + bitVector.allocateNew(); + varCharVector.allocateNew(); + for (int i = 0; i < 10; i++) { + bitVector.setSafe(i, i % 2 == 0 ? 0 : 1); + varCharVector.setSafe(i, ("test" + i).getBytes(StandardCharsets.UTF_8)); + } + bitVector.setValueCount(10); + varCharVector.setValueCount(10); + + List fields = Arrays.asList(bitVector.getField(), varCharVector.getField()); + List vectors = Arrays.asList(bitVector, varCharVector); + VectorSchemaRoot vectorSchemaRoot = new VectorSchemaRoot(fields, vectors); + +Data can be loaded into/unloaded from a VectorSchemaRoot via `VectorLoader`_ +and `VectorUnloader`_. They handle converting between VectorSchemaRoot and +`ArrowRecordBatch`_ (a representation of a RecordBatch +:external+arrow:ref:`IPC ` message). For example: + +.. code-block:: Java + + // create a VectorSchemaRoot root1 and convert its data into recordBatch + VectorSchemaRoot root1 = new VectorSchemaRoot(fields, vectors); + VectorUnloader unloader = new VectorUnloader(root1); + ArrowRecordBatch recordBatch = unloader.getRecordBatch(); + + // create a VectorSchemaRoot root2 and load the recordBatch + VectorSchemaRoot root2 = VectorSchemaRoot.create(root1.getSchema(), allocator); + VectorLoader loader = new VectorLoader(root2); + loader.load(recordBatch); + +A new VectorSchemaRoot can be sliced from an existing root without copying +data: + +.. code-block:: Java + + // 0 indicates start index (inclusive) and 5 indicated length (exclusive). + VectorSchemaRoot newRoot = vectorSchemaRoot.slice(0, 5); + +Table +===== + +A `Table`_ is an immutable tabular data structure, very similar to VectorSchemaRoot, in that it is also built on ValueVectors and schemas. Unlike VectorSchemaRoot, Table is not designed for batch processing. Here is a version of the example above, showing how to create a Table, rather than a VectorSchemaRoot: + +.. code-block:: Java + + BitVector bitVector = new BitVector("boolean", allocator); + VarCharVector varCharVector = new VarCharVector("varchar", allocator); + bitVector.allocateNew(); + varCharVector.allocateNew(); + for (int i = 0; i < 10; i++) { + bitVector.setSafe(i, i % 2 == 0 ? 0 : 1); + varCharVector.setSafe(i, ("test" + i).getBytes(StandardCharsets.UTF_8)); + } + bitVector.setValueCount(10); + varCharVector.setValueCount(10); + + List vectors = Arrays.asList(bitVector, varCharVector); + Table table = new Table(vectors); + +See the :doc:`table` documentation for more information. + +.. _`ArrowRecordBatch`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/ipc/message/ArrowRecordBatch.html +.. _`Field`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/types/pojo/Field.html +.. _`Flight`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/flight/package-summary.html +.. _`Schema`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/types/pojo/Schema.html +.. _`Table`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/table/Table.html +.. _`VectorLoader`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/VectorLoader.html +.. _`VectorSchemaRoot`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/VectorSchemaRoot.html +.. _`VectorUnloader`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/VectorUnloader.html From b4d6cc0dc47822b370d2a77d783dad078aea566b Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 27 Jan 2025 09:20:17 +0900 Subject: [PATCH 055/408] GH-547: [CI][Docs] Publish docs to https://arrow.apache.org/java/main/ on push (#561) Fixes #537. This also enables preview in fork at `https://${YOUR_GITHUB_ACCOUNT}.github.io/arrow-java/main/` after you enable GitHub Pages on `gh-pages` branch at `https://github.com/${YOUR_GITHUB_ACCOUNT}/arrow-java/settings/pages`. --- .asf.yaml | 3 ++ .github/workflows/rc.yml | 68 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/.asf.yaml b/.asf.yaml index bf17bd0ff6..c895ca1553 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -38,3 +38,6 @@ notifications: issues_status: issues@arrow.apache.org issues_comment: github@arrow.apache.org pullrequests: github@arrow.apache.org +publish: + whoami: asf-site + subdir: java diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 9dc04fee05..8e7f7f5fb2 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -471,6 +471,74 @@ jobs: with: name: release-html path: html.tar.gz + public-docs: + name: Publish docs + # Run only when: + # * We push to a branch + # * If the target repository is apache/arrow-java: + # * The target branch is main + # * Else (fork repositories): + # * All branches + # * We can preview the last pushed content + # at https://${YOUR_GITHUB_ACCOUNT}.github.io/arrow-java/main/ + if: >- + github.event_name == 'push' && + github.ref_type == 'branch' && + ((github.repository == 'apache/arrow-java' && github.ref_name == 'main') || + github.repository != 'apache/arrow-java') + needs: + - docs + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: site + persist-credentials: true + - name: Prepare branch + run: | + if [ "${GITHUB_REPOSITORY}" = "apache/arrow-java" ]; then + BRANCH=asf-site + else + BRANCH=gh-pages + fi + echo "BRANCH=${BRANCH}" >> ${GITHUB_ENV} + + cp site/.asf.yaml ./ + cd site + git fetch + if ! git switch -c "${BRANCH}" "origin/${BRANCH}"; then + git switch --orphan "${BRANCH}" + fi + touch .nojekyll + cp ../.asf.yaml ./ + git add .nojekyll .asf.yaml + - name: Download + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: release-html + - name: Extract + run: | + tar -xf html.tar.gz + rm -rf site/main + mv html site/main + git -C site add main + if [ "$(git status --prcelain)" == "" ]; then + NEED_PUSH=true + else + NEED_PUSH=false + fi + echo "NEED_PUSH=${NEED_PUSH}" >> ${GITHUB_ENV} + - name: Push + if: env.NEED_PUSH == 'true' + run: | + cd site + git config --global user.name 'github-actions[bot]' + git config --global user.email 'github-actions[bot]@users.noreply.github.com' + git commit -m "Publish documentation (${GITHUB_SHA})" + git push origin "${BRANCH}" verify: name: Verify needs: From d4f8074dd82ea3f59618fd43e58a1ba04186718c Mon Sep 17 00:00:00 2001 From: ViggoC Date: Mon, 27 Jan 2025 12:31:43 +0800 Subject: [PATCH 056/408] GH-41: BaseVariableWidthViewVector setZero only if necessary (#557) Closes #41. --- .../vector/BaseVariableWidthViewVector.java | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java index 15d2182783..e0e16762f2 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java @@ -1367,11 +1367,13 @@ protected ArrowBuf allocateOrGetLastDataBuffer(int length) { protected final void setBytes(int index, byte[] value, int start, int length) { int writePosition = index * ELEMENT_SIZE; - // to clear the memory segment of view being written to - // this is helpful in case of overwriting the value - viewBuffer.setZero(writePosition, ELEMENT_SIZE); - if (length <= INLINE_SIZE) { + // Check if the memory segment has been written, and clear it if it has been set. + // It is recommended to batch initialize the viewBuffer before setBytes. + if (viewBuffer.getLong(writePosition) != 0 || viewBuffer.getLong(writePosition + 8) != 0) { + viewBuffer.setZero(writePosition, ELEMENT_SIZE); + } + // allocate inline buffer // set length viewBuffer.setInt(writePosition, length); @@ -1411,11 +1413,13 @@ protected final void setBytes(int index, byte[] value, int start, int length) { protected final void setBytes(int index, ArrowBuf valueBuf, int start, int length) { int writePosition = index * ELEMENT_SIZE; - // to clear the memory segment of view being written to - // this is helpful in case of overwriting the value - viewBuffer.setZero(writePosition, ELEMENT_SIZE); - if (length <= INLINE_SIZE) { + // Check if the memory segment has been written, and clear it if it has been set. + // It is recommended to batch initialize the viewBuffer before setBytes. + if (viewBuffer.getLong(writePosition) != 0 || viewBuffer.getLong(writePosition + 8) != 0) { + viewBuffer.setZero(writePosition, ELEMENT_SIZE); + } + // allocate inline buffer // set length viewBuffer.setInt(writePosition, length); From bbbe4b488ba4d3cf665f0a7393b8e44eae4f84a1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 05:48:27 -0500 Subject: [PATCH 057/408] MINOR: Bump io.netty:netty-bom from 4.1.115.Final to 4.1.117.Final (#567) Bumps [io.netty:netty-bom](https://github.com/netty/netty) from 4.1.115.Final to 4.1.117.Final. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index af7f5bd06e..2e8f0278e2 100644 --- a/pom.xml +++ b/pom.xml @@ -96,7 +96,7 @@ under the License. 5.11.4 2.0.16 33.4.0-jre - 4.1.115.Final + 4.1.117.Final 1.69.1 3.25.5 2.18.2 From 95df256dd4bf64a0a9032bbc6e56b6e3611bdfc7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 05:51:07 -0500 Subject: [PATCH 058/408] MINOR: Bump org.assertj:assertj-core from 3.27.2 to 3.27.3 (#566) Bumps [org.assertj:assertj-core](https://github.com/assertj/assertj) from 3.27.2 to 3.27.3. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2e8f0278e2..e92489766f 100644 --- a/pom.xml +++ b/pom.xml @@ -172,7 +172,7 @@ under the License. org.assertj assertj-core - 3.27.2 + 3.27.3 test From c7587b8ca2209f0efcf8c8d64f31c5e67208ce2e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 06:08:26 -0500 Subject: [PATCH 059/408] MINOR: Bump org.apache.commons:commons-dbcp2 from 2.12.0 to 2.13.0 (#564) Bumps org.apache.commons:commons-dbcp2 from 2.12.0 to 2.13.0. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql/pom.xml b/flight/flight-sql/pom.xml index 668fcb148b..be93aa2496 100644 --- a/flight/flight-sql/pom.xml +++ b/flight/flight-sql/pom.xml @@ -95,7 +95,7 @@ under the License. org.apache.commons commons-dbcp2 - 2.12.0 + 2.13.0 test From 92fece14c88b37c3d6b00f7a6d962386a6d7dafb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 07:26:42 -0500 Subject: [PATCH 060/408] MINOR: Bump io.grpc:grpc-bom from 1.69.1 to 1.70.0 (#565) Bumps [io.grpc:grpc-bom](https://github.com/grpc/grpc-java) from 1.69.1 to 1.70.0. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e92489766f..f7cbcc8d25 100644 --- a/pom.xml +++ b/pom.xml @@ -97,7 +97,7 @@ under the License. 2.0.16 33.4.0-jre 4.1.117.Final - 1.69.1 + 1.70.0 3.25.5 2.18.2 3.4.1 From 1895e2f7c653d41efe7fd59000754bf1d47706b3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 20:38:26 -0500 Subject: [PATCH 061/408] MINOR: Bump com.gradle:develocity-maven-extension from 1.23 to 1.23.1 (#562) Bumps com.gradle:develocity-maven-extension from 1.23 to 1.23.1. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .mvn/extensions.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index 47cffa9b57..1dd32d62d6 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -23,7 +23,7 @@ com.gradle develocity-maven-extension - 1.23 + 1.23.1 com.gradle From 730f9c09edc736a8cdb342807e430b0a247eeb5c Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 29 Jan 2025 12:04:36 +0900 Subject: [PATCH 062/408] GH-570: [Docs] Use https://arrow.apache.org/java/ as the official site URL (#572) Fixes GH-570. --- .asf.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.asf.yaml b/.asf.yaml index c895ca1553..ead2581149 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -19,7 +19,7 @@ github: description: "Official Java implementation of Apache Arrow" - homepage: https://arrow.apache.org/ + homepage: https://arrow.apache.org/java/ labels: - apache-arrow - java From f6b4f98f8269955c25edac11a0870e4193c7b902 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 30 Jan 2025 13:27:58 +0900 Subject: [PATCH 063/408] GH-548: [Release][Docs] Publish docs to https://arrow.apache.org/java/current/ on release (#569) Fixes GH-548. --- .github/workflows/rc.yml | 5 ++-- .github/workflows/release.yml | 46 ++++++++++++++++++++++++++++++----- dev/release/release_rc.sh | 12 ++++++--- 3 files changed, 51 insertions(+), 12 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 8e7f7f5fb2..9717f30a9c 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -471,7 +471,7 @@ jobs: with: name: release-html path: html.tar.gz - public-docs: + publish-docs: name: Publish docs # Run only when: # * We push to a branch @@ -496,7 +496,6 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: site - persist-credentials: true - name: Prepare branch run: | if [ "${GITHUB_REPOSITORY}" = "apache/arrow-java" ]; then @@ -599,7 +598,7 @@ jobs: gh release create ${GITHUB_REF_NAME} \ --generate-notes \ --prerelease \ - --repo ${{ github.repository }} \ + --repo ${GITHUB_REPOSITORY} \ --title "Apache Arrow Java ${version} RC${rc}" \ --verify-tag \ artifacts/*/* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4982c6b343..e801fd0493 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -34,21 +34,55 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Download RC contents run: | set -x - latest_rc_tag=$(gh release list --json tagName --jq '.[].tagName' | \ - grep -F "${GITHUB_REF_NAME}-rc" | \ - head -n1) - gh release download ${latest_rc_tag} --dir dists + latest_rc_tag=$(gh release list \ + --jq '.[].tagName' \ + --json tagName \ + --repo ${GITHUB_REPOSITORY} | \ + grep -F "${GITHUB_REF_NAME}-rc" | \ + head -n1) + gh release download ${latest_rc_tag} \ + --repo ${GITHUB_REPOSITORY} \ + --dir dists - name: Create GitHub Release run: | # GH-499: How to create release notes? version=${GITHUB_REF_NAME#v} gh release create ${GITHUB_REF_NAME} \ --generate-notes \ + --repo ${GITHUB_REPOSITORY} \ --title "Apache Arrow Java ${version}" \ --verify-tag \ dists/* + - name: Checkout for publishing docs + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: site + - name: Publish docs + run: | + set -x + + tar -xf dists/html.tar.gz + version=${GITHUB_REF_NAME#v} + + if [ "${GITHUB_REPOSITORY}" = "apache/arrow-java" ]; then + BRANCH=asf-site + else + BRANCH=gh-pages + fi + + cd site + git fetch + git switch -c "${BRANCH}" "origin/${BRANCH}" + + rm -rf current ${version} + cp -a ../html current + cp -a ../html ${version} + git add current ${version} + + git config --global user.name 'github-actions[bot]' + git config --global user.email 'github-actions[bot]@users.noreply.github.com' + git commit -m "Publish documentation (${GITHUB_REF_NAME})" + git push origin "${BRANCH}" diff --git a/dev/release/release_rc.sh b/dev/release/release_rc.sh index dd61f9c9e6..1375376463 100755 --- a/dev/release/release_rc.sh +++ b/dev/release/release_rc.sh @@ -97,7 +97,7 @@ if [ "${RELEASE_SIGN}" -gt 0 ]; then echo "Looking for GitHub Actions workflow on ${repository}:${rc_tag}" run_id="" - while [ -z "${run_id}" ]; do + while true; do echo "Waiting for run to start..." run_id=$(gh run list \ --branch "${rc_tag}" \ @@ -106,11 +106,17 @@ if [ "${RELEASE_SIGN}" -gt 0 ]; then --limit 1 \ --repo "${repository}" \ --workflow rc.yml) - sleep 1 + if [ -n "${run_id}" ]; then + break + fi + sleep 60 done echo "Found GitHub Actions workflow with ID: ${run_id}" - gh run watch --repo "${repository}" --exit-status "${run_id}" + gh run watch \ + --exit-status "${run_id}" \ + --interval 60 \ + --repo "${repository}" echo "Downloading artifacts from GitHub Releases" gh release download "${rc_tag}" \ From bd2173c59c52983a9e1a6cadffb8345294960949 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Thu, 30 Jan 2025 15:21:26 +0100 Subject: [PATCH 064/408] MINOR: Add jbonofre to collaborators (#576) We can add collaborator in the `.asf.yaml`. A collaborator can be assigned to GitHub Issue for instance. --- .asf.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.asf.yaml b/.asf.yaml index ead2581149..05a8663e3c 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -33,6 +33,8 @@ github: protected_branches: main: required_linear_history: true + collaborators: # Note: the number of collaborators is limited to 10 + - jbonofre notifications: commits: commits@arrow.apache.org issues_status: issues@arrow.apache.org From 22d172fc8d172d5a35f0230c18b48c1a51a3ab91 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Feb 2025 08:52:36 +0900 Subject: [PATCH 065/408] MINOR: Bump com.google.api.grpc:proto-google-common-protos from 2.50.1 to 2.51.0 (#584) Bumps [com.google.api.grpc:proto-google-common-protos](https://github.com/googleapis/sdk-platform-java) from 2.50.1 to 2.51.0.

Release notes

Sourced from com.google.api.grpc:proto-google-common-protos's releases.

v2.51.0

2.51.0 (2024-12-12)

Features

  • [iam] add ResourcePolicyMember to google/iam/v1 (b8e2859)

Bug Fixes

Dependencies

  • update dependency com.fasterxml.jackson:jackson-bom to v2.18.2 (#3422) (fdc3a54)
  • update dependency com.google.api-client:google-api-client-bom to v2.7.1 (#3450) (35f1310)
  • update dependency com.google.auth:google-auth-library-oauth2-http to v1.30.0 (#3428) (78cbff1)
  • update dependency com.google.errorprone:error_prone_annotations to v2.36.0 (#3425) (52dcc0d)
  • update dependency com.google.errorprone:error_prone_annotations to v2.36.0 (#3426) (77dd85d)
  • update dependency dev.cel:cel to v0.8.0 (#3429) (79cde20)
  • update dependency io.github.java-diff-utils:java-diff-utils to v4.15 (#3430) (91fd5cb)
  • update dependency net.bytebuddy:byte-buddy to v1.15.10 (#3298) (7b503de)
  • update dependency org.checkerframework:checker-qual to v3.48.3 (#3278) (c5fd1b4)
  • update dependency org.easymock:easymock to v5.5.0 (#3431) (3c22f5e)
  • update dependency packaging to v24.2 (#3432) (c1e7c81)
  • update docker.io/library/alpine docker tag to v3.21.0 (#3433) (c8318c2)
  • update docker.io/library/alpine:3.20.3 docker digest to 1e42bbe (#3417) (0f4ef19)
  • update docker.io/library/maven:3.9.9-eclipse-temurin-11-alpine docker digest to 3bab9f2 (#3447) (ccf7eac)
  • update docker.io/library/maven:3.9.9-eclipse-temurin-11-alpine docker digest to cdfb386 (#3418) (2559ff8)
  • update docker.io/library/python docker tag to v3.13.1 (#3434) (feefd27)
  • update google api dependencies (#3435) (3a1e8f2)
  • update google auth library dependencies to v1.30.1 (#3453) (d7b7dd9)
  • update google http client dependencies to v1.45.2 (#3394) (f8ee892)
  • update google http client dependencies to v1.45.3 (#3454) (a112559)
  • update google.cloud.opentelemetry.version to v0.33.0 (#3436) (e519626)
  • update googleapis/java-cloud-bom digest to 93740d5 (#3280) (f987db5)
  • update grpc dependencies to v1.68.2 (#3420) (6d0a169)
  • update grpc dependencies to v1.69.0 (#3451) (0effb37)
  • update junit5 monorepo to v5.11.3 (#3310) (4e33f2b)
  • update netty dependencies (#3423) (22ea7b5)
  • update opentelemetry-java monorepo to v1.44.1 (#3437) (66b85da)
  • update opentelemetry-java monorepo to v1.45.0 (#3439) (d29475a)
  • update repo-automation-bots digest to 6662ddc (#3421) (dec5de0)
  • update repo-automation-bots digest to 8230b4e (#3448) (e53199b)
  • update repo-automation-bots digest to ae0a72f (#3452) (331bacf)
  • update repo-automation-bots digest to f3fbbab (#3419) (343a029)

... (truncated)

Changelog

Sourced from com.google.api.grpc:proto-google-common-protos's changelog.

2.51.0 (2024-12-12)

Features

  • [iam] add ResourcePolicyMember to google/iam/v1 (b8e2859)

Bug Fixes

Dependencies

  • update dependency com.fasterxml.jackson:jackson-bom to v2.18.2 (#3422) (fdc3a54)
  • update dependency com.google.api-client:google-api-client-bom to v2.7.1 (#3450) (35f1310)
  • update dependency com.google.auth:google-auth-library-oauth2-http to v1.30.0 (#3428) (78cbff1)
  • update dependency com.google.errorprone:error_prone_annotations to v2.36.0 (#3425) (52dcc0d)
  • update dependency com.google.errorprone:error_prone_annotations to v2.36.0 (#3426) (77dd85d)
  • update dependency dev.cel:cel to v0.8.0 (#3429) (79cde20)
  • update dependency io.github.java-diff-utils:java-diff-utils to v4.15 (#3430) (91fd5cb)
  • update dependency net.bytebuddy:byte-buddy to v1.15.10 (#3298) (7b503de)
  • update dependency org.checkerframework:checker-qual to v3.48.3 (#3278) (c5fd1b4)
  • update dependency org.easymock:easymock to v5.5.0 (#3431) (3c22f5e)
  • update dependency packaging to v24.2 (#3432) (c1e7c81)
  • update docker.io/library/alpine docker tag to v3.21.0 (#3433) (c8318c2)
  • update docker.io/library/alpine:3.20.3 docker digest to 1e42bbe (#3417) (0f4ef19)
  • update docker.io/library/maven:3.9.9-eclipse-temurin-11-alpine docker digest to 3bab9f2 (#3447) (ccf7eac)
  • update docker.io/library/maven:3.9.9-eclipse-temurin-11-alpine docker digest to cdfb386 (#3418) (2559ff8)
  • update docker.io/library/python docker tag to v3.13.1 (#3434) (feefd27)
  • update google api dependencies (#3435) (3a1e8f2)
  • update google auth library dependencies to v1.30.1 (#3453) (d7b7dd9)
  • update google http client dependencies to v1.45.2 (#3394) (f8ee892)
  • update google http client dependencies to v1.45.3 (#3454) (a112559)
  • update google.cloud.opentelemetry.version to v0.33.0 (#3436) (e519626)
  • update googleapis/java-cloud-bom digest to 93740d5 (#3280) (f987db5)
  • update grpc dependencies to v1.68.2 (#3420) (6d0a169)
  • update grpc dependencies to v1.69.0 (#3451) (0effb37)
  • update junit5 monorepo to v5.11.3 (#3310) (4e33f2b)
  • update netty dependencies (#3423) (22ea7b5)
  • update opentelemetry-java monorepo to v1.44.1 (#3437) (66b85da)
  • update opentelemetry-java monorepo to v1.45.0 (#3439) (d29475a)
  • update repo-automation-bots digest to 6662ddc (#3421) (dec5de0)
  • update repo-automation-bots digest to 8230b4e (#3448) (e53199b)
  • update repo-automation-bots digest to ae0a72f (#3452) (331bacf)
  • update repo-automation-bots digest to f3fbbab (#3419) (343a029)

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.api.grpc:proto-google-common-protos&package-manager=maven&previous-version=2.50.1&new-version=2.51.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml index 476b994127..3960c412b6 100644 --- a/flight/flight-core/pom.xml +++ b/flight/flight-core/pom.xml @@ -134,7 +134,7 @@ under the License. com.google.api.grpc proto-google-common-protos - 2.50.1 + 2.51.0 test From a3c95d17c8ca4c98deec2e79d9f530000c1bfd8e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Feb 2025 08:52:53 +0900 Subject: [PATCH 066/408] MINOR: Bump com.puppycrawl.tools:checkstyle from 10.21.1 to 10.21.2 (#580) Bumps [com.puppycrawl.tools:checkstyle](https://github.com/checkstyle/checkstyle) from 10.21.1 to 10.21.2.
Release notes

Sourced from com.puppycrawl.tools:checkstyle's releases.

checkstyle-10.21.2

Checkstyle 10.21.2 - https://checkstyle.org/releasenotes.html#Release_10.21.2

Bug fixes:

#15939 - lineWrappingIndentation falsely detects incorrect indentation for text blocks #16101 - ignoreFieldDeclaration property should have the highest priority in MagicNumberCheck

Commits
  • 197b10f [maven-release-plugin] prepare release checkstyle-10.21.2
  • dc7c5be doc: release notes for 10.21.2
  • edc2dbe minor: update suppressioon for linkcheck to fix CI failure
  • 01ec09a Issue #16077: fixed large margins and paddings
  • 7ff1ebf Issue #13345: Enable example test for SuppressWithNearbyCommentFilter
  • 9201875 Issue #13213: remove OK comments from javadocmethod
  • 69c4c65 Issue #13345: enabling tests for RegexpCheckExampleTest
  • 05b2a32 Issue #13213: remove OK comments from javadocblocktaglocation
  • 95eeefd supplemental: test-cases for unnecessary parentheses on typecasting
  • ad7b6ba supplemental: disable locale runs until #16233
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.puppycrawl.tools:checkstyle&package-manager=maven&previous-version=10.21.1&new-version=10.21.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f7cbcc8d25..6e82083b74 100644 --- a/pom.xml +++ b/pom.xml @@ -105,7 +105,7 @@ under the License. 1.12.0 2 - 10.21.1 + 10.21.2 true 2.31.0 5.15.2 From ef680f9c03939fd6ef3f636ac25cb969aebd0283 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Feb 2025 08:53:08 +0900 Subject: [PATCH 067/408] MINOR: Bump org.apache.commons:commons-pool2 from 2.12.0 to 2.12.1 (#581) Bumps org.apache.commons:commons-pool2 from 2.12.0 to 2.12.1. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.commons:commons-pool2&package-manager=maven&previous-version=2.12.0&new-version=2.12.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql/pom.xml b/flight/flight-sql/pom.xml index be93aa2496..7c7059de2d 100644 --- a/flight/flight-sql/pom.xml +++ b/flight/flight-sql/pom.xml @@ -107,7 +107,7 @@ under the License. org.apache.commons commons-pool2 - 2.12.0 + 2.12.1 test From 28e778a4d9fb7dd626d49048fd57433ff33a15ff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Feb 2025 08:53:36 +0900 Subject: [PATCH 068/408] MINOR: Bump commons-codec:commons-codec from 1.17.2 to 1.18.0 (#582) Bumps [commons-codec:commons-codec](https://github.com/apache/commons-codec) from 1.17.2 to 1.18.0.
Changelog

Sourced from commons-codec:commons-codec's changelog.

Apache Commons Codec 1.18.0 Release Notes

The Apache Commons Codec team is pleased to announce the release of Apache Commons Codec 1.18.0.

The Apache Commons Codec component contains encoders and decoders for formats such as Base16, Base32, Base64, digest, and Hexadecimal. In addition to these widely used encoders and decoders, the codec package also maintains a collection of phonetic encoding utilities.

This is a feature and maintenance release. Java 8 or later is required.

New features

  •  Add Base32.Builder.setHexDecodeTable(boolean). Thanks to
Gary Gregory, Julian Reschke.

  •  Add Base32.Builder.setHexEncodeTable(boolean). Thanks to
Gary Gregory, Julian Reschke.

Changes

  •  Bump org.apache.commons:commons-parent from 78 to 79. Thanks
to Gary Gregory.

For complete information on Apache Commons Codec, including instructions on how to submit bug reports, patches, or suggestions for improvement, see the Apache Commons Codec website:

https://commons.apache.org/proper/commons-codec/

Download page: https://commons.apache.org/proper/commons-codec/download_codec.cgi

Commits
  • 5f76abb Update contributing file from user feedback
  • a38de95 Prepare for the next release candidate
  • 0689dc8 Prepare for the next release candidate
  • 4098222 Port from Doxia 1 to 2
  • ebb29e9 Make test fixture package private
  • 110a9f8 Don't need to end a paragraph with an extra line break
  • 6444237 Remove background color from site page
  • 9062af4 Merge branch 'master' of
  • 963ee73 Include more in the source assembly
  • 94b9c51 Add a file extension to TODO file
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=commons-codec:commons-codec&package-manager=maven&previous-version=1.17.2&new-version=1.18.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- vector/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vector/pom.xml b/vector/pom.xml index 673f8eed17..94fde96e14 100644 --- a/vector/pom.xml +++ b/vector/pom.xml @@ -60,7 +60,7 @@ under the License. commons-codec commons-codec - 1.17.2 + 1.18.0 org.apache.arrow From a10b900e7b35b92b88a1854a31d832fce964547b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Feb 2025 08:53:54 +0900 Subject: [PATCH 069/408] MINOR: Bump org.apache.commons:commons-text from 1.12.0 to 1.13.0 (#583) Bumps org.apache.commons:commons-text from 1.12.0 to 1.13.0. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.commons:commons-text&package-manager=maven&previous-version=1.12.0&new-version=1.13.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql/pom.xml b/flight/flight-sql/pom.xml index 7c7059de2d..4b5c5560c9 100644 --- a/flight/flight-sql/pom.xml +++ b/flight/flight-sql/pom.xml @@ -113,7 +113,7 @@ under the License. org.apache.commons commons-text - 1.12.0 + 1.13.0 test From 993536f8601a71cec4e65d110ae8d81ac112b3d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Wed, 5 Feb 2025 06:30:06 +0100 Subject: [PATCH 070/408] GH-579: Add and fix LICENSE.txt and NOTICE.txt in the distributed artifacts (#578) Fixes #579. In our distributed artifacts, especially the shading ones, we have to provide `LICENSE.txt` and `NOTICE.txt` with all "bundled" dependencies. --- LICENSE.txt | 23 +- NOTICE.txt | 271 +++- flight/flight-integration-tests/pom.xml | 20 + .../src/shade/LICENSE.txt | 1127 +++++++++++++++++ .../src/shade/NOTICE.txt | 340 +++++ flight/flight-sql-jdbc-driver/pom.xml | 15 + .../src/shade/LICENSE.txt | 367 ++++++ .../src/shade/NOTICE.txt | 340 +++++ .../driver/jdbc/ITDriverJarValidation.java | 2 + tools/pom.xml | 18 + tools/src/shade/LICENSE.txt | 331 +++++ tools/src/shade/NOTICE.txt | 273 ++++ vector/pom.xml | 19 + vector/src/shade/LICENSE.txt | 209 +++ vector/src/shade/NOTICE.txt | 5 + 15 files changed, 3344 insertions(+), 16 deletions(-) create mode 100644 flight/flight-integration-tests/src/shade/LICENSE.txt create mode 100644 flight/flight-integration-tests/src/shade/NOTICE.txt create mode 100644 flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt create mode 100644 flight/flight-sql-jdbc-driver/src/shade/NOTICE.txt create mode 100644 tools/src/shade/LICENSE.txt create mode 100644 tools/src/shade/NOTICE.txt create mode 100644 vector/src/shade/LICENSE.txt create mode 100644 vector/src/shade/NOTICE.txt diff --git a/LICENSE.txt b/LICENSE.txt index c45ec81a93..f2e413b63d 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -202,16 +202,21 @@ limitations under the License. -------------------------------------------------------------------------------- -vector/src/main/java/org/apache/arrow/vector/util/IntObjectHashMap.java -vector/src/main/java/org/apache/arrow/vector/util/IntObjectMap.java +This product includes code from Netty 4.1.117.Final: -These file are derived from code from Netty(4.1.117), which is made available under the -Apache License 2.0. +* vector/src/main/java/org/apache/arrow/vector/util/IntObjectHashMap.java +* vector/src/main/java/org/apache/arrow/vector/util/IntObjectMap.java +* memory/memory-core/src/main/java/org/apache/arrow/memory/rounding/DefaultRoundingPolicy.java + +Copyright: 2014 The Netty Project +Home page: https://netty.io/ +License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- -memory/memory-core/src/main/java/org/apache/arrow/util/Preconditions.java +This product includes code from Google Guava 33.4.0-jre + +* memory/memory-core/src/main/java/org/apache/arrow/util/Preconditions.java -This product includes software from Google Guava(33.4.0), which is made available under the -Apache License 2.0. - * Copyright (C) 2007 The Guava Authors - * https://github.com/google/guava +Copyright: (C) 2011 The Guava Authors +Home page: https://github.com/google/guava/ +License: https://www.apache.org/licenses/LICENSE-2.0 diff --git a/NOTICE.txt b/NOTICE.txt index da061b7926..fd8dd2802a 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -4,12 +4,269 @@ Copyright 2016-2025 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). -This product contains code from the Netty Project: --------------------------------------------------- -The Netty Project -================= -Please visit the Netty web site for more information: - * http://netty.io/ +This product includes code from Netty 4.1.117.Final, with the following in its NOTICE: -Copyright 2014 The Netty Project ---------------------------------------------------- +| The Netty Project +| ================= +| +| Please visit the Netty web site for more information: +| +| * https://netty.io/ +| +| Copyright 2014 The Netty Project +| +| The Netty Project licenses this file to you under the Apache License, +| version 2.0 (the "License"); you may not use this file except in compliance +| with the License. You may obtain a copy of the License at: +| +| https://www.apache.org/licenses/LICENSE-2.0 +| +| Unless required by applicable law or agreed to in writing, software +| distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +| WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +| License for the specific language governing permissions and limitations +| under the License. +| +| Also, please refer to each LICENSE..txt file, which is located in +| the 'license' directory of the distribution file, for the license terms of the +| components that this product depends on. +| +| ------------------------------------------------------------------------------- +| This product contains the extensions to Java Collections Framework which has +| been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: +| +| * LICENSE: +| * license/LICENSE.jsr166y.txt (Public Domain) +| * HOMEPAGE: +| * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ +| * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ +| +| This product contains a modified version of Robert Harder's Public Domain +| Base64 Encoder and Decoder, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.base64.txt (Public Domain) +| * HOMEPAGE: +| * http://iharder.sourceforge.net/current/java/base64/ +| +| This product contains a modified portion of 'Webbit', an event based +| WebSocket and HTTP server, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.webbit.txt (BSD License) +| * HOMEPAGE: +| * https://github.com/joewalnes/webbit +| +| This product contains a modified portion of 'SLF4J', a simple logging +| facade for Java, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.slf4j.txt (MIT License) +| * HOMEPAGE: +| * https://www.slf4j.org/ +| +| This product contains a modified portion of 'Apache Harmony', an open source +| Java SE, which can be obtained at: +| +| * NOTICE: +| * license/NOTICE.harmony.txt +| * LICENSE: +| * license/LICENSE.harmony.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://archive.apache.org/dist/harmony/ +| +| This product contains a modified portion of 'jbzip2', a Java bzip2 compression +| and decompression library written by Matthew J. Francis. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.jbzip2.txt (MIT License) +| * HOMEPAGE: +| * https://code.google.com/p/jbzip2/ +| +| This product contains a modified portion of 'libdivsufsort', a C API library to construct +| the suffix array and the Burrows-Wheeler transformed string for any input string of +| a constant-size alphabet written by Yuta Mori. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.libdivsufsort.txt (MIT License) +| * HOMEPAGE: +| * https://github.com/y-256/libdivsufsort +| +| This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, +| which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.jctools.txt (ASL2 License) +| * HOMEPAGE: +| * https://github.com/JCTools/JCTools +| +| This product optionally depends on 'JZlib', a re-implementation of zlib in +| pure Java, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.jzlib.txt (BSD style License) +| * HOMEPAGE: +| * http://www.jcraft.com/jzlib/ +| +| This product optionally depends on 'Compress-LZF', a Java library for encoding and +| decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.compress-lzf.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/ning/compress +| +| This product optionally depends on 'lz4', a LZ4 Java compression +| and decompression library written by Adrien Grand. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.lz4.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/jpountz/lz4-java +| +| This product optionally depends on 'lzma-java', a LZMA Java compression +| and decompression library, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.lzma-java.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/jponge/lzma-java +| +| This product optionally depends on 'zstd-jni', a zstd-jni Java compression +| and decompression library, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.zstd-jni.txt (BSD) +| * HOMEPAGE: +| * https://github.com/luben/zstd-jni +| +| This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression +| and decompression library written by William Kinney. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.jfastlz.txt (MIT License) +| * HOMEPAGE: +| * https://code.google.com/p/jfastlz/ +| +| This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data +| interchange format, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.protobuf.txt (New BSD License) +| * HOMEPAGE: +| * https://github.com/google/protobuf +| +| This product optionally depends on 'Bouncy Castle Crypto APIs' to generate +| a temporary self-signed X.509 certificate when the JVM does not provide the +| equivalent functionality. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.bouncycastle.txt (MIT License) +| * HOMEPAGE: +| * https://www.bouncycastle.org/ +| +| This product optionally depends on 'Snappy', a compression library produced +| by Google Inc, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.snappy.txt (New BSD License) +| * HOMEPAGE: +| * https://github.com/google/snappy +| +| This product optionally depends on 'JBoss Marshalling', an alternative Java +| serialization API, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.jboss-marshalling.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/jboss-remoting/jboss-marshalling +| +| This product optionally depends on 'Caliper', Google's micro- +| benchmarking framework, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.caliper.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/google/caliper +| +| This product optionally depends on 'Apache Commons Logging', a logging +| framework, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.commons-logging.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://commons.apache.org/logging/ +| +| This product optionally depends on 'Apache Log4J', a logging framework, which +| can be obtained at: +| +| * LICENSE: +| * license/LICENSE.log4j.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://logging.apache.org/log4j/ +| +| This product optionally depends on 'Aalto XML', an ultra-high performance +| non-blocking XML processor, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.aalto-xml.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://wiki.fasterxml.com/AaltoHome +| +| This product contains a modified version of 'HPACK', a Java implementation of +| the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.hpack.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/twitter/hpack +| +| This product contains a modified version of 'HPACK', a Java implementation of +| the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.hyper-hpack.txt (MIT License) +| * HOMEPAGE: +| * https://github.com/python-hyper/hpack/ +| +| This product contains a modified version of 'HPACK', a Java implementation of +| the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.nghttp2-hpack.txt (MIT License) +| * HOMEPAGE: +| * https://github.com/nghttp2/nghttp2/ +| +| This product contains a modified portion of 'Apache Commons Lang', a Java library +| provides utilities for the java.lang API, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.commons-lang.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://commons.apache.org/proper/commons-lang/ +| +| +| This product contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. +| +| * LICENSE: +| * license/LICENSE.mvn-wrapper.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/takari/maven-wrapper +| +| This product contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. +| This private header is also used by Apple's open source +| mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). +| +| * LICENSE: +| * license/LICENSE.dnsinfo.txt (Apple Public Source License 2.0) +| * HOMEPAGE: +| * https://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h +| +| This product optionally depends on 'Brotli4j', Brotli compression and +| decompression for Java., which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.brotli4j.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/hyperxpro/Brotli4j diff --git a/flight/flight-integration-tests/pom.xml b/flight/flight-integration-tests/pom.xml index 704a26b549..b1e06d2f07 100644 --- a/flight/flight-integration-tests/pom.xml +++ b/flight/flight-integration-tests/pom.xml @@ -68,7 +68,9 @@ under the License. + + org.apache.maven.plugins maven-shade-plugin @@ -87,9 +89,27 @@ under the License. **/module-info.class + + *:* + + LICENSE.txt + NOTICE.txt + META-INF/*LICENSE* + META-INF/*NOTICE* + META-INF/license/* + + + + META-INF/LICENSE.txt + src/shade/LICENSE.txt + + + META-INF/NOTICE.txt + src/shade/NOTICE.txt + diff --git a/flight/flight-integration-tests/src/shade/LICENSE.txt b/flight/flight-integration-tests/src/shade/LICENSE.txt new file mode 100644 index 0000000000..0c411ce9ba --- /dev/null +++ b/flight/flight-integration-tests/src/shade/LICENSE.txt @@ -0,0 +1,1127 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------------------------------------- + +This binary artifact contains Jackson 2.18.2. + +Home page: https://github.com/FasterXML/jackson +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains Apache Commons Codec 1.18.0. + +Copyright: Copyright 2002-2024 The Apache Software Foundation +Home page: https://commons.apache.org/proper/commons-codec/ +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains Google Flatbuffers 24.3.25. + +Home page: https://flatbuffers.dev/ +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains Netty 4.1.117.Final. + +Copyright: Copyright 2014 The Netty Project +Home page: https://netty.io/ +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains gRPC 1.70.0. + +Copyright: Copyright 2014 The gRPC Authors +Home page: https://grpc.io/ +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains Animal Sniffer Annotations 1.24. + +Copyright: Copyright (c) 2009 codehaus.org. +Home page: https://www.mojohaus.org/animal-sniffer/animal-sniffer-annotations/ +License: MIT (https://github.com/mojohaus/animal-sniffer/blob/animal-sniffer-1.24/LICENSE) + +-------------------------------------------------------------------------------- + +This binary artifact contains Perfmark 0.27.0. + +Copyright: Copyright 2019 Google LLC +Home page: https://github.com/perfmark/perfmark +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains Gson 2.11.0. + +Copyright: Copyright 2008 Google Inc. +Home page: https://github.com/google/gson +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains Guava 33.4.0-jre. + +Home page: https://github.com/google/guava +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains Google Protobuf 3.25.5. + +Copyright: Copyright 2008 Google Inc. All rights reserved. +Home page: https://protobuf.dev/ +License: https://github.com/protocolbuffers/protobuf/blob/v3.25.5/LICENSE (BSD) +License text: + +| Copyright 2008 Google Inc. All rights reserved. +| +| Redistribution and use in source and binary forms, with or without +| modification, are permitted provided that the following conditions are +| met: +| +| * Redistributions of source code must retain the above copyright +| notice, this list of conditions and the following disclaimer. +| * Redistributions in binary form must reproduce the above +| copyright notice, this list of conditions and the following disclaimer +| in the documentation and/or other materials provided with the +| distribution. +| * Neither the name of Google Inc. nor the names of its +| contributors may be used to endorse or promote products derived from +| this software without specific prior written permission. +| +| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +| "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +| LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +| A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +| OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +| SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +| LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +| DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +| THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +| (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +| OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +| +| Code generated by the Protocol Buffer compiler is owned by the owner +| of the input file used when generating it. This code is not +| standalone and requires a support library to be linked with it. This +| support library is itself covered by the above license. + +-------------------------------------------------------------------------------- + +This binary artifact contains Javax Annotation 1.3.2. + +Home page: https://github.com/javaee/javax.annotation +License: CDDL 1.1 +License text: + +| COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.1 +| +| 1. Definitions. +| +| 1.1. "Contributor" means each individual or entity that creates or +| contributes to the creation of Modifications. +| +| 1.2. "Contributor Version" means the combination of the Original +| Software, prior Modifications used by a Contributor (if any), and +| the Modifications made by that particular Contributor. +| +| 1.3. "Covered Software" means (a) the Original Software, or (b) +| Modifications, or (c) the combination of files containing Original +| Software with files containing Modifications, in each case including +| portions thereof. +| +| 1.4. "Executable" means the Covered Software in any form other than +| Source Code. +| +| 1.5. "Initial Developer" means the individual or entity that first +| makes Original Software available under this License. +| +| 1.6. "Larger Work" means a work which combines Covered Software or +| portions thereof with code not governed by the terms of this License. +| +| 1.7. "License" means this document. +| +| 1.8. "Licensable" means having the right to grant, to the maximum +| extent possible, whether at the time of the initial grant or +| subsequently acquired, any and all of the rights conveyed herein. +| +| 1.9. "Modifications" means the Source Code and Executable form of +| any of the following: +| +| A. Any file that results from an addition to, deletion from or +| modification of the contents of a file containing Original Software +| or previous Modifications; +| +| B. Any new file that contains any part of the Original Software or +| previous Modification; or +| +| C. Any new file that is contributed or otherwise made available +| under the terms of this License. +| +| 1.10. "Original Software" means the Source Code and Executable form +| of computer software code that is originally released under this +| License. +| +| 1.11. "Patent Claims" means any patent claim(s), now owned or +| hereafter acquired, including without limitation, method, process, +| and apparatus claims, in any patent Licensable by grantor. +| +| 1.12. "Source Code" means (a) the common form of computer software +| code in which modifications are made and (b) associated +| documentation included in or with such code. +| +| 1.13. "You" (or "Your") means an individual or a legal entity +| exercising rights under, and complying with all of the terms of, +| this License. For legal entities, "You" includes any entity which +| controls, is controlled by, or is under common control with You. For +| purposes of this definition, "control" means (a) the power, direct +| or indirect, to cause the direction or management of such entity, +| whether by contract or otherwise, or (b) ownership of more than +| fifty percent (50%) of the outstanding shares or beneficial +| ownership of such entity. +| +| 2. License Grants. +| +| 2.1. The Initial Developer Grant. +| +| Conditioned upon Your compliance with Section 3.1 below and subject +| to third party intellectual property claims, the Initial Developer +| hereby grants You a world-wide, royalty-free, non-exclusive license: +| +| (a) under intellectual property rights (other than patent or +| trademark) Licensable by Initial Developer, to use, reproduce, +| modify, display, perform, sublicense and distribute the Original +| Software (or portions thereof), with or without Modifications, +| and/or as part of a Larger Work; and +| +| (b) under Patent Claims infringed by the making, using or selling of +| Original Software, to make, have made, use, practice, sell, and +| offer for sale, and/or otherwise dispose of the Original Software +| (or portions thereof). +| +| (c) The licenses granted in Sections 2.1(a) and (b) are effective on +| the date Initial Developer first distributes or otherwise makes the +| Original Software available to a third party under the terms of this +| License. +| +| (d) Notwithstanding Section 2.1(b) above, no patent license is +| granted: (1) for code that You delete from the Original Software, or +| (2) for infringements caused by: (i) the modification of the +| Original Software, or (ii) the combination of the Original Software +| with other software or devices. +| +| 2.2. Contributor Grant. +| +| Conditioned upon Your compliance with Section 3.1 below and subject +| to third party intellectual property claims, each Contributor hereby +| grants You a world-wide, royalty-free, non-exclusive license: +| +| (a) under intellectual property rights (other than patent or +| trademark) Licensable by Contributor to use, reproduce, modify, +| display, perform, sublicense and distribute the Modifications +| created by such Contributor (or portions thereof), either on an +| unmodified basis, with other Modifications, as Covered Software +| and/or as part of a Larger Work; and +| +| (b) under Patent Claims infringed by the making, using, or selling +| of Modifications made by that Contributor either alone and/or in +| combination with its Contributor Version (or portions of such +| combination), to make, use, sell, offer for sale, have made, and/or +| otherwise dispose of: (1) Modifications made by that Contributor (or +| portions thereof); and (2) the combination of Modifications made by +| that Contributor with its Contributor Version (or portions of such +| combination). +| +| (c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective +| on the date Contributor first distributes or otherwise makes the +| Modifications available to a third party. +| +| (d) Notwithstanding Section 2.2(b) above, no patent license is +| granted: (1) for any code that Contributor has deleted from the +| Contributor Version; (2) for infringements caused by: (i) third +| party modifications of Contributor Version, or (ii) the combination +| of Modifications made by that Contributor with other software +| (except as part of the Contributor Version) or other devices; or (3) +| under Patent Claims infringed by Covered Software in the absence of +| Modifications made by that Contributor. +| +| 3. Distribution Obligations. +| +| 3.1. Availability of Source Code. +| +| Any Covered Software that You distribute or otherwise make available +| in Executable form must also be made available in Source Code form +| and that Source Code form must be distributed only under the terms +| of this License. You must include a copy of this License with every +| copy of the Source Code form of the Covered Software You distribute +| or otherwise make available. You must inform recipients of any such +| Covered Software in Executable form as to how they can obtain such +| Covered Software in Source Code form in a reasonable manner on or +| through a medium customarily used for software exchange. +| +| 3.2. Modifications. +| +| The Modifications that You create or to which You contribute are +| governed by the terms of this License. You represent that You +| believe Your Modifications are Your original creation(s) and/or You +| have sufficient rights to grant the rights conveyed by this License. +| +| 3.3. Required Notices. +| +| You must include a notice in each of Your Modifications that +| identifies You as the Contributor of the Modification. You may not +| remove or alter any copyright, patent or trademark notices contained +| within the Covered Software, or any notices of licensing or any +| descriptive text giving attribution to any Contributor or the +| Initial Developer. +| +| 3.4. Application of Additional Terms. +| +| You may not offer or impose any terms on any Covered Software in +| Source Code form that alters or restricts the applicable version of +| this License or the recipients' rights hereunder. You may choose to +| offer, and to charge a fee for, warranty, support, indemnity or +| liability obligations to one or more recipients of Covered Software. +| However, you may do so only on Your own behalf, and not on behalf of +| the Initial Developer or any Contributor. You must make it +| absolutely clear that any such warranty, support, indemnity or +| liability obligation is offered by You alone, and You hereby agree +| to indemnify the Initial Developer and every Contributor for any +| liability incurred by the Initial Developer or such Contributor as a +| result of warranty, support, indemnity or liability terms You offer. +| +| 3.5. Distribution of Executable Versions. +| +| You may distribute the Executable form of the Covered Software under +| the terms of this License or under the terms of a license of Your +| choice, which may contain terms different from this License, +| provided that You are in compliance with the terms of this License +| and that the license for the Executable form does not attempt to +| limit or alter the recipient's rights in the Source Code form from +| the rights set forth in this License. If You distribute the Covered +| Software in Executable form under a different license, You must make +| it absolutely clear that any terms which differ from this License +| are offered by You alone, not by the Initial Developer or +| Contributor. You hereby agree to indemnify the Initial Developer and +| every Contributor for any liability incurred by the Initial +| Developer or such Contributor as a result of any such terms You offer. +| +| 3.6. Larger Works. +| +| You may create a Larger Work by combining Covered Software with +| other code not governed by the terms of this License and distribute +| the Larger Work as a single product. In such a case, You must make +| sure the requirements of this License are fulfilled for the Covered +| Software. +| +| 4. Versions of the License. +| +| 4.1. New Versions. +| +| Oracle is the initial license steward and may publish revised and/or +| new versions of this License from time to time. Each version will be +| given a distinguishing version number. Except as provided in Section +| 4.3, no one other than the license steward has the right to modify +| this License. +| +| 4.2. Effect of New Versions. +| +| You may always continue to use, distribute or otherwise make the +| Covered Software available under the terms of the version of the +| License under which You originally received the Covered Software. If +| the Initial Developer includes a notice in the Original Software +| prohibiting it from being distributed or otherwise made available +| under any subsequent version of the License, You must distribute and +| make the Covered Software available under the terms of the version +| of the License under which You originally received the Covered +| Software. Otherwise, You may also choose to use, distribute or +| otherwise make the Covered Software available under the terms of any +| subsequent version of the License published by the license steward. +| +| 4.3. Modified Versions. +| +| When You are an Initial Developer and You want to create a new +| license for Your Original Software, You may create and use a +| modified version of this License if You: (a) rename the license and +| remove any references to the name of the license steward (except to +| note that the license differs from this License); and (b) otherwise +| make it clear that the license contains terms which differ from this +| License. +| +| 5. DISCLAIMER OF WARRANTY. +| +| COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS, +| WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, +| INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE +| IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR +| NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF +| THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE +| DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY +| OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, +| REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN +| ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED SOFTWARE IS +| AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. +| +| 6. TERMINATION. +| +| 6.1. This License and the rights granted hereunder will terminate +| automatically if You fail to comply with terms herein and fail to +| cure such breach within 30 days of becoming aware of the breach. +| Provisions which, by their nature, must remain in effect beyond the +| termination of this License shall survive. +| +| 6.2. If You assert a patent infringement claim (excluding +| declaratory judgment actions) against Initial Developer or a +| Contributor (the Initial Developer or Contributor against whom You +| assert such claim is referred to as "Participant") alleging that the +| Participant Software (meaning the Contributor Version where the +| Participant is a Contributor or the Original Software where the +| Participant is the Initial Developer) directly or indirectly +| infringes any patent, then any and all rights granted directly or +| indirectly to You by such Participant, the Initial Developer (if the +| Initial Developer is not the Participant) and all Contributors under +| Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice +| from Participant terminate prospectively and automatically at the +| expiration of such 60 day notice period, unless if within such 60 +| day period You withdraw Your claim with respect to the Participant +| Software against such Participant either unilaterally or pursuant to +| a written agreement with Participant. +| +| 6.3. If You assert a patent infringement claim against Participant +| alleging that the Participant Software directly or indirectly +| infringes any patent where such claim is resolved (such as by +| license or settlement) prior to the initiation of patent +| infringement litigation, then the reasonable value of the licenses +| granted by such Participant under Sections 2.1 or 2.2 shall be taken +| into account in determining the amount or value of any payment or +| license. +| +| 6.4. In the event of termination under Sections 6.1 or 6.2 above, +| all end user licenses that have been validly granted by You or any +| distributor hereunder prior to termination (excluding licenses +| granted to You by any distributor) shall survive termination. +| +| 7. LIMITATION OF LIABILITY. +| +| UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT +| (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE +| INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF +| COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE +| TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR +| CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT +| LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER +| FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR +| LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE +| POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT +| APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH +| PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW PROHIBITS SUCH +| LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR +| LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION +| AND LIMITATION MAY NOT APPLY TO YOU. +| +| 8. U.S. GOVERNMENT END USERS. +| +| The Covered Software is a "commercial item," as that term is defined +| in 48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial computer +| software" (as that term is defined at 48 C.F.R. § +| 252.227-7014(a)(1)) and "commercial computer software documentation" +| as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent +| with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 +| (June 1995), all U.S. Government End Users acquire Covered Software +| with only those rights set forth herein. This U.S. Government Rights +| clause is in lieu of, and supersedes, any other FAR, DFAR, or other +| clause or provision that addresses Government rights in computer +| software under this License. +| +| 9. MISCELLANEOUS. +| +| This License represents the complete agreement concerning subject +| matter hereof. If any provision of this License is held to be +| unenforceable, such provision shall be reformed only to the extent +| necessary to make it enforceable. This License shall be governed by +| the law of the jurisdiction specified in a notice contained within +| the Original Software (except to the extent applicable law, if any, +| provides otherwise), excluding such jurisdiction's conflict-of-law +| provisions. Any litigation relating to this License shall be subject +| to the jurisdiction of the courts located in the jurisdiction and +| venue specified in a notice contained within the Original Software, +| with the losing party responsible for costs, including, without +| limitation, court costs and reasonable attorneys' fees and expenses. +| The application of the United Nations Convention on Contracts for +| the International Sale of Goods is expressly excluded. Any law or +| regulation which provides that the language of a contract shall be +| construed against the drafter shall not apply to this License. You +| agree that You alone are responsible for compliance with the United +| States export administration regulations (and the export control +| laws and regulation of any other countries) when You use, distribute +| or otherwise make available any Covered Software. +| +| 10. RESPONSIBILITY FOR CLAIMS. +| +| As between Initial Developer and the Contributors, each party is +| responsible for claims and damages arising, directly or indirectly, +| out of its utilization of rights under this License and You agree to +| work with Initial Developer and Contributors to distribute such +| responsibility on an equitable basis. Nothing herein is intended or +| shall be deemed to constitute any admission of liability. +| +| ------------------------------------------------------------------------ +| +| NOTICE PURSUANT TO SECTION 9 OF THE COMMON DEVELOPMENT AND DISTRIBUTION +| LICENSE (CDDL) +| +| The code released under the CDDL shall be governed by the laws of the +| State of California (excluding conflict-of-law provisions). Any +| litigation relating to this License shall be subject to the jurisdiction +| of the Federal Courts of the Northern District of California and the +| state courts of the State of California, with venue lying in Santa Clara +| County, California. +| +| +| +| The GNU General Public License (GPL) Version 2, June 1991 +| +| Copyright (C) 1989, 1991 Free Software Foundation, Inc. +| 51 Franklin Street, Fifth Floor +| Boston, MA 02110-1335 +| USA +| +| Everyone is permitted to copy and distribute verbatim copies +| of this license document, but changing it is not allowed. +| +| Preamble +| +| The licenses for most software are designed to take away your freedom to +| share and change it. By contrast, the GNU General Public License is +| intended to guarantee your freedom to share and change free software--to +| make sure the software is free for all its users. This General Public +| License applies to most of the Free Software Foundation's software and +| to any other program whose authors commit to using it. (Some other Free +| Software Foundation software is covered by the GNU Library General +| Public License instead.) You can apply it to your programs, too. +| +| When we speak of free software, we are referring to freedom, not price. +| Our General Public Licenses are designed to make sure that you have the +| freedom to distribute copies of free software (and charge for this +| service if you wish), that you receive source code or can get it if you +| want it, that you can change the software or use pieces of it in new +| free programs; and that you know you can do these things. +| +| To protect your rights, we need to make restrictions that forbid anyone +| to deny you these rights or to ask you to surrender the rights. These +| restrictions translate to certain responsibilities for you if you +| distribute copies of the software, or if you modify it. +| +| For example, if you distribute copies of such a program, whether gratis +| or for a fee, you must give the recipients all the rights that you have. +| You must make sure that they, too, receive or can get the source code. +| And you must show them these terms so they know their rights. +| +| We protect your rights with two steps: (1) copyright the software, and +| (2) offer you this license which gives you legal permission to copy, +| distribute and/or modify the software. +| +| Also, for each author's protection and ours, we want to make certain +| that everyone understands that there is no warranty for this free +| software. If the software is modified by someone else and passed on, we +| want its recipients to know that what they have is not the original, so +| that any problems introduced by others will not reflect on the original +| authors' reputations. +| +| Finally, any free program is threatened constantly by software patents. +| We wish to avoid the danger that redistributors of a free program will +| individually obtain patent licenses, in effect making the program +| proprietary. To prevent this, we have made it clear that any patent must +| be licensed for everyone's free use or not licensed at all. +| +| The precise terms and conditions for copying, distribution and +| modification follow. +| +| TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION +| +| 0. This License applies to any program or other work which contains a +| notice placed by the copyright holder saying it may be distributed under +| the terms of this General Public License. The "Program", below, refers +| to any such program or work, and a "work based on the Program" means +| either the Program or any derivative work under copyright law: that is +| to say, a work containing the Program or a portion of it, either +| verbatim or with modifications and/or translated into another language. +| (Hereinafter, translation is included without limitation in the term +| "modification".) Each licensee is addressed as "you". +| +| Activities other than copying, distribution and modification are not +| covered by this License; they are outside its scope. The act of running +| the Program is not restricted, and the output from the Program is +| covered only if its contents constitute a work based on the Program +| (independent of having been made by running the Program). Whether that +| is true depends on what the Program does. +| +| 1. You may copy and distribute verbatim copies of the Program's source +| code as you receive it, in any medium, provided that you conspicuously +| and appropriately publish on each copy an appropriate copyright notice +| and disclaimer of warranty; keep intact all the notices that refer to +| this License and to the absence of any warranty; and give any other +| recipients of the Program a copy of this License along with the Program. +| +| You may charge a fee for the physical act of transferring a copy, and +| you may at your option offer warranty protection in exchange for a fee. +| +| 2. You may modify your copy or copies of the Program or any portion of +| it, thus forming a work based on the Program, and copy and distribute +| such modifications or work under the terms of Section 1 above, provided +| that you also meet all of these conditions: +| +| a) You must cause the modified files to carry prominent notices +| stating that you changed the files and the date of any change. +| +| b) You must cause any work that you distribute or publish, that in +| whole or in part contains or is derived from the Program or any part +| thereof, to be licensed as a whole at no charge to all third parties +| under the terms of this License. +| +| c) If the modified program normally reads commands interactively +| when run, you must cause it, when started running for such +| interactive use in the most ordinary way, to print or display an +| announcement including an appropriate copyright notice and a notice +| that there is no warranty (or else, saying that you provide a +| warranty) and that users may redistribute the program under these +| conditions, and telling the user how to view a copy of this License. +| (Exception: if the Program itself is interactive but does not +| normally print such an announcement, your work based on the Program +| is not required to print an announcement.) +| +| These requirements apply to the modified work as a whole. If +| identifiable sections of that work are not derived from the Program, and +| can be reasonably considered independent and separate works in +| themselves, then this License, and its terms, do not apply to those +| sections when you distribute them as separate works. But when you +| distribute the same sections as part of a whole which is a work based on +| the Program, the distribution of the whole must be on the terms of this +| License, whose permissions for other licensees extend to the entire +| whole, and thus to each and every part regardless of who wrote it. +| +| Thus, it is not the intent of this section to claim rights or contest +| your rights to work written entirely by you; rather, the intent is to +| exercise the right to control the distribution of derivative or +| collective works based on the Program. +| +| In addition, mere aggregation of another work not based on the Program +| with the Program (or with a work based on the Program) on a volume of a +| storage or distribution medium does not bring the other work under the +| scope of this License. +| +| 3. You may copy and distribute the Program (or a work based on it, +| under Section 2) in object code or executable form under the terms of +| Sections 1 and 2 above provided that you also do one of the following: +| +| a) Accompany it with the complete corresponding machine-readable +| source code, which must be distributed under the terms of Sections 1 +| and 2 above on a medium customarily used for software interchange; or, +| +| b) Accompany it with a written offer, valid for at least three +| years, to give any third party, for a charge no more than your cost +| of physically performing source distribution, a complete +| machine-readable copy of the corresponding source code, to be +| distributed under the terms of Sections 1 and 2 above on a medium +| customarily used for software interchange; or, +| +| c) Accompany it with the information you received as to the offer to +| distribute corresponding source code. (This alternative is allowed +| only for noncommercial distribution and only if you received the +| program in object code or executable form with such an offer, in +| accord with Subsection b above.) +| +| The source code for a work means the preferred form of the work for +| making modifications to it. For an executable work, complete source code +| means all the source code for all modules it contains, plus any +| associated interface definition files, plus the scripts used to control +| compilation and installation of the executable. However, as a special +| exception, the source code distributed need not include anything that is +| normally distributed (in either source or binary form) with the major +| components (compiler, kernel, and so on) of the operating system on +| which the executable runs, unless that component itself accompanies the +| executable. +| +| If distribution of executable or object code is made by offering access +| to copy from a designated place, then offering equivalent access to copy +| the source code from the same place counts as distribution of the source +| code, even though third parties are not compelled to copy the source +| along with the object code. +| +| 4. You may not copy, modify, sublicense, or distribute the Program +| except as expressly provided under this License. Any attempt otherwise +| to copy, modify, sublicense or distribute the Program is void, and will +| automatically terminate your rights under this License. However, parties +| who have received copies, or rights, from you under this License will +| not have their licenses terminated so long as such parties remain in +| full compliance. +| +| 5. You are not required to accept this License, since you have not +| signed it. However, nothing else grants you permission to modify or +| distribute the Program or its derivative works. These actions are +| prohibited by law if you do not accept this License. Therefore, by +| modifying or distributing the Program (or any work based on the +| Program), you indicate your acceptance of this License to do so, and all +| its terms and conditions for copying, distributing or modifying the +| Program or works based on it. +| +| 6. Each time you redistribute the Program (or any work based on the +| Program), the recipient automatically receives a license from the +| original licensor to copy, distribute or modify the Program subject to +| these terms and conditions. You may not impose any further restrictions +| on the recipients' exercise of the rights granted herein. You are not +| responsible for enforcing compliance by third parties to this License. +| +| 7. If, as a consequence of a court judgment or allegation of patent +| infringement or for any other reason (not limited to patent issues), +| conditions are imposed on you (whether by court order, agreement or +| otherwise) that contradict the conditions of this License, they do not +| excuse you from the conditions of this License. If you cannot distribute +| so as to satisfy simultaneously your obligations under this License and +| any other pertinent obligations, then as a consequence you may not +| distribute the Program at all. For example, if a patent license would +| not permit royalty-free redistribution of the Program by all those who +| receive copies directly or indirectly through you, then the only way you +| could satisfy both it and this License would be to refrain entirely from +| distribution of the Program. +| +| If any portion of this section is held invalid or unenforceable under +| any particular circumstance, the balance of the section is intended to +| apply and the section as a whole is intended to apply in other +| circumstances. +| +| It is not the purpose of this section to induce you to infringe any +| patents or other property right claims or to contest validity of any +| such claims; this section has the sole purpose of protecting the +| integrity of the free software distribution system, which is implemented +| by public license practices. Many people have made generous +| contributions to the wide range of software distributed through that +| system in reliance on consistent application of that system; it is up to +| the author/donor to decide if he or she is willing to distribute +| software through any other system and a licensee cannot impose that choice. +| +| This section is intended to make thoroughly clear what is believed to be +| a consequence of the rest of this License. +| +| 8. If the distribution and/or use of the Program is restricted in +| certain countries either by patents or by copyrighted interfaces, the +| original copyright holder who places the Program under this License may +| add an explicit geographical distribution limitation excluding those +| countries, so that distribution is permitted only in or among countries +| not thus excluded. In such case, this License incorporates the +| limitation as if written in the body of this License. +| +| 9. The Free Software Foundation may publish revised and/or new +| versions of the General Public License from time to time. Such new +| versions will be similar in spirit to the present version, but may +| differ in detail to address new problems or concerns. +| +| Each version is given a distinguishing version number. If the Program +| specifies a version number of this License which applies to it and "any +| later version", you have the option of following the terms and +| conditions either of that version or of any later version published by +| the Free Software Foundation. If the Program does not specify a version +| number of this License, you may choose any version ever published by the +| Free Software Foundation. +| +| 10. If you wish to incorporate parts of the Program into other free +| programs whose distribution conditions are different, write to the +| author to ask for permission. For software which is copyrighted by the +| Free Software Foundation, write to the Free Software Foundation; we +| sometimes make exceptions for this. Our decision will be guided by the +| two goals of preserving the free status of all derivatives of our free +| software and of promoting the sharing and reuse of software generally. +| +| NO WARRANTY +| +| 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO +| WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +| EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +| OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, +| EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +| WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE +| ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH +| YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL +| NECESSARY SERVICING, REPAIR OR CORRECTION. +| +| 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +| WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +| AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR +| DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL +| DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM +| (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED +| INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF +| THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR +| OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. +| +| END OF TERMS AND CONDITIONS +| +| How to Apply These Terms to Your New Programs +| +| If you develop a new program, and you want it to be of the greatest +| possible use to the public, the best way to achieve this is to make it +| free software which everyone can redistribute and change under these terms. +| +| To do so, attach the following notices to the program. It is safest to +| attach them to the start of each source file to most effectively convey +| the exclusion of warranty; and each file should have at least the +| "copyright" line and a pointer to where the full notice is found. +| +| One line to give the program's name and a brief idea of what it does. +| Copyright (C) +| +| This program is free software; you can redistribute it and/or modify +| it under the terms of the GNU General Public License as published by +| the Free Software Foundation; either version 2 of the License, or +| (at your option) any later version. +| +| This program is distributed in the hope that it will be useful, but +| WITHOUT ANY WARRANTY; without even the implied warranty of +| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +| General Public License for more details. +| +| You should have received a copy of the GNU General Public License +| along with this program; if not, write to the Free Software +| Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA +| +| Also add information on how to contact you by electronic and paper mail. +| +| If the program is interactive, make it output a short notice like this +| when it starts in an interactive mode: +| +| Gnomovision version 69, Copyright (C) year name of author +| Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type +| `show w'. This is free software, and you are welcome to redistribute +| it under certain conditions; type `show c' for details. +| +| The hypothetical commands `show w' and `show c' should show the +| appropriate parts of the General Public License. Of course, the commands +| you use may be called something other than `show w' and `show c'; they +| could even be mouse-clicks or menu items--whatever suits your program. +| +| You should also get your employer (if you work as a programmer) or your +| school, if any, to sign a "copyright disclaimer" for the program, if +| necessary. Here is a sample; alter the names: +| +| Yoyodyne, Inc., hereby disclaims all copyright interest in the +| program `Gnomovision' (which makes passes at compilers) written by +| James Hacker. +| +| signature of Ty Coon, 1 April 1989 +| Ty Coon, President of Vice +| +| This General Public License does not permit incorporating your program +| into proprietary programs. If your program is a subroutine library, you +| may consider it more useful to permit linking proprietary applications +| with the library. If this is what you want to do, use the GNU Library +| General Public License instead of this License. +| +| # +| +| Certain source files distributed by Oracle America, Inc. and/or its +| affiliates are subject to the following clarification and special +| exception to the GPLv2, based on the GNU Project exception for its +| Classpath libraries, known as the GNU Classpath Exception, but only +| where Oracle has expressly included in the particular source file's +| header the words "Oracle designates this particular file as subject to +| the "Classpath" exception as provided by Oracle in the LICENSE file +| that accompanied this code." +| +| You should also note that Oracle includes multiple, independent +| programs in this software package. Some of those programs are provided +| under licenses deemed incompatible with the GPLv2 by the Free Software +| Foundation and others. For example, the package includes programs +| licensed under the Apache License, Version 2.0. Such programs are +| licensed to you under their original licenses. +| +| Oracle facilitates your further distribution of this package by adding +| the Classpath Exception to the necessary parts of its GPLv2 code, which +| permits you to use that code in combination with other independent +| modules not licensed under the GPLv2. However, note that this would +| not permit you to commingle code under an incompatible license with +| Oracle's GPLv2 licensed code by, for example, cutting and pasting such +| code into a file also containing Oracle's GPLv2 licensed code and then +| distributing the result. Additionally, if you were to remove the +| Classpath Exception from any of the files to which it applies and +| distribute the result, you would likely be required to license some or +| all of the other code in that distribution under the GPLv2 as well, and +| since the GPLv2 is incompatible with the license terms of some items +| included in the distribution by Oracle, removing the Classpath +| Exception could therefore effectively compromise your ability to +| further distribute the package. +| +| Proceed with caution and we recommend that you obtain the advice of a +| lawyer skilled in open source matters before removing the Classpath +| Exception or making modifications to this package which may +| subsequently be redistributed and/or involve the use of third party +| software. +| +| CLASSPATH EXCEPTION +| Linking this library statically or dynamically with other modules is +| making a combined work based on this library. Thus, the terms and +| conditions of the GNU General Public License version 2 cover the whole +| combination. +| +| As a special exception, the copyright holders of this library give you +| permission to link this library with independent modules to produce an +| executable, regardless of the license terms of these independent +| modules, and to copy and distribute the resulting executable under +| terms of your choice, provided that you also meet, for each linked +| independent module, the terms and conditions of the license of that +| module. An independent module is a module which is not derived from or +| based on this library. If you modify this library, you may extend this +| exception to your version of the library, but you are not obligated to +| do so. If you do not wish to do so, delete this exception statement +| from your version. + +-------------------------------------------------------------------------------- + +This binary artifact contains Google j2objc 3.0.0. + +Home page: http://j2objc.org/ +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains Apache Commons CLI 1.9.0. + +Copyright: Copyright 2002-2024 The Apache Software Foundation +Home page: https://commons.apache.org/proper/commons-cli/ +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains Slf4j 2.0.16. + +Copyright: Copyright (c) 2004-2022 QOS.ch Sarl (Switzerland) +Home page: http://www.slf4j.org/ +License: MIT +License text: + +| Copyright (c) 2004-2022 QOS.ch Sarl (Switzerland) +| All rights reserved. +| +| Permission is hereby granted, free of charge, to any person obtaining +| a copy of this software and associated documentation files (the +| "Software"), to deal in the Software without restriction, including +| without limitation the rights to use, copy, modify, merge, publish, +| distribute, sublicense, and/or sell copies of the Software, and to +| permit persons to whom the Software is furnished to do so, subject to +| the following conditions: +| +| The above copyright notice and this permission notice shall be +| included in all copies or substantial portions of the Software. +| +| THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +| EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +| MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +| NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +| LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +| OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +| WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/flight/flight-integration-tests/src/shade/NOTICE.txt b/flight/flight-integration-tests/src/shade/NOTICE.txt new file mode 100644 index 0000000000..f8cd4979c0 --- /dev/null +++ b/flight/flight-integration-tests/src/shade/NOTICE.txt @@ -0,0 +1,340 @@ +Apache Arrow Java +Copyright 2016-2025 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +--------------------------------------------------------- + +This product includes Netty 4.1.117.Final, with the following in its NOTICE: + +| The Netty Project +| ================= +| +| Please visit the Netty web site for more information: +| +| * https://netty.io/ +| +| Copyright 2014 The Netty Project +| +| The Netty Project licenses this file to you under the Apache License, +| version 2.0 (the "License"); you may not use this file except in compliance +| with the License. You may obtain a copy of the License at: +| +| https://www.apache.org/licenses/LICENSE-2.0 +| +| Unless required by applicable law or agreed to in writing, software +| distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +| WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +| License for the specific language governing permissions and limitations +| under the License. +| +| Also, please refer to each LICENSE..txt file, which is located in +| the 'license' directory of the distribution file, for the license terms of the +| components that this product depends on. +| +| ------------------------------------------------------------------------------- +| This product contains the extensions to Java Collections Framework which has +| been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: +| +| * LICENSE: +| * license/LICENSE.jsr166y.txt (Public Domain) +| * HOMEPAGE: +| * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ +| * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ +| +| This product contains a modified version of Robert Harder's Public Domain +| Base64 Encoder and Decoder, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.base64.txt (Public Domain) +| * HOMEPAGE: +| * http://iharder.sourceforge.net/current/java/base64/ +| +| This product contains a modified portion of 'Webbit', an event based +| WebSocket and HTTP server, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.webbit.txt (BSD License) +| * HOMEPAGE: +| * https://github.com/joewalnes/webbit +| +| This product contains a modified portion of 'SLF4J', a simple logging +| facade for Java, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.slf4j.txt (MIT License) +| * HOMEPAGE: +| * https://www.slf4j.org/ +| +| This product contains a modified portion of 'Apache Harmony', an open source +| Java SE, which can be obtained at: +| +| * NOTICE: +| * license/NOTICE.harmony.txt +| * LICENSE: +| * license/LICENSE.harmony.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://archive.apache.org/dist/harmony/ +| +| This product contains a modified portion of 'jbzip2', a Java bzip2 compression +| and decompression library written by Matthew J. Francis. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.jbzip2.txt (MIT License) +| * HOMEPAGE: +| * https://code.google.com/p/jbzip2/ +| +| This product contains a modified portion of 'libdivsufsort', a C API library to construct +| the suffix array and the Burrows-Wheeler transformed string for any input string of +| a constant-size alphabet written by Yuta Mori. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.libdivsufsort.txt (MIT License) +| * HOMEPAGE: +| * https://github.com/y-256/libdivsufsort +| +| This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, +| which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.jctools.txt (ASL2 License) +| * HOMEPAGE: +| * https://github.com/JCTools/JCTools +| +| This product optionally depends on 'JZlib', a re-implementation of zlib in +| pure Java, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.jzlib.txt (BSD style License) +| * HOMEPAGE: +| * http://www.jcraft.com/jzlib/ +| +| This product optionally depends on 'Compress-LZF', a Java library for encoding and +| decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.compress-lzf.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/ning/compress +| +| This product optionally depends on 'lz4', a LZ4 Java compression +| and decompression library written by Adrien Grand. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.lz4.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/jpountz/lz4-java +| +| This product optionally depends on 'lzma-java', a LZMA Java compression +| and decompression library, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.lzma-java.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/jponge/lzma-java +| +| This product optionally depends on 'zstd-jni', a zstd-jni Java compression +| and decompression library, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.zstd-jni.txt (BSD) +| * HOMEPAGE: +| * https://github.com/luben/zstd-jni +| +| This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression +| and decompression library written by William Kinney. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.jfastlz.txt (MIT License) +| * HOMEPAGE: +| * https://code.google.com/p/jfastlz/ +| +| This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data +| interchange format, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.protobuf.txt (New BSD License) +| * HOMEPAGE: +| * https://github.com/google/protobuf +| +| This product optionally depends on 'Bouncy Castle Crypto APIs' to generate +| a temporary self-signed X.509 certificate when the JVM does not provide the +| equivalent functionality. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.bouncycastle.txt (MIT License) +| * HOMEPAGE: +| * https://www.bouncycastle.org/ +| +| This product optionally depends on 'Snappy', a compression library produced +| by Google Inc, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.snappy.txt (New BSD License) +| * HOMEPAGE: +| * https://github.com/google/snappy +| +| This product optionally depends on 'JBoss Marshalling', an alternative Java +| serialization API, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.jboss-marshalling.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/jboss-remoting/jboss-marshalling +| +| This product optionally depends on 'Caliper', Google's micro- +| benchmarking framework, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.caliper.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/google/caliper +| +| This product optionally depends on 'Apache Commons Logging', a logging +| framework, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.commons-logging.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://commons.apache.org/logging/ +| +| This product optionally depends on 'Apache Log4J', a logging framework, which +| can be obtained at: +| +| * LICENSE: +| * license/LICENSE.log4j.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://logging.apache.org/log4j/ +| +| This product optionally depends on 'Aalto XML', an ultra-high performance +| non-blocking XML processor, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.aalto-xml.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://wiki.fasterxml.com/AaltoHome +| +| This product contains a modified version of 'HPACK', a Java implementation of +| the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.hpack.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/twitter/hpack +| +| This product contains a modified version of 'HPACK', a Java implementation of +| the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.hyper-hpack.txt (MIT License) +| * HOMEPAGE: +| * https://github.com/python-hyper/hpack/ +| +| This product contains a modified version of 'HPACK', a Java implementation of +| the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.nghttp2-hpack.txt (MIT License) +| * HOMEPAGE: +| * https://github.com/nghttp2/nghttp2/ +| +| This product contains a modified portion of 'Apache Commons Lang', a Java library +| provides utilities for the java.lang API, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.commons-lang.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://commons.apache.org/proper/commons-lang/ +| +| +| This product contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. +| +| * LICENSE: +| * license/LICENSE.mvn-wrapper.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/takari/maven-wrapper +| +| This product contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. +| This private header is also used by Apple's open source +| mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). +| +| * LICENSE: +| * license/LICENSE.dnsinfo.txt (Apple Public Source License 2.0) +| * HOMEPAGE: +| * https://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h +| +| This product optionally depends on 'Brotli4j', Brotli compression and +| decompression for Java., which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.brotli4j.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/hyperxpro/Brotli4j + +--------------------------------------------------------- + +This product includes gRPC 1.70.0, with the following in its NOTICE: + +| Copyright 2014 The gRPC Authors +| +| Licensed under the Apache License, Version 2.0 (the "License"); +| you may not use this file except in compliance with the License. +| You may obtain a copy of the License at +| +| http://www.apache.org/licenses/LICENSE-2.0 +| +| Unless required by applicable law or agreed to in writing, software +| distributed under the License is distributed on an "AS IS" BASIS, +| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +| See the License for the specific language governing permissions and +| limitations under the License. +| +| ----------------------------------------------------------------------- +| +| This product contains a modified portion of 'OkHttp', an open source +| HTTP & SPDY client for Android and Java applications, which can be obtained +| at: +| +| * LICENSE: +| * okhttp/third_party/okhttp/LICENSE (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/square/okhttp +| * LOCATION_IN_GRPC: +| * okhttp/third_party/okhttp +| +| This product contains a modified portion of 'Envoy', an open source +| cloud-native high-performance edge/middle/service proxy, which can be +| obtained at: +| +| * LICENSE: +| * xds/third_party/envoy/LICENSE (Apache License 2.0) +| * NOTICE: +| * xds/third_party/envoy/NOTICE +| * HOMEPAGE: +| * https://www.envoyproxy.io +| * LOCATION_IN_GRPC: +| * xds/third_party/envoy +| +| This product contains a modified portion of 'protoc-gen-validate (PGV)', +| an open source protoc plugin to generate polyglot message validators, +| which can be obtained at: +| +| * LICENSE: +| * xds/third_party/protoc-gen-validate/LICENSE (Apache License 2.0) +| * NOTICE: +| * xds/third_party/protoc-gen-validate/NOTICE +| * HOMEPAGE: +| * https://github.com/envoyproxy/protoc-gen-validate +| * LOCATION_IN_GRPC: +| * xds/third_party/protoc-gen-validate +| +| This product contains a modified portion of 'udpa', +| an open source universal data plane API, which can be obtained at: +| +| * LICENSE: +| * xds/third_party/udpa/LICENSE (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/cncf/udpa +| * LOCATION_IN_GRPC: +| * xds/third_party/udpa diff --git a/flight/flight-sql-jdbc-driver/pom.xml b/flight/flight-sql-jdbc-driver/pom.xml index f8d829a142..fd08acd359 100644 --- a/flight/flight-sql-jdbc-driver/pom.xml +++ b/flight/flight-sql-jdbc-driver/pom.xml @@ -139,6 +139,14 @@ under the License. + + META-INF/LICENSE.txt + src/shade/LICENSE.txt + + + META-INF/NOTICE.txt + src/shade/NOTICE.txt + @@ -166,6 +174,13 @@ under the License. META-INF/versions/ **/*.proto **/module-info.class + + LICENSE.txt + NOTICE.txt + META-INF/*LICENSE* + META-INF/*NOTICE* + META-INF/license/* + META-INF/licenses/**/* diff --git a/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt b/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt new file mode 100644 index 0000000000..de3efb6960 --- /dev/null +++ b/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt @@ -0,0 +1,367 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------------------------------------- + +This binary artifact contains gRPC 1.70.0. + +Copyright: Copyright 2014 The gRPC Authors +Home page: https://grpc.io/ +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains Netty 4.1.117.Final. + +Copyright: Copyright 2014 The Netty Project +Home page: https://netty.io/ +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains Gson 2.11.0. + +Copyright: Copyright 2008 Google Inc. +Home page: https://github.com/google/gson +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains Guava 33.4.0-jre. + +Home page: https://github.com/google/guava +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains Protobuf 3.25.3. + +Copyright: Copyright 2008 Google Inc. All rights reserved. +Home page: https://protobuf.dev/ +License: https://github.com/protocolbuffers/protobuf/blob/v3.25.3/LICENSE (BSD) +License text: + +| Copyright 2008 Google Inc. All rights reserved. +| +| Redistribution and use in source and binary forms, with or without +| modification, are permitted provided that the following conditions are +| met: +| +| * Redistributions of source code must retain the above copyright +| notice, this list of conditions and the following disclaimer. +| * Redistributions in binary form must reproduce the above +| copyright notice, this list of conditions and the following disclaimer +| in the documentation and/or other materials provided with the +| distribution. +| * Neither the name of Google Inc. nor the names of its +| contributors may be used to endorse or promote products derived from +| this software without specific prior written permission. +| +| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +| "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +| LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +| A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +| OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +| SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +| LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +| DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +| THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +| (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +| OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +| +| Code generated by the Protocol Buffer compiler is owned by the owner +| of the input file used when generating it. This code is not +| standalone and requires a support library to be linked with it. This +| support library is itself covered by the above license. + +-------------------------------------------------------------------------------- + +This binary artifact contains Jackson 2.18.2. + +Home page: https://github.com/FasterXML/jackson +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains Apache Commons Codec 1.18.0. + +Copyright: Copyright 2002-2024 The Apache Software Foundation +Home page: https://commons.apache.org/proper/commons-codec/ +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains Google Flatbuffers 24.3.25. + +Home page: https://flatbuffers.dev/ +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains Slf4j 2.0.16. + +Copyright: Copyright (c) 2004-2022 QOS.ch Sarl (Switzerland) +Home page: http://www.slf4j.org/ +License: MIT +License text: + +| Copyright (c) 2004-2022 QOS.ch Sarl (Switzerland) +| All rights reserved. +| +| Permission is hereby granted, free of charge, to any person obtaining +| a copy of this software and associated documentation files (the +| "Software"), to deal in the Software without restriction, including +| without limitation the rights to use, copy, modify, merge, publish, +| distribute, sublicense, and/or sell copies of the Software, and to +| permit persons to whom the Software is furnished to do so, subject to +| the following conditions: +| +| The above copyright notice and this permission notice shall be +| included in all copies or substantial portions of the Software. +| +| THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +| EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +| MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +| NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +| LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +| OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +| WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +This binary artifact contains Apache Calcite Avatica 1.25.0. + +Copyright: Copyright 2012-2024 The Apache Software Foundation +Home page: https://calcite.apache.org/avatica/ +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains Apache HttpComponents HttpClient 4.5.13. + +Copyright: Copyright 1999-2020 The Apache Software Foundation +Home page: https://hc.apache.org/index.html +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains Bouncycastle 1.79. + +Copyright: Copyright (c) 2000-2024 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org). +Home page: https://www.bouncycastle.org/ +License: MIT +License text: + +| Copyright (c) 2000-2024 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org). +| Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +| associated documentation files (the "Software"), to deal in the Software without restriction, +| including without limitation the rights to use, copy, modify, merge, publish, distribute, +| sub license, and/or sell copies of the Software, and to permit persons to whom the Software is +| furnished to do so, subject to the following conditions: The above copyright notice and this +| permission notice shall be included in all copies or substantial portions of the Software. +| +| **THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +| NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +| NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +| DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +| OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.** diff --git a/flight/flight-sql-jdbc-driver/src/shade/NOTICE.txt b/flight/flight-sql-jdbc-driver/src/shade/NOTICE.txt new file mode 100644 index 0000000000..fcc90b2ed8 --- /dev/null +++ b/flight/flight-sql-jdbc-driver/src/shade/NOTICE.txt @@ -0,0 +1,340 @@ +Apache Arrow Java +Copyright 2016-2025 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +--------------------------------------------------- + +This product includes gRPC 1.70.0, with the following in its NOTICE: + +| Copyright 2014 The gRPC Authors +| +| Licensed under the Apache License, Version 2.0 (the "License"); +| you may not use this file except in compliance with the License. +| You may obtain a copy of the License at +| +| http://www.apache.org/licenses/LICENSE-2.0 +| +| Unless required by applicable law or agreed to in writing, software +| distributed under the License is distributed on an "AS IS" BASIS, +| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +| See the License for the specific language governing permissions and +| limitations under the License. +| +| ----------------------------------------------------------------------- +| +| This product contains a modified portion of 'OkHttp', an open source +| HTTP & SPDY client for Android and Java applications, which can be obtained +| at: +| +| * LICENSE: +| * okhttp/third_party/okhttp/LICENSE (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/square/okhttp +| * LOCATION_IN_GRPC: +| * okhttp/third_party/okhttp +| +| This product contains a modified portion of 'Envoy', an open source +| cloud-native high-performance edge/middle/service proxy, which can be +| obtained at: +| +| * LICENSE: +| * xds/third_party/envoy/LICENSE (Apache License 2.0) +| * NOTICE: +| * xds/third_party/envoy/NOTICE +| * HOMEPAGE: +| * https://www.envoyproxy.io +| * LOCATION_IN_GRPC: +| * xds/third_party/envoy +| +| This product contains a modified portion of 'protoc-gen-validate (PGV)', +| an open source protoc plugin to generate polyglot message validators, +| which can be obtained at: +| +| * LICENSE: +| * xds/third_party/protoc-gen-validate/LICENSE (Apache License 2.0) +| * NOTICE: +| * xds/third_party/protoc-gen-validate/NOTICE +| * HOMEPAGE: +| * https://github.com/envoyproxy/protoc-gen-validate +| * LOCATION_IN_GRPC: +| * xds/third_party/protoc-gen-validate +| +| This product contains a modified portion of 'udpa', +| an open source universal data plane API, which can be obtained at: +| +| * LICENSE: +| * xds/third_party/udpa/LICENSE (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/cncf/udpa +| * LOCATION_IN_GRPC: +| * xds/third_party/udpa + +--------------------------------------------------- + +This product includes Netty 4.1.117.Final, with the following in its NOTICE: + +| The Netty Project +| ================= +| +| Please visit the Netty web site for more information: +| +| * https://netty.io/ +| +| Copyright 2014 The Netty Project +| +| The Netty Project licenses this file to you under the Apache License, +| version 2.0 (the "License"); you may not use this file except in compliance +| with the License. You may obtain a copy of the License at: +| +| https://www.apache.org/licenses/LICENSE-2.0 +| +| Unless required by applicable law or agreed to in writing, software +| distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +| WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +| License for the specific language governing permissions and limitations +| under the License. +| +| Also, please refer to each LICENSE..txt file, which is located in +| the 'license' directory of the distribution file, for the license terms of the +| components that this product depends on. +| +| ------------------------------------------------------------------------------- +| This product contains the extensions to Java Collections Framework which has +| been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: +| +| * LICENSE: +| * license/LICENSE.jsr166y.txt (Public Domain) +| * HOMEPAGE: +| * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ +| * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ +| +| This product contains a modified version of Robert Harder's Public Domain +| Base64 Encoder and Decoder, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.base64.txt (Public Domain) +| * HOMEPAGE: +| * http://iharder.sourceforge.net/current/java/base64/ +| +| This product contains a modified portion of 'Webbit', an event based +| WebSocket and HTTP server, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.webbit.txt (BSD License) +| * HOMEPAGE: +| * https://github.com/joewalnes/webbit +| +| This product contains a modified portion of 'SLF4J', a simple logging +| facade for Java, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.slf4j.txt (MIT License) +| * HOMEPAGE: +| * https://www.slf4j.org/ +| +| This product contains a modified portion of 'Apache Harmony', an open source +| Java SE, which can be obtained at: +| +| * NOTICE: +| * license/NOTICE.harmony.txt +| * LICENSE: +| * license/LICENSE.harmony.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://archive.apache.org/dist/harmony/ +| +| This product contains a modified portion of 'jbzip2', a Java bzip2 compression +| and decompression library written by Matthew J. Francis. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.jbzip2.txt (MIT License) +| * HOMEPAGE: +| * https://code.google.com/p/jbzip2/ +| +| This product contains a modified portion of 'libdivsufsort', a C API library to construct +| the suffix array and the Burrows-Wheeler transformed string for any input string of +| a constant-size alphabet written by Yuta Mori. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.libdivsufsort.txt (MIT License) +| * HOMEPAGE: +| * https://github.com/y-256/libdivsufsort +| +| This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, +| which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.jctools.txt (ASL2 License) +| * HOMEPAGE: +| * https://github.com/JCTools/JCTools +| +| This product optionally depends on 'JZlib', a re-implementation of zlib in +| pure Java, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.jzlib.txt (BSD style License) +| * HOMEPAGE: +| * http://www.jcraft.com/jzlib/ +| +| This product optionally depends on 'Compress-LZF', a Java library for encoding and +| decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.compress-lzf.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/ning/compress +| +| This product optionally depends on 'lz4', a LZ4 Java compression +| and decompression library written by Adrien Grand. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.lz4.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/jpountz/lz4-java +| +| This product optionally depends on 'lzma-java', a LZMA Java compression +| and decompression library, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.lzma-java.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/jponge/lzma-java +| +| This product optionally depends on 'zstd-jni', a zstd-jni Java compression +| and decompression library, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.zstd-jni.txt (BSD) +| * HOMEPAGE: +| * https://github.com/luben/zstd-jni +| +| This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression +| and decompression library written by William Kinney. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.jfastlz.txt (MIT License) +| * HOMEPAGE: +| * https://code.google.com/p/jfastlz/ +| +| This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data +| interchange format, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.protobuf.txt (New BSD License) +| * HOMEPAGE: +| * https://github.com/google/protobuf +| +| This product optionally depends on 'Bouncy Castle Crypto APIs' to generate +| a temporary self-signed X.509 certificate when the JVM does not provide the +| equivalent functionality. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.bouncycastle.txt (MIT License) +| * HOMEPAGE: +| * https://www.bouncycastle.org/ +| +| This product optionally depends on 'Snappy', a compression library produced +| by Google Inc, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.snappy.txt (New BSD License) +| * HOMEPAGE: +| * https://github.com/google/snappy +| +| This product optionally depends on 'JBoss Marshalling', an alternative Java +| serialization API, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.jboss-marshalling.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/jboss-remoting/jboss-marshalling +| +| This product optionally depends on 'Caliper', Google's micro- +| benchmarking framework, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.caliper.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/google/caliper +| +| This product optionally depends on 'Apache Commons Logging', a logging +| framework, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.commons-logging.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://commons.apache.org/logging/ +| +| This product optionally depends on 'Apache Log4J', a logging framework, which +| can be obtained at: +| +| * LICENSE: +| * license/LICENSE.log4j.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://logging.apache.org/log4j/ +| +| This product optionally depends on 'Aalto XML', an ultra-high performance +| non-blocking XML processor, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.aalto-xml.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://wiki.fasterxml.com/AaltoHome +| +| This product contains a modified version of 'HPACK', a Java implementation of +| the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.hpack.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/twitter/hpack +| +| This product contains a modified version of 'HPACK', a Java implementation of +| the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.hyper-hpack.txt (MIT License) +| * HOMEPAGE: +| * https://github.com/python-hyper/hpack/ +| +| This product contains a modified version of 'HPACK', a Java implementation of +| the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.nghttp2-hpack.txt (MIT License) +| * HOMEPAGE: +| * https://github.com/nghttp2/nghttp2/ +| +| This product contains a modified portion of 'Apache Commons Lang', a Java library +| provides utilities for the java.lang API, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.commons-lang.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://commons.apache.org/proper/commons-lang/ +| +| +| This product contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. +| +| * LICENSE: +| * license/LICENSE.mvn-wrapper.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/takari/maven-wrapper +| +| This product contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. +| This private header is also used by Apple's open source +| mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). +| +| * LICENSE: +| * license/LICENSE.dnsinfo.txt (Apple Public Source License 2.0) +| * HOMEPAGE: +| * https://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h +| +| This product optionally depends on 'Brotli4j', Brotli compression and +| decompression for Java., which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.brotli4j.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/hyperxpro/Brotli4j diff --git a/flight/flight-sql-jdbc-driver/src/test/java/org/apache/arrow/driver/jdbc/ITDriverJarValidation.java b/flight/flight-sql-jdbc-driver/src/test/java/org/apache/arrow/driver/jdbc/ITDriverJarValidation.java index a0e108d6a0..c1bd111fb9 100644 --- a/flight/flight-sql-jdbc-driver/src/test/java/org/apache/arrow/driver/jdbc/ITDriverJarValidation.java +++ b/flight/flight-sql-jdbc-driver/src/test/java/org/apache/arrow/driver/jdbc/ITDriverJarValidation.java @@ -67,6 +67,8 @@ public class ITDriverJarValidation { /** List of allowed files a jar entry may match. */ public static final Set ALLOWED_FILES = ImmutableSet.of( + "LICENSE.txt", + "NOTICE.txt", "arrow-git.properties", "properties/flight.properties", "META-INF/io.netty.versions.properties", diff --git a/tools/pom.xml b/tools/pom.xml index a64b84857c..73edd218b1 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -96,6 +96,7 @@ under the License. + org.apache.maven.plugins maven-shade-plugin @@ -114,9 +115,26 @@ under the License. **/module-info.class + + *:* + + LICENSE.txt + NOTICE.txt + META-INF/*LICENSE* + META-INF/*NOTICE* + + + + META-INF/LICENSE.txt + src/shade/LICENSE.txt + + + META-INF/NOTICE.txt + src/shade/NOTICE.txt + diff --git a/tools/src/shade/LICENSE.txt b/tools/src/shade/LICENSE.txt new file mode 100644 index 0000000000..93d32757c6 --- /dev/null +++ b/tools/src/shade/LICENSE.txt @@ -0,0 +1,331 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------------------------------------- + +This binary artifact contains Netty 4.1.117.Final. + +Copyright: Copyright 2014 The Netty Project +Home page: https://netty.io/ +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains Jackson 2.18.2. + +Home page: https://github.com/FasterXML/jackson +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains Apache Commons Codec 1.18.0. + +Copyright: Copyright 2002-2024 The Apache Software Foundation +Home page: https://commons.apache.org/proper/commons-codec/ +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains Apache Commons Compress 1.27.1. + +Copyright: Copyright 2002-2024 The Apache Software Foundation +Home page: https://commons.apache.org/proper/commons-compress/ +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains Apache Commons IO 2.16.1. + +Copyright: Copyright 2002-2024 The Apache Software Foundation +Home page: https://commons.apache.org/proper/commons-io/ +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains Apache Commons Lang 3.16.0. + +Copyright: Copyright 2001-2024 The Apache Software Foundation +Home page: https://commons.apache.org/proper/commons-lang/ +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains Luben Zstd 1.5.6-9. + +Copyright: Copyright (c) 2015-present, Luben Karavelov/ All rights reserved. +Home page: https://github.com/luben/zstd-jni +License: BSD License +License text: + +| Zstd-jni: JNI bindings to Zstd Library +| +| Copyright (c) 2015-present, Luben Karavelov/ All rights reserved. +| +| BSD License +| +| Redistribution and use in source and binary forms, with or without modification, +| are permitted provided that the following conditions are met: +| +| * Redistributions of source code must retain the above copyright notice, this +| list of conditions and the following disclaimer. +| +| * Redistributions in binary form must reproduce the above copyright notice, this +| list of conditions and the following disclaimer in the documentation and/or +| other materials provided with the distribution. +| +| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +| ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +| WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +| DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +| ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +| (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +| LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +| ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +| (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +| SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This binary artifact contains Apache Commons CLI 1.9.0. + +Copyright: Copyright 2002-2024 The Apache Software Foundation +Home page: https://commons.apache.org/proper/commons-cli/ +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains Google Flatbuffers 24.3.25. + +Home page: https://flatbuffers.dev/ +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary artifact contains Slf4j 2.0.16. + +Copyright: Copyright (c) 2004-2022 QOS.ch Sarl (Switzerland) +Home page: http://www.slf4j.org/ +License: MIT +License text: + +| Copyright (c) 2004-2022 QOS.ch Sarl (Switzerland) +| All rights reserved. +| +| Permission is hereby granted, free of charge, to any person obtaining +| a copy of this software and associated documentation files (the +| "Software"), to deal in the Software without restriction, including +| without limitation the rights to use, copy, modify, merge, publish, +| distribute, sublicense, and/or sell copies of the Software, and to +| permit persons to whom the Software is furnished to do so, subject to +| the following conditions: +| +| The above copyright notice and this permission notice shall be +| included in all copies or substantial portions of the Software. +| +| THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +| EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +| MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +| NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +| LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +| OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +| WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/tools/src/shade/NOTICE.txt b/tools/src/shade/NOTICE.txt new file mode 100644 index 0000000000..7c5b36d714 --- /dev/null +++ b/tools/src/shade/NOTICE.txt @@ -0,0 +1,273 @@ +Apache Arrow Java +Copyright 2016-2025 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +--------------------------------------------------- + +This product includes Netty 4.1.117.Final, with the following in its NOTICE: + +| The Netty Project +| ================= +| +| Please visit the Netty web site for more information: +| +| * https://netty.io/ +| +| Copyright 2014 The Netty Project +| +| The Netty Project licenses this file to you under the Apache License, +| version 2.0 (the "License"); you may not use this file except in compliance +| with the License. You may obtain a copy of the License at: +| +| https://www.apache.org/licenses/LICENSE-2.0 +| +| Unless required by applicable law or agreed to in writing, software +| distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +| WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +| License for the specific language governing permissions and limitations +| under the License. +| +| Also, please refer to each LICENSE..txt file, which is located in +| the 'license' directory of the distribution file, for the license terms of the +| components that this product depends on. +| +| ------------------------------------------------------------------------------- +| This product contains the extensions to Java Collections Framework which has +| been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: +| +| * LICENSE: +| * license/LICENSE.jsr166y.txt (Public Domain) +| * HOMEPAGE: +| * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ +| * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ +| +| This product contains a modified version of Robert Harder's Public Domain +| Base64 Encoder and Decoder, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.base64.txt (Public Domain) +| * HOMEPAGE: +| * http://iharder.sourceforge.net/current/java/base64/ +| +| This product contains a modified portion of 'Webbit', an event based +| WebSocket and HTTP server, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.webbit.txt (BSD License) +| * HOMEPAGE: +| * https://github.com/joewalnes/webbit +| +| This product contains a modified portion of 'SLF4J', a simple logging +| facade for Java, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.slf4j.txt (MIT License) +| * HOMEPAGE: +| * https://www.slf4j.org/ +| +| This product contains a modified portion of 'Apache Harmony', an open source +| Java SE, which can be obtained at: +| +| * NOTICE: +| * license/NOTICE.harmony.txt +| * LICENSE: +| * license/LICENSE.harmony.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://archive.apache.org/dist/harmony/ +| +| This product contains a modified portion of 'jbzip2', a Java bzip2 compression +| and decompression library written by Matthew J. Francis. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.jbzip2.txt (MIT License) +| * HOMEPAGE: +| * https://code.google.com/p/jbzip2/ +| +| This product contains a modified portion of 'libdivsufsort', a C API library to construct +| the suffix array and the Burrows-Wheeler transformed string for any input string of +| a constant-size alphabet written by Yuta Mori. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.libdivsufsort.txt (MIT License) +| * HOMEPAGE: +| * https://github.com/y-256/libdivsufsort +| +| This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, +| which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.jctools.txt (ASL2 License) +| * HOMEPAGE: +| * https://github.com/JCTools/JCTools +| +| This product optionally depends on 'JZlib', a re-implementation of zlib in +| pure Java, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.jzlib.txt (BSD style License) +| * HOMEPAGE: +| * http://www.jcraft.com/jzlib/ +| +| This product optionally depends on 'Compress-LZF', a Java library for encoding and +| decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.compress-lzf.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/ning/compress +| +| This product optionally depends on 'lz4', a LZ4 Java compression +| and decompression library written by Adrien Grand. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.lz4.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/jpountz/lz4-java +| +| This product optionally depends on 'lzma-java', a LZMA Java compression +| and decompression library, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.lzma-java.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/jponge/lzma-java +| +| This product optionally depends on 'zstd-jni', a zstd-jni Java compression +| and decompression library, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.zstd-jni.txt (BSD) +| * HOMEPAGE: +| * https://github.com/luben/zstd-jni +| +| This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression +| and decompression library written by William Kinney. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.jfastlz.txt (MIT License) +| * HOMEPAGE: +| * https://code.google.com/p/jfastlz/ +| +| This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data +| interchange format, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.protobuf.txt (New BSD License) +| * HOMEPAGE: +| * https://github.com/google/protobuf +| +| This product optionally depends on 'Bouncy Castle Crypto APIs' to generate +| a temporary self-signed X.509 certificate when the JVM does not provide the +| equivalent functionality. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.bouncycastle.txt (MIT License) +| * HOMEPAGE: +| * https://www.bouncycastle.org/ +| +| This product optionally depends on 'Snappy', a compression library produced +| by Google Inc, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.snappy.txt (New BSD License) +| * HOMEPAGE: +| * https://github.com/google/snappy +| +| This product optionally depends on 'JBoss Marshalling', an alternative Java +| serialization API, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.jboss-marshalling.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/jboss-remoting/jboss-marshalling +| +| This product optionally depends on 'Caliper', Google's micro- +| benchmarking framework, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.caliper.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/google/caliper +| +| This product optionally depends on 'Apache Commons Logging', a logging +| framework, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.commons-logging.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://commons.apache.org/logging/ +| +| This product optionally depends on 'Apache Log4J', a logging framework, which +| can be obtained at: +| +| * LICENSE: +| * license/LICENSE.log4j.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://logging.apache.org/log4j/ +| +| This product optionally depends on 'Aalto XML', an ultra-high performance +| non-blocking XML processor, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.aalto-xml.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://wiki.fasterxml.com/AaltoHome +| +| This product contains a modified version of 'HPACK', a Java implementation of +| the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.hpack.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/twitter/hpack +| +| This product contains a modified version of 'HPACK', a Java implementation of +| the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.hyper-hpack.txt (MIT License) +| * HOMEPAGE: +| * https://github.com/python-hyper/hpack/ +| +| This product contains a modified version of 'HPACK', a Java implementation of +| the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: +| +| * LICENSE: +| * license/LICENSE.nghttp2-hpack.txt (MIT License) +| * HOMEPAGE: +| * https://github.com/nghttp2/nghttp2/ +| +| This product contains a modified portion of 'Apache Commons Lang', a Java library +| provides utilities for the java.lang API, which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.commons-lang.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://commons.apache.org/proper/commons-lang/ +| +| +| This product contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. +| +| * LICENSE: +| * license/LICENSE.mvn-wrapper.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/takari/maven-wrapper +| +| This product contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. +| This private header is also used by Apple's open source +| mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). +| +| * LICENSE: +| * license/LICENSE.dnsinfo.txt (Apple Public Source License 2.0) +| * HOMEPAGE: +| * https://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h +| +| This product optionally depends on 'Brotli4j', Brotli compression and +| decompression for Java., which can be obtained at: +| +| * LICENSE: +| * license/LICENSE.brotli4j.txt (Apache License 2.0) +| * HOMEPAGE: +| * https://github.com/hyperxpro/Brotli4j diff --git a/vector/pom.xml b/vector/pom.xml index 94fde96e14..503bfaeac5 100644 --- a/vector/pom.xml +++ b/vector/pom.xml @@ -171,6 +171,25 @@ under the License. arrow.vector.com.google.flatbuffers + + + *:* + + META-INF/LICENSE + META-INF/NOTICE + + + + + + META-INF/LICENSE.txt + src/shade/LICENSE.txt + + + META-INF/NOTICE.txt + src/shade/NOTICE.txt + + diff --git a/vector/src/shade/LICENSE.txt b/vector/src/shade/LICENSE.txt new file mode 100644 index 0000000000..2598ec941f --- /dev/null +++ b/vector/src/shade/LICENSE.txt @@ -0,0 +1,209 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------------------------------------- + +This binary artifact contains Google Flatbuffers 24.3.25. + +Home page: https://flatbuffers.dev/ +License: https://www.apache.org/licenses/LICENSE-2.0 diff --git a/vector/src/shade/NOTICE.txt b/vector/src/shade/NOTICE.txt new file mode 100644 index 0000000000..0d96a64614 --- /dev/null +++ b/vector/src/shade/NOTICE.txt @@ -0,0 +1,5 @@ +Apache Arrow Java +Copyright 2016-2025 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). From b933d2ebebca11b4e22fe78b73f16173fc550a40 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 6 Feb 2025 09:55:42 +0900 Subject: [PATCH 071/408] GH-588: [Release] Increase wait interval in `dev/release/release_rc.sh` (#589) Fixes GH-588. `release.yml` takes at least 30min now. So 60s wait interval is too short. We can use longer interval to avoid needless API requests. --- dev/release/release_rc.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/release/release_rc.sh b/dev/release/release_rc.sh index 1375376463..b0107dce85 100755 --- a/dev/release/release_rc.sh +++ b/dev/release/release_rc.sh @@ -109,13 +109,13 @@ if [ "${RELEASE_SIGN}" -gt 0 ]; then if [ -n "${run_id}" ]; then break fi - sleep 60 + sleep 600 done echo "Found GitHub Actions workflow with ID: ${run_id}" gh run watch \ --exit-status "${run_id}" \ - --interval 60 \ + --interval 600 \ --repo "${repository}" echo "Downloading artifacts from GitHub Releases" From 4c3dcb4087083e7daeaf5d7bc3a61d44729635ac Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 5 Feb 2025 22:50:11 -0500 Subject: [PATCH 072/408] MINOR: Upload release artifacts individually with sleep (#590) According to https://github.com/cli/cli/issues/9586, GitHub CLI doesn't respect the GitHub rate limits and GitHub is uninterested in fixing this. --- .github/workflows/rc.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 9717f30a9c..67248a408a 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -600,7 +600,12 @@ jobs: --prerelease \ --repo ${GITHUB_REPOSITORY} \ --title "Apache Arrow Java ${version} RC${rc}" \ - --verify-tag \ - artifacts/*/* + --verify-tag + # GitHub CLI does not respect their own rate limits + # https://github.com/cli/cli/issues/9586 + for artifact in artifacts/*/*; do + sleep 1 + gh release upload ${GITHUB_REF_NAME} $artifact + done env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From a5b86049261530d41fbbdd23324420f0c1d14431 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 6 Feb 2025 15:01:58 +0900 Subject: [PATCH 073/408] MINOR: Specify --repo explicitly (#591) We don't checkout apache/arrow-java in this job. So we need to specify --repo explicitly. --- .github/workflows/rc.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 67248a408a..faad59832b 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -605,7 +605,9 @@ jobs: # https://github.com/cli/cli/issues/9586 for artifact in artifacts/*/*; do sleep 1 - gh release upload ${GITHUB_REF_NAME} $artifact + gh release upload ${GITHUB_REF_NAME} \ + --repo ${GITHUB_REPOSITORY} \ + $artifact done env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 05b13f247088c6393582347b79e673ce03fcb3d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Wed, 12 Feb 2025 06:26:01 +0100 Subject: [PATCH 074/408] MINOR: Upgrade to Google flatbuffers 25.1.24 (#600) --- README.md | 4 ++-- flight/flight-integration-tests/src/shade/LICENSE.txt | 2 +- flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Binary.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/BinaryView.java | 2 +- .../main/java/org/apache/arrow/flatbuf/BodyCompression.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Bool.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Date.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Decimal.java | 2 +- .../main/java/org/apache/arrow/flatbuf/DictionaryBatch.java | 2 +- .../java/org/apache/arrow/flatbuf/DictionaryEncoding.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Duration.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Field.java | 2 +- .../main/java/org/apache/arrow/flatbuf/FixedSizeBinary.java | 2 +- .../src/main/java/org/apache/arrow/flatbuf/FixedSizeList.java | 2 +- .../src/main/java/org/apache/arrow/flatbuf/FloatingPoint.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Footer.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Int.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Interval.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/KeyValue.java | 2 +- .../src/main/java/org/apache/arrow/flatbuf/LargeBinary.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/LargeList.java | 2 +- .../src/main/java/org/apache/arrow/flatbuf/LargeListView.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/LargeUtf8.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/List.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/ListView.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Map.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Message.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Null.java | 2 +- .../src/main/java/org/apache/arrow/flatbuf/RecordBatch.java | 2 +- .../src/main/java/org/apache/arrow/flatbuf/RunEndEncoded.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Schema.java | 2 +- .../java/org/apache/arrow/flatbuf/SparseMatrixIndexCSX.java | 2 +- .../src/main/java/org/apache/arrow/flatbuf/SparseTensor.java | 2 +- .../java/org/apache/arrow/flatbuf/SparseTensorIndexCOO.java | 2 +- .../java/org/apache/arrow/flatbuf/SparseTensorIndexCSF.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Struct_.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Tensor.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/TensorDim.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Time.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Timestamp.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Union.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Utf8.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Utf8View.java | 2 +- pom.xml | 2 +- tools/src/shade/LICENSE.txt | 2 +- vector/src/shade/LICENSE.txt | 2 +- 47 files changed, 48 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 9f1b1c63c8..c46b61c49e 100644 --- a/README.md +++ b/README.md @@ -48,10 +48,10 @@ a version of your choosing. ```bash $ flatc --version -flatc version 24.3.25 +flatc version 25.1.24 $ grep "dep.fbs.version" java/pom.xml - 24.3.25 + 25.1.24 ``` 2. Generate the flatbuffer java files by performing the following: diff --git a/flight/flight-integration-tests/src/shade/LICENSE.txt b/flight/flight-integration-tests/src/shade/LICENSE.txt index 0c411ce9ba..499a3993fd 100644 --- a/flight/flight-integration-tests/src/shade/LICENSE.txt +++ b/flight/flight-integration-tests/src/shade/LICENSE.txt @@ -218,7 +218,7 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- -This binary artifact contains Google Flatbuffers 24.3.25. +This binary artifact contains Google Flatbuffers 25.1.24. Home page: https://flatbuffers.dev/ License: https://www.apache.org/licenses/LICENSE-2.0 diff --git a/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt b/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt index de3efb6960..3363121bb9 100644 --- a/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt +++ b/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt @@ -291,7 +291,7 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- -This binary artifact contains Google Flatbuffers 24.3.25. +This binary artifact contains Google Flatbuffers 25.1.24. Home page: https://flatbuffers.dev/ License: https://www.apache.org/licenses/LICENSE-2.0 diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Binary.java b/format/src/main/java/org/apache/arrow/flatbuf/Binary.java index 65b3f4f577..2c90e97754 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Binary.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Binary.java @@ -38,7 +38,7 @@ */ @SuppressWarnings("unused") public final class Binary extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static Binary getRootAsBinary(ByteBuffer _bb) { return getRootAsBinary(_bb, new Binary()); } public static Binary getRootAsBinary(ByteBuffer _bb, Binary obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/BinaryView.java b/format/src/main/java/org/apache/arrow/flatbuf/BinaryView.java index f20245ca28..3b5212f60b 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/BinaryView.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/BinaryView.java @@ -44,7 +44,7 @@ */ @SuppressWarnings("unused") public final class BinaryView extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static BinaryView getRootAsBinaryView(ByteBuffer _bb) { return getRootAsBinaryView(_bb, new BinaryView()); } public static BinaryView getRootAsBinaryView(ByteBuffer _bb, BinaryView obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/BodyCompression.java b/format/src/main/java/org/apache/arrow/flatbuf/BodyCompression.java index a7e4288467..e94c108f35 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/BodyCompression.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/BodyCompression.java @@ -40,7 +40,7 @@ */ @SuppressWarnings("unused") public final class BodyCompression extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static BodyCompression getRootAsBodyCompression(ByteBuffer _bb) { return getRootAsBodyCompression(_bb, new BodyCompression()); } public static BodyCompression getRootAsBodyCompression(ByteBuffer _bb, BodyCompression obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Bool.java b/format/src/main/java/org/apache/arrow/flatbuf/Bool.java index 7d03705ec8..173af2eaa2 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Bool.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Bool.java @@ -35,7 +35,7 @@ @SuppressWarnings("unused") public final class Bool extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static Bool getRootAsBool(ByteBuffer _bb) { return getRootAsBool(_bb, new Bool()); } public static Bool getRootAsBool(ByteBuffer _bb, Bool obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Date.java b/format/src/main/java/org/apache/arrow/flatbuf/Date.java index b9af04b120..413419535e 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Date.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Date.java @@ -43,7 +43,7 @@ */ @SuppressWarnings("unused") public final class Date extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static Date getRootAsDate(ByteBuffer _bb) { return getRootAsDate(_bb, new Date()); } public static Date getRootAsDate(ByteBuffer _bb, Date obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Decimal.java b/format/src/main/java/org/apache/arrow/flatbuf/Decimal.java index e2f38558d4..0cc2cbda26 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Decimal.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Decimal.java @@ -41,7 +41,7 @@ */ @SuppressWarnings("unused") public final class Decimal extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static Decimal getRootAsDecimal(ByteBuffer _bb) { return getRootAsDecimal(_bb, new Decimal()); } public static Decimal getRootAsDecimal(ByteBuffer _bb, Decimal obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/DictionaryBatch.java b/format/src/main/java/org/apache/arrow/flatbuf/DictionaryBatch.java index 891df5d164..52cc347093 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/DictionaryBatch.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/DictionaryBatch.java @@ -43,7 +43,7 @@ */ @SuppressWarnings("unused") public final class DictionaryBatch extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static DictionaryBatch getRootAsDictionaryBatch(ByteBuffer _bb) { return getRootAsDictionaryBatch(_bb, new DictionaryBatch()); } public static DictionaryBatch getRootAsDictionaryBatch(ByteBuffer _bb, DictionaryBatch obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/DictionaryEncoding.java b/format/src/main/java/org/apache/arrow/flatbuf/DictionaryEncoding.java index 28c52d579e..3d797f7028 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/DictionaryEncoding.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/DictionaryEncoding.java @@ -35,7 +35,7 @@ @SuppressWarnings("unused") public final class DictionaryEncoding extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static DictionaryEncoding getRootAsDictionaryEncoding(ByteBuffer _bb) { return getRootAsDictionaryEncoding(_bb, new DictionaryEncoding()); } public static DictionaryEncoding getRootAsDictionaryEncoding(ByteBuffer _bb, DictionaryEncoding obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Duration.java b/format/src/main/java/org/apache/arrow/flatbuf/Duration.java index 442310dbfd..a77c7b77cd 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Duration.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Duration.java @@ -35,7 +35,7 @@ @SuppressWarnings("unused") public final class Duration extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static Duration getRootAsDuration(ByteBuffer _bb) { return getRootAsDuration(_bb, new Duration()); } public static Duration getRootAsDuration(ByteBuffer _bb, Duration obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Field.java b/format/src/main/java/org/apache/arrow/flatbuf/Field.java index 9cd50d0b98..7ea73e8b6a 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Field.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Field.java @@ -40,7 +40,7 @@ */ @SuppressWarnings("unused") public final class Field extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static Field getRootAsField(ByteBuffer _bb) { return getRootAsField(_bb, new Field()); } public static Field getRootAsField(ByteBuffer _bb, Field obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/FixedSizeBinary.java b/format/src/main/java/org/apache/arrow/flatbuf/FixedSizeBinary.java index 7c4c08567f..54bf8395e1 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/FixedSizeBinary.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/FixedSizeBinary.java @@ -35,7 +35,7 @@ @SuppressWarnings("unused") public final class FixedSizeBinary extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static FixedSizeBinary getRootAsFixedSizeBinary(ByteBuffer _bb) { return getRootAsFixedSizeBinary(_bb, new FixedSizeBinary()); } public static FixedSizeBinary getRootAsFixedSizeBinary(ByteBuffer _bb, FixedSizeBinary obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/FixedSizeList.java b/format/src/main/java/org/apache/arrow/flatbuf/FixedSizeList.java index 44eceafee6..e3264eff30 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/FixedSizeList.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/FixedSizeList.java @@ -35,7 +35,7 @@ @SuppressWarnings("unused") public final class FixedSizeList extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static FixedSizeList getRootAsFixedSizeList(ByteBuffer _bb) { return getRootAsFixedSizeList(_bb, new FixedSizeList()); } public static FixedSizeList getRootAsFixedSizeList(ByteBuffer _bb, FixedSizeList obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/FloatingPoint.java b/format/src/main/java/org/apache/arrow/flatbuf/FloatingPoint.java index 59741cde59..8bf8839649 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/FloatingPoint.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/FloatingPoint.java @@ -35,7 +35,7 @@ @SuppressWarnings("unused") public final class FloatingPoint extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static FloatingPoint getRootAsFloatingPoint(ByteBuffer _bb) { return getRootAsFloatingPoint(_bb, new FloatingPoint()); } public static FloatingPoint getRootAsFloatingPoint(ByteBuffer _bb, FloatingPoint obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Footer.java b/format/src/main/java/org/apache/arrow/flatbuf/Footer.java index 156215b202..b5a4bbe449 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Footer.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Footer.java @@ -40,7 +40,7 @@ */ @SuppressWarnings("unused") public final class Footer extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static Footer getRootAsFooter(ByteBuffer _bb) { return getRootAsFooter(_bb, new Footer()); } public static Footer getRootAsFooter(ByteBuffer _bb, Footer obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Int.java b/format/src/main/java/org/apache/arrow/flatbuf/Int.java index 9dd1d448a4..b307770e32 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Int.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Int.java @@ -35,7 +35,7 @@ @SuppressWarnings("unused") public final class Int extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static Int getRootAsInt(ByteBuffer _bb) { return getRootAsInt(_bb, new Int()); } public static Int getRootAsInt(ByteBuffer _bb, Int obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Interval.java b/format/src/main/java/org/apache/arrow/flatbuf/Interval.java index 7b2be74c5c..c91de4210f 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Interval.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Interval.java @@ -35,7 +35,7 @@ @SuppressWarnings("unused") public final class Interval extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static Interval getRootAsInterval(ByteBuffer _bb) { return getRootAsInterval(_bb, new Interval()); } public static Interval getRootAsInterval(ByteBuffer _bb, Interval obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/KeyValue.java b/format/src/main/java/org/apache/arrow/flatbuf/KeyValue.java index 8f242ef3a9..05b26faace 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/KeyValue.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/KeyValue.java @@ -40,7 +40,7 @@ */ @SuppressWarnings("unused") public final class KeyValue extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static KeyValue getRootAsKeyValue(ByteBuffer _bb) { return getRootAsKeyValue(_bb, new KeyValue()); } public static KeyValue getRootAsKeyValue(ByteBuffer _bb, KeyValue obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/LargeBinary.java b/format/src/main/java/org/apache/arrow/flatbuf/LargeBinary.java index 72135992a0..60d7cf33f8 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/LargeBinary.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/LargeBinary.java @@ -39,7 +39,7 @@ */ @SuppressWarnings("unused") public final class LargeBinary extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static LargeBinary getRootAsLargeBinary(ByteBuffer _bb) { return getRootAsLargeBinary(_bb, new LargeBinary()); } public static LargeBinary getRootAsLargeBinary(ByteBuffer _bb, LargeBinary obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/LargeList.java b/format/src/main/java/org/apache/arrow/flatbuf/LargeList.java index 1cce67712d..9ef8f7b287 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/LargeList.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/LargeList.java @@ -39,7 +39,7 @@ */ @SuppressWarnings("unused") public final class LargeList extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static LargeList getRootAsLargeList(ByteBuffer _bb) { return getRootAsLargeList(_bb, new LargeList()); } public static LargeList getRootAsLargeList(ByteBuffer _bb, LargeList obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/LargeListView.java b/format/src/main/java/org/apache/arrow/flatbuf/LargeListView.java index 234ce35f68..558c8e31d3 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/LargeListView.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/LargeListView.java @@ -39,7 +39,7 @@ */ @SuppressWarnings("unused") public final class LargeListView extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static LargeListView getRootAsLargeListView(ByteBuffer _bb) { return getRootAsLargeListView(_bb, new LargeListView()); } public static LargeListView getRootAsLargeListView(ByteBuffer _bb, LargeListView obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/LargeUtf8.java b/format/src/main/java/org/apache/arrow/flatbuf/LargeUtf8.java index 377bfbb2cf..6451124c4f 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/LargeUtf8.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/LargeUtf8.java @@ -39,7 +39,7 @@ */ @SuppressWarnings("unused") public final class LargeUtf8 extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static LargeUtf8 getRootAsLargeUtf8(ByteBuffer _bb) { return getRootAsLargeUtf8(_bb, new LargeUtf8()); } public static LargeUtf8 getRootAsLargeUtf8(ByteBuffer _bb, LargeUtf8 obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/List.java b/format/src/main/java/org/apache/arrow/flatbuf/List.java index 7f06dda072..af88b8db91 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/List.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/List.java @@ -35,7 +35,7 @@ @SuppressWarnings("unused") public final class List extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static List getRootAsList(ByteBuffer _bb) { return getRootAsList(_bb, new List()); } public static List getRootAsList(ByteBuffer _bb, List obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/ListView.java b/format/src/main/java/org/apache/arrow/flatbuf/ListView.java index 5d87df5e4c..6a0c1f4883 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/ListView.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/ListView.java @@ -40,7 +40,7 @@ */ @SuppressWarnings("unused") public final class ListView extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static ListView getRootAsListView(ByteBuffer _bb) { return getRootAsListView(_bb, new ListView()); } public static ListView getRootAsListView(ByteBuffer _bb, ListView obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Map.java b/format/src/main/java/org/apache/arrow/flatbuf/Map.java index 10652c0544..096ae530f3 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Map.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Map.java @@ -62,7 +62,7 @@ */ @SuppressWarnings("unused") public final class Map extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static Map getRootAsMap(ByteBuffer _bb) { return getRootAsMap(_bb, new Map()); } public static Map getRootAsMap(ByteBuffer _bb, Map obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Message.java b/format/src/main/java/org/apache/arrow/flatbuf/Message.java index f2518f9046..c5eb9321ba 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Message.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Message.java @@ -35,7 +35,7 @@ @SuppressWarnings("unused") public final class Message extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static Message getRootAsMessage(ByteBuffer _bb) { return getRootAsMessage(_bb, new Message()); } public static Message getRootAsMessage(ByteBuffer _bb, Message obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Null.java b/format/src/main/java/org/apache/arrow/flatbuf/Null.java index 60196193ea..2ef526d040 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Null.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Null.java @@ -38,7 +38,7 @@ */ @SuppressWarnings("unused") public final class Null extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static Null getRootAsNull(ByteBuffer _bb) { return getRootAsNull(_bb, new Null()); } public static Null getRootAsNull(ByteBuffer _bb, Null obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/RecordBatch.java b/format/src/main/java/org/apache/arrow/flatbuf/RecordBatch.java index 83deda7480..f3004cca23 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/RecordBatch.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/RecordBatch.java @@ -40,7 +40,7 @@ */ @SuppressWarnings("unused") public final class RecordBatch extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static RecordBatch getRootAsRecordBatch(ByteBuffer _bb) { return getRootAsRecordBatch(_bb, new RecordBatch()); } public static RecordBatch getRootAsRecordBatch(ByteBuffer _bb, RecordBatch obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/RunEndEncoded.java b/format/src/main/java/org/apache/arrow/flatbuf/RunEndEncoded.java index 070ad9499e..87a968f197 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/RunEndEncoded.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/RunEndEncoded.java @@ -42,7 +42,7 @@ */ @SuppressWarnings("unused") public final class RunEndEncoded extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static RunEndEncoded getRootAsRunEndEncoded(ByteBuffer _bb) { return getRootAsRunEndEncoded(_bb, new RunEndEncoded()); } public static RunEndEncoded getRootAsRunEndEncoded(ByteBuffer _bb, RunEndEncoded obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Schema.java b/format/src/main/java/org/apache/arrow/flatbuf/Schema.java index 373071f4bb..0b2d1880e5 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Schema.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Schema.java @@ -39,7 +39,7 @@ */ @SuppressWarnings("unused") public final class Schema extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static Schema getRootAsSchema(ByteBuffer _bb) { return getRootAsSchema(_bb, new Schema()); } public static Schema getRootAsSchema(ByteBuffer _bb, Schema obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/SparseMatrixIndexCSX.java b/format/src/main/java/org/apache/arrow/flatbuf/SparseMatrixIndexCSX.java index e5c1dc8a6f..ad8d6ffad7 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/SparseMatrixIndexCSX.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/SparseMatrixIndexCSX.java @@ -38,7 +38,7 @@ */ @SuppressWarnings("unused") public final class SparseMatrixIndexCSX extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static SparseMatrixIndexCSX getRootAsSparseMatrixIndexCSX(ByteBuffer _bb) { return getRootAsSparseMatrixIndexCSX(_bb, new SparseMatrixIndexCSX()); } public static SparseMatrixIndexCSX getRootAsSparseMatrixIndexCSX(ByteBuffer _bb, SparseMatrixIndexCSX obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/SparseTensor.java b/format/src/main/java/org/apache/arrow/flatbuf/SparseTensor.java index 6ca7f29f12..fb75d872a1 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/SparseTensor.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/SparseTensor.java @@ -35,7 +35,7 @@ @SuppressWarnings("unused") public final class SparseTensor extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static SparseTensor getRootAsSparseTensor(ByteBuffer _bb) { return getRootAsSparseTensor(_bb, new SparseTensor()); } public static SparseTensor getRootAsSparseTensor(ByteBuffer _bb, SparseTensor obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/SparseTensorIndexCOO.java b/format/src/main/java/org/apache/arrow/flatbuf/SparseTensorIndexCOO.java index b43915c6af..204d6fa2b3 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/SparseTensorIndexCOO.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/SparseTensorIndexCOO.java @@ -69,7 +69,7 @@ */ @SuppressWarnings("unused") public final class SparseTensorIndexCOO extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static SparseTensorIndexCOO getRootAsSparseTensorIndexCOO(ByteBuffer _bb) { return getRootAsSparseTensorIndexCOO(_bb, new SparseTensorIndexCOO()); } public static SparseTensorIndexCOO getRootAsSparseTensorIndexCOO(ByteBuffer _bb, SparseTensorIndexCOO obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/SparseTensorIndexCSF.java b/format/src/main/java/org/apache/arrow/flatbuf/SparseTensorIndexCSF.java index a32e29a6e8..a67eddd074 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/SparseTensorIndexCSF.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/SparseTensorIndexCSF.java @@ -38,7 +38,7 @@ */ @SuppressWarnings("unused") public final class SparseTensorIndexCSF extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static SparseTensorIndexCSF getRootAsSparseTensorIndexCSF(ByteBuffer _bb) { return getRootAsSparseTensorIndexCSF(_bb, new SparseTensorIndexCSF()); } public static SparseTensorIndexCSF getRootAsSparseTensorIndexCSF(ByteBuffer _bb, SparseTensorIndexCSF obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Struct_.java b/format/src/main/java/org/apache/arrow/flatbuf/Struct_.java index 9065d30067..d1758b7014 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Struct_.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Struct_.java @@ -40,7 +40,7 @@ */ @SuppressWarnings("unused") public final class Struct_ extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static Struct_ getRootAsStruct_(ByteBuffer _bb) { return getRootAsStruct_(_bb, new Struct_()); } public static Struct_ getRootAsStruct_(ByteBuffer _bb, Struct_ obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Tensor.java b/format/src/main/java/org/apache/arrow/flatbuf/Tensor.java index 7bba1e3455..7fe869cfe5 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Tensor.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Tensor.java @@ -35,7 +35,7 @@ @SuppressWarnings("unused") public final class Tensor extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static Tensor getRootAsTensor(ByteBuffer _bb) { return getRootAsTensor(_bb, new Tensor()); } public static Tensor getRootAsTensor(ByteBuffer _bb, Tensor obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/TensorDim.java b/format/src/main/java/org/apache/arrow/flatbuf/TensorDim.java index 566ec2e3a2..c7d215811b 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/TensorDim.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/TensorDim.java @@ -40,7 +40,7 @@ */ @SuppressWarnings("unused") public final class TensorDim extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static TensorDim getRootAsTensorDim(ByteBuffer _bb) { return getRootAsTensorDim(_bb, new TensorDim()); } public static TensorDim getRootAsTensorDim(ByteBuffer _bb, TensorDim obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Time.java b/format/src/main/java/org/apache/arrow/flatbuf/Time.java index 0c635ea103..4e8e24d3ab 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Time.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Time.java @@ -51,7 +51,7 @@ */ @SuppressWarnings("unused") public final class Time extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static Time getRootAsTime(ByteBuffer _bb) { return getRootAsTime(_bb, new Time()); } public static Time getRootAsTime(ByteBuffer _bb, Time obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Timestamp.java b/format/src/main/java/org/apache/arrow/flatbuf/Timestamp.java index 2f33b2ee5e..587c81527f 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Timestamp.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Timestamp.java @@ -142,7 +142,7 @@ */ @SuppressWarnings("unused") public final class Timestamp extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static Timestamp getRootAsTimestamp(ByteBuffer _bb) { return getRootAsTimestamp(_bb, new Timestamp()); } public static Timestamp getRootAsTimestamp(ByteBuffer _bb, Timestamp obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Union.java b/format/src/main/java/org/apache/arrow/flatbuf/Union.java index 9c3c6b8e3e..6ea670edc4 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Union.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Union.java @@ -41,7 +41,7 @@ */ @SuppressWarnings("unused") public final class Union extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static Union getRootAsUnion(ByteBuffer _bb) { return getRootAsUnion(_bb, new Union()); } public static Union getRootAsUnion(ByteBuffer _bb, Union obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Utf8.java b/format/src/main/java/org/apache/arrow/flatbuf/Utf8.java index 6aa4baa7b7..0297f39553 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Utf8.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Utf8.java @@ -38,7 +38,7 @@ */ @SuppressWarnings("unused") public final class Utf8 extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static Utf8 getRootAsUtf8(ByteBuffer _bb) { return getRootAsUtf8(_bb, new Utf8()); } public static Utf8 getRootAsUtf8(ByteBuffer _bb, Utf8 obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Utf8View.java b/format/src/main/java/org/apache/arrow/flatbuf/Utf8View.java index 1da9ef27f3..b8f9eddd94 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Utf8View.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Utf8View.java @@ -44,7 +44,7 @@ */ @SuppressWarnings("unused") public final class Utf8View extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } public static Utf8View getRootAsUtf8View(ByteBuffer _bb) { return getRootAsUtf8View(_bb, new Utf8View()); } public static Utf8View getRootAsUtf8View(ByteBuffer _bb, Utf8View obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/pom.xml b/pom.xml index 6e82083b74..f32edf6997 100644 --- a/pom.xml +++ b/pom.xml @@ -101,7 +101,7 @@ under the License. 3.25.5 2.18.2 3.4.1 - 24.3.25 + 25.1.24 1.12.0 2 diff --git a/tools/src/shade/LICENSE.txt b/tools/src/shade/LICENSE.txt index 93d32757c6..61fbdf256f 100644 --- a/tools/src/shade/LICENSE.txt +++ b/tools/src/shade/LICENSE.txt @@ -294,7 +294,7 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- -This binary artifact contains Google Flatbuffers 24.3.25. +This binary artifact contains Google Flatbuffers 25.1.24. Home page: https://flatbuffers.dev/ License: https://www.apache.org/licenses/LICENSE-2.0 diff --git a/vector/src/shade/LICENSE.txt b/vector/src/shade/LICENSE.txt index 2598ec941f..d14211db2b 100644 --- a/vector/src/shade/LICENSE.txt +++ b/vector/src/shade/LICENSE.txt @@ -203,7 +203,7 @@ -------------------------------------------------------------------------------- -This binary artifact contains Google Flatbuffers 24.3.25. +This binary artifact contains Google Flatbuffers 25.1.24. Home page: https://flatbuffers.dev/ License: https://www.apache.org/licenses/LICENSE-2.0 From a6212a7ded258763b0daafa5abf0dbb04cbf5860 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 12 Feb 2025 14:42:10 +0900 Subject: [PATCH 075/408] GH-603: [Release] Suppress a nested tag warning in `dev/release/release.sh` (#606) Fixes GH-603. Message from Git: hint: You have created a nested tag. The object referred to by your new tag is hint: already a tag. If you meant to tag the object that it points to, use: hint: hint: git tag -f v18.2.0 v18.2.0-rc5^{} hint: Disable this message with "git config advice.nestedTag false" --- dev/release/release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/release.sh b/dev/release/release.sh index c0c7d3328a..3a1b46f0ce 100755 --- a/dev/release/release.sh +++ b/dev/release/release.sh @@ -45,7 +45,7 @@ esac tag="v${version}" rc_tag="${tag}-rc${rc}" echo "Tagging for release: ${tag}" -git tag -a -m "${version}" "${tag}" "${rc_tag}" +git tag -a -m "${version}" "${tag}" "${rc_tag}^{}" git push origin "${tag}" release_id="apache-arrow-java-${version}" From d2fd2832e16badffed37884ffa4a5dcc1098ced7 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 12 Feb 2025 14:42:37 +0900 Subject: [PATCH 076/408] GH-605: [Release] Show the repository.apache.org step (#608) Fixes GH-605. If `dev/release/release.sh` shows what should we do at repository.apache.org, it's helpful. --- dev/release/release.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dev/release/release.sh b/dev/release/release.sh index 3a1b46f0ce..180f9907bd 100755 --- a/dev/release/release.sh +++ b/dev/release/release.sh @@ -87,3 +87,8 @@ echo " https://dist.apache.org/repos/dist/release/arrow/${release_id}" echo echo "Add this release to ASF's report database:" echo " https://reporter.apache.org/addrelease.html?arrow" +echo +echo "Release binary artifacts in repository.apache.org:" +echo "1. Open https://repository.apache.org/#stagingRepositories" +echo "2. Select the repository for RC: orgapachearrow-XXXX" +echo "3. Click the Release button" From 94ca5a049287ded559e39b1b9750537e96f846cc Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 12 Feb 2025 14:42:56 +0900 Subject: [PATCH 077/408] GH-604: [Release] Upload only source archive to dist.apache.org (#607) Fixes GH-604. For example, here are target files for 18.2.0: * apache-arrow-java-18.2.0.tar.gz * apache-arrow-java-18.2.0.tar.gz.asc * apache-arrow-java-18.2.0.tar.gz.sha256 * apache-arrow-java-18.2.0.tar.gz.sha512 --- dev/release/release.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev/release/release.sh b/dev/release/release.sh index 180f9907bd..c5e67907e2 100755 --- a/dev/release/release.sh +++ b/dev/release/release.sh @@ -49,6 +49,7 @@ git tag -a -m "${version}" "${tag}" "${rc_tag}^{}" git push origin "${tag}" release_id="apache-arrow-java-${version}" +source_archive="apache-arrow-java-${version}.tar.gz" dist_url="https://dist.apache.org/repos/dist/release/arrow" dist_base_dir="dev/release/dist" dist_dir="${dist_base_dir}/${release_id}" @@ -56,8 +57,9 @@ echo "Checking out ${dist_url}" rm -rf "${dist_base_dir}" svn co --depth=empty "${dist_url}" "${dist_base_dir}" gh release download "${rc_tag}" \ - --repo "${repository}" \ --dir "${dist_dir}" \ + --pattern "${source_archive}*" \ + --repo "${repository}" \ --skip-existing echo "Uploading to release/" From 34e2b08ff1b06c864f63f3ad1a3a771d0e7b058b Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 12 Feb 2025 19:33:09 -0500 Subject: [PATCH 078/408] GH-609: [Release] Wait between artifact uploads (#610) The GitHub CLI does not respect GitHub API rate limits and will not fix this (https://github.com/cli/cli/issues/9586). Closes #609. --- .github/workflows/release.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e801fd0493..a955964cd8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -54,8 +54,16 @@ jobs: --generate-notes \ --repo ${GITHUB_REPOSITORY} \ --title "Apache Arrow Java ${version}" \ - --verify-tag \ - dists/* + --verify-tag + + # GitHub CLI does not respect their own rate limits + # https://github.com/cli/cli/issues/9586 + for artifact in dists/*; do + sleep 1 + gh release upload ${GITHUB_REF_NAME} \ + --repo ${GITHUB_REPOSITORY} \ + $artifact + done - name: Checkout for publishing docs uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: From cb52f737a840cec6fe5ddb866f4f7e0a30329e51 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 13 Feb 2025 14:23:02 +0900 Subject: [PATCH 079/408] GH-611: [Docs] Detect version from pom.xml (#613) Fixes GH-611. --- docs/source/conf.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 166a3bc4ea..3b17f326e1 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -23,10 +23,17 @@ # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information +import pathlib +import xml.etree.ElementTree as ET + project = 'arrow-java' copyright = '2025, Apache Arrow Developers' author = 'Apache Arrow Developers' -release = '18.1.0' + +top_level_pom_xml = pathlib.Path(__file__).parents[2] / "pom.xml" +tree = ET.parse(top_level_pom_xml) +ns = {"maven": "http://maven.apache.org/POM/4.0.0"} +release = tree.getroot().find("maven:version", ns).text # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration From b47e499bbc4ef414f16fb950ad546a625bb00262 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Fri, 14 Feb 2025 02:54:56 +0100 Subject: [PATCH 080/408] MINOR: Bump version to 19.0.0-SNAPSHOT (#614) --- adapter/avro/pom.xml | 2 +- adapter/jdbc/pom.xml | 2 +- adapter/orc/pom.xml | 2 +- algorithm/pom.xml | 2 +- bom/pom.xml | 4 ++-- c/pom.xml | 2 +- compression/pom.xml | 2 +- dataset/pom.xml | 2 +- flight/flight-core/pom.xml | 2 +- flight/flight-integration-tests/pom.xml | 2 +- flight/flight-sql-jdbc-core/pom.xml | 2 +- flight/flight-sql-jdbc-driver/pom.xml | 2 +- flight/flight-sql/pom.xml | 2 +- flight/pom.xml | 2 +- format/pom.xml | 2 +- gandiva/pom.xml | 2 +- memory/memory-core/pom.xml | 2 +- memory/memory-netty-buffer-patch/pom.xml | 2 +- memory/memory-netty/pom.xml | 2 +- memory/memory-unsafe/pom.xml | 2 +- memory/pom.xml | 2 +- performance/pom.xml | 2 +- pom.xml | 4 ++-- tools/pom.xml | 2 +- vector/pom.xml | 2 +- 25 files changed, 27 insertions(+), 27 deletions(-) diff --git a/adapter/avro/pom.xml b/adapter/avro/pom.xml index 2eb77d7539..827d19f2a2 100644 --- a/adapter/avro/pom.xml +++ b/adapter/avro/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 18.2.0 + 19.0.0-SNAPSHOT ../../pom.xml diff --git a/adapter/jdbc/pom.xml b/adapter/jdbc/pom.xml index 57a728124b..2f621d7a05 100644 --- a/adapter/jdbc/pom.xml +++ b/adapter/jdbc/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 18.2.0 + 19.0.0-SNAPSHOT ../../pom.xml diff --git a/adapter/orc/pom.xml b/adapter/orc/pom.xml index b8ff0aa409..8b45dd72c9 100644 --- a/adapter/orc/pom.xml +++ b/adapter/orc/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 18.2.0 + 19.0.0-SNAPSHOT ../../pom.xml diff --git a/algorithm/pom.xml b/algorithm/pom.xml index 3bcbfa5062..898c2605b6 100644 --- a/algorithm/pom.xml +++ b/algorithm/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.2.0 + 19.0.0-SNAPSHOT arrow-algorithm Arrow Algorithms diff --git a/bom/pom.xml b/bom/pom.xml index 192b494eb2..61441048ff 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -29,7 +29,7 @@ under the License. org.apache.arrow arrow-bom - 18.2.0 + 19.0.0-SNAPSHOT pom Arrow Bill of Materials @@ -68,7 +68,7 @@ under the License. scm:git:https://github.com/apache/arrow-java.git scm:git:https://github.com/apache/arrow-java.git - v18.2.0 + main https://github.com/apache/arrow-java/tree/${project.scm.tag} diff --git a/c/pom.xml b/c/pom.xml index 48e9fafdf5..c90b6dc0ef 100644 --- a/c/pom.xml +++ b/c/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.2.0 + 19.0.0-SNAPSHOT arrow-c-data diff --git a/compression/pom.xml b/compression/pom.xml index 6ef36fe8ab..3c339d557a 100644 --- a/compression/pom.xml +++ b/compression/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.2.0 + 19.0.0-SNAPSHOT arrow-compression Arrow Compression diff --git a/dataset/pom.xml b/dataset/pom.xml index ad21c7754a..4ff931d307 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.2.0 + 19.0.0-SNAPSHOT arrow-dataset diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml index 3960c412b6..7484349730 100644 --- a/flight/flight-core/pom.xml +++ b/flight/flight-core/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 18.2.0 + 19.0.0-SNAPSHOT flight-core diff --git a/flight/flight-integration-tests/pom.xml b/flight/flight-integration-tests/pom.xml index b1e06d2f07..78a2d08ee1 100644 --- a/flight/flight-integration-tests/pom.xml +++ b/flight/flight-integration-tests/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 18.2.0 + 19.0.0-SNAPSHOT flight-integration-tests diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index ab3560459b..204e17efe8 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 18.2.0 + 19.0.0-SNAPSHOT flight-sql-jdbc-core diff --git a/flight/flight-sql-jdbc-driver/pom.xml b/flight/flight-sql-jdbc-driver/pom.xml index fd08acd359..559c42597d 100644 --- a/flight/flight-sql-jdbc-driver/pom.xml +++ b/flight/flight-sql-jdbc-driver/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 18.2.0 + 19.0.0-SNAPSHOT flight-sql-jdbc-driver diff --git a/flight/flight-sql/pom.xml b/flight/flight-sql/pom.xml index 4b5c5560c9..5d8e720fd6 100644 --- a/flight/flight-sql/pom.xml +++ b/flight/flight-sql/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 18.2.0 + 19.0.0-SNAPSHOT flight-sql diff --git a/flight/pom.xml b/flight/pom.xml index 35dfe90468..2fc3e89ef8 100644 --- a/flight/pom.xml +++ b/flight/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.2.0 + 19.0.0-SNAPSHOT arrow-flight diff --git a/format/pom.xml b/format/pom.xml index 292f23bcd2..d3578b63d2 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 18.2.0 + 19.0.0-SNAPSHOT arrow-format diff --git a/gandiva/pom.xml b/gandiva/pom.xml index 384dcb8019..5367bfdedf 100644 --- a/gandiva/pom.xml +++ b/gandiva/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.2.0 + 19.0.0-SNAPSHOT org.apache.arrow.gandiva diff --git a/memory/memory-core/pom.xml b/memory/memory-core/pom.xml index 9432dd2461..72ee69d60a 100644 --- a/memory/memory-core/pom.xml +++ b/memory/memory-core/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 18.2.0 + 19.0.0-SNAPSHOT arrow-memory-core diff --git a/memory/memory-netty-buffer-patch/pom.xml b/memory/memory-netty-buffer-patch/pom.xml index 696fadeba3..07dc7d2403 100644 --- a/memory/memory-netty-buffer-patch/pom.xml +++ b/memory/memory-netty-buffer-patch/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 18.2.0 + 19.0.0-SNAPSHOT arrow-memory-netty-buffer-patch diff --git a/memory/memory-netty/pom.xml b/memory/memory-netty/pom.xml index b81e13cd90..6d660da117 100644 --- a/memory/memory-netty/pom.xml +++ b/memory/memory-netty/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 18.2.0 + 19.0.0-SNAPSHOT arrow-memory-netty diff --git a/memory/memory-unsafe/pom.xml b/memory/memory-unsafe/pom.xml index a476da71b1..92dc0c9fe5 100644 --- a/memory/memory-unsafe/pom.xml +++ b/memory/memory-unsafe/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 18.2.0 + 19.0.0-SNAPSHOT arrow-memory-unsafe diff --git a/memory/pom.xml b/memory/pom.xml index af6babe841..bc34c26050 100644 --- a/memory/pom.xml +++ b/memory/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.2.0 + 19.0.0-SNAPSHOT arrow-memory pom diff --git a/performance/pom.xml b/performance/pom.xml index 507222706d..3f18188e3a 100644 --- a/performance/pom.xml +++ b/performance/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.2.0 + 19.0.0-SNAPSHOT arrow-performance jar diff --git a/pom.xml b/pom.xml index f32edf6997..eca403363c 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ under the License. org.apache.arrow arrow-java-root - 18.2.0 + 19.0.0-SNAPSHOT pom Apache Arrow Java Root POM @@ -81,7 +81,7 @@ under the License. scm:git:https://github.com/apache/arrow-java.git scm:git:https://github.com/apache/arrow-java.git - v18.2.0 + main https://github.com/apache/arrow-java/tree/${project.scm.tag} diff --git a/tools/pom.xml b/tools/pom.xml index 73edd218b1..cb9a161308 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.2.0 + 19.0.0-SNAPSHOT arrow-tools Arrow Tools diff --git a/vector/pom.xml b/vector/pom.xml index 503bfaeac5..52ad5105ea 100644 --- a/vector/pom.xml +++ b/vector/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.2.0 + 19.0.0-SNAPSHOT arrow-vector Arrow Vectors From f1fd621f3ad5726677a8aac1b8a7189432372865 Mon Sep 17 00:00:00 2001 From: Benchao Li Date: Tue, 18 Feb 2025 10:10:08 +0800 Subject: [PATCH 081/408] MINOR: ZstdCompressionCodec should use decompressedSize to get error name (#619) --- .../java/org/apache/arrow/compression/ZstdCompressionCodec.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compression/src/main/java/org/apache/arrow/compression/ZstdCompressionCodec.java b/compression/src/main/java/org/apache/arrow/compression/ZstdCompressionCodec.java index 6e48aae71f..46b1e223a4 100644 --- a/compression/src/main/java/org/apache/arrow/compression/ZstdCompressionCodec.java +++ b/compression/src/main/java/org/apache/arrow/compression/ZstdCompressionCodec.java @@ -68,7 +68,7 @@ protected ArrowBuf doDecompress(BufferAllocator allocator, ArrowBuf compressedBu compressedBuffer.writerIndex() - CompressionUtil.SIZE_OF_UNCOMPRESSED_LENGTH); if (Zstd.isError(decompressedSize)) { uncompressedBuffer.close(); - throw new RuntimeException("Error decompressing: " + Zstd.getErrorName(decompressedLength)); + throw new RuntimeException("Error decompressing: " + Zstd.getErrorName(decompressedSize)); } if (decompressedLength != decompressedSize) { uncompressedBuffer.close(); From 52cf1f28dbf58c215e5b56887ef0245d74edd056 Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 18 Feb 2025 03:20:36 -0500 Subject: [PATCH 082/408] GH-499: Require PR labels for changelog (#594) - Validate that at least one of certain labels are present on a PR. - Validate the format of the title. - Add label for things that are not dependencies but are other chores (release/build work). - Add label for documentation updates. - Validate that a PR titled `GH-499` is actually linked to issue `#499` by poking the GitHub API. - Add the GitHub config to use these labels for changelog generation. Closes #499. --------- Co-authored-by: Sutou Kouhei --- .github/pull_request_template.md | 7 + .github/release.yml | 34 +++++ .github/workflows/dev_pr.js | 241 ++++++++++++++++++++++++++++++ .github/workflows/dev_pr.yml | 77 ++++++++++ CONTRIBUTING.md | 32 +++- dev/release/rat_exclude_files.txt | 1 + 6 files changed, 390 insertions(+), 2 deletions(-) create mode 100644 .github/pull_request_template.md create mode 100644 .github/release.yml create mode 100644 .github/workflows/dev_pr.js create mode 100644 .github/workflows/dev_pr.yml diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000000..2ae17814a1 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,7 @@ +## What's Changed + +Please fill in a description of the changes here. + +**This contains breaking changes.** + +Closes #NNN. diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 0000000000..f4a1bc9da3 --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +changelog: + categories: + - title: Breaking Changes + labels: + - breaking-change + + - title: New Features and Enhancements + labels: + - enhancement + + - title: Bug Fixes + labels: + - bug-fix + + - title: Other Changes + labels: + - "*" diff --git a/.github/workflows/dev_pr.js b/.github/workflows/dev_pr.js new file mode 100644 index 0000000000..19fd6e5695 --- /dev/null +++ b/.github/workflows/dev_pr.js @@ -0,0 +1,241 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +async function have_comment(github, context, pr_number, tag) { + console.log(`Looking for existing comment on ${pr_number} with substring ${tag}`); + const query = ` +query($owner: String!, $name: String!, $number: Int!, $cursor: String) { + repository(owner: $owner, name: $name) { + pullRequest(number: $number) { + id + comments (after:$cursor, first: 50) { + nodes { + id + body + author { + login + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + } +}`; + const tag_regexp = new RegExp(tag); + + let cursor = null; + let pr_id = null; + while (true) { + const result = await github.graphql(query, { + owner: context.repo.owner, + name: context.repo.repo, + number: pr_number, + cursor, + }); + pr_id = result.repository.pullRequest.id; + cursor = result.repository.pullRequest.comments.pageInfo; + const comments = result.repository.pullRequest.comments.nodes; + + for (const comment of comments) { + console.log(comment); + if (comment.author.login === "github-actions" && + comment.body.match(tag_regexp) !== null) { + console.log("Found existing comment"); + return {pr_id, comment_id: comment.id}; + } + } + + if (!result.repository.pullRequest.comments.hasNextPage || + comments.length === 0) { + break; + } + } + console.log("No existing comment"); + return {pr_id, comment_id: null}; +} + +async function upsert_comment(github, {pr_id, comment_id}, body, visible) { + console.log(`Upsert comment (pr_id=${pr_id}, comment_id=${comment_id}, visible=${visible})`); + if (!visible) { + if (comment_id === null) return; + + const query = ` +mutation makeComment($comment: ID!) { + minimizeComment(input: {subjectId: $comment, classifier: RESOLVED}) { + clientMutationId + } +}`; + await github.graphql(query, { + comment: comment_id, + body, + }); + return; + } + + if (comment_id === null) { + const query = ` +mutation makeComment($pr: ID!, $body: String!) { + addComment(input: {subjectId: $pr, body: $body}) { + clientMutationId + } +}`; + await github.graphql(query, { + pr: pr_id, + body, + }); + } else { + const query = ` +mutation makeComment($comment: ID!, $body: String!) { + unminimizeComment(input: {subjectId: $comment}) { + clientMutationId + } + updateIssueComment(input: {id: $comment, body: $body}) { + clientMutationId + } +}`; + await github.graphql(query, { + comment: comment_id, + body, + }); + } +} + +module.exports = { + check_title_format: function({core, github, context}) { + const title = context.payload.pull_request.title; + if (title.startsWith("MINOR: ")) { + context.log("PR is a minor PR"); + return {"issue": null}; + } + + const match = title.match(/^GH-([0-9]+): .*$/); + if (match === null) { + core.setFailed("Invalid PR title format. Must either be MINOR: or GH-NNN:"); + return {"issue": null}; + } + return {"issue": parseInt(match[1], 10)}; + }, + + apply_labels: async function({core, github, context}) { + const body = (context.payload.pull_request.body || "").split(/\n/g); + var has_breaking = false; + for (const line of body) { + if (line.trim().startsWith("**This contains breaking changes.**")) { + has_breaking = true; + break; + } + } + if (has_breaking) { + console.log("PR has breaking changes"); + await github.rest.issues.addLabels({ + issue_number: context.payload.pull_request.number, + owner: context.repo.owner, + repo: context.repo.repo, + labels: ["breaking-change"], + }); + } else { + console.log("PR has no breaking changes"); + } + }, + + check_labels: async function({core, github, context}) { + const categories = ["bug-fix", "chore", "dependencies", "documentation", "enhancement"]; + const labels = (context.payload.pull_request.labels || []); + const required = new Set(categories); + var found = false; + + for (const label of labels) { + console.log(`Found label ${label.name}`); + if (required.has(label.name)) { + found = true; + break; + } + } + + // Look to see if we left a comment before. + const comment_tag = "label_helper_comment"; + const maybe_comment_id = await have_comment(github, context, context.payload.pull_request.number, comment_tag); + console.log("Found comment?"); + console.log(maybe_comment_id); + const body_text = ` + +Thank you for opening a pull request! + +Please label the PR with one or more of: + +${categories.map(c => `- ${c}`).join("\n")} + +Also, add the 'breaking-change' label if appropriate. + +See [CONTRIBUTING.md](https://github.com/apache/arrow-java/blob/main/CONTRIBUTING.md) for details. +`; + + if (found) { + console.log("PR has appropriate label(s)"); + await upsert_comment(github, maybe_comment_id, body_text, false); + } else { + console.log(body_text); + await upsert_comment(github, maybe_comment_id, body_text, true); + core.setFailed("Missing required labels. See CONTRIBUTING.md"); + } + }, + + check_linked_issue: async function({core, github, context, issue}) { + console.log(issue); + if (issue.issue === null) { + console.log("This is a MINOR PR"); + return; + } + const expected = `https://github.com/apache/arrow-java/issues/${issue.issue}`; + + const query = ` +query($owner: String!, $name: String!, $number: Int!) { + repository(owner: $owner, name: $name) { + pullRequest(number: $number) { + closingIssuesReferences(first: 50) { + edges { + node { + number + } + } + } + } + } +}`; + + const result = await github.graphql(query, { + owner: context.repo.owner, + name: context.repo.repo, + number: context.payload.pull_request.number, + }); + const issues = result.repository.pullRequest.closingIssuesReferences.edges; + console.log(issues); + + for (const link of issues) { + console.log(`PR is linked to ${link.node.number}`); + if (link.node.number === issue.issue) { + console.log(`Found link to ${expected}`); + return; + } + } + console.log(`Did not find link to ${expected}`); + core.setFailed("Missing link to issue in title"); + }, +}; diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml new file mode 100644 index 0000000000..8da3ab5fd5 --- /dev/null +++ b/.github/workflows/dev_pr.yml @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Dev PR + +on: + pull_request_target: + types: + - labeled + - unlabeled + - opened + - edited + - reopened + - synchronize + - ready_for_review + - review_requested + +concurrency: + group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write + +jobs: + pr-label: + name: "Ensure PR format" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Ensure PR title format + id: title-format + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const scripts = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr.js`); + return scripts.check_title_format({core, github, context}); + + - name: Label PR + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const scripts = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr.js`); + await scripts.apply_labels({core, github, context}); + + - name: Ensure PR is labeled + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const scripts = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr.js`); + await scripts.check_labels({core, github, context}); + + - name: Ensure PR is linked to an issue + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const scripts = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr.js`); + await scripts.check_linked_issue({core, github, context, issue: ${{ steps.title-format.outputs.result }}}); diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8388b1d6c7..680750070f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,8 +30,36 @@ existing Arrow issues in [GitHub](https://github.com/apache/arrow-java/issues). ## Did you write a patch that fixes a bug or brings an improvement? -Create a GitHub issue and submit your changes as a GitHub Pull Request. -Please make sure to [reference the issue](https://docs.github.com/en/issues/tracking-your-work-with-issues/using-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword) in your PR description. +- Create a GitHub issue and submit your changes as a GitHub Pull Request. +- [Reference the issue](https://docs.github.com/en/issues/tracking-your-work-with-issues/using-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword) in your PR description. +- Add one or more of the labels "bug-fix", "chore", "dependencies", "documentation", and "enhancement" to your PR as appropriate. + - "bug-fix" is for PRs that fix a bug. + - "chore" is for other administrative work (build system, release process, etc.). + - "dependencies" is for PRs that upgrade a dependency. (Usually only used by dependabot.) + - "documentation" is for documentation updates. + - "enhancement" is for PRs that add new features. +- Add the "breaking-change" label to your PR if there are breaking API changes. +- Add the PR title. The PR title will be used as the eventual commit message, so please make it descriptive but succinct. + +Example #1: + +``` +GH-12345: Document the pull request process + +Explain how to open a pull request and what the title, body, and labels should be. + +Closes #12345. +``` + +Example #2: + +``` +GH-42424: Expose Netty server builder in Flight + +Allow direct usage of gRPC APIs for low-level control. + +Closes #42424. +``` ### Minor Fixes diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 8324d32cf6..0999f1a275 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -16,5 +16,6 @@ # under the License. .gitmodules +.github/pull_request_template.md dataset/src/test/resources/data/student.csv docs/Makefile From e1dd6d87b27e2e52939c0cefae6ca378dbfe45dd Mon Sep 17 00:00:00 2001 From: lriggs Date: Wed, 19 Feb 2025 15:24:32 -0800 Subject: [PATCH 083/408] GH-55: [Gandiva] Re-enable tests (#595) Fixes GH-55. ### Rationale for this change I fixed the linking error here: https://github.com/apache/arrow/pull/45114/ ### What changes are included in this PR? Reenabling some disabled tests. ### Are these changes tested? Yes, they are tests. ### Are there any user-facing changes? No. --- .env | 2 +- .github/workflows/rc.yml | 32 +++++++++---------- compose.yaml | 4 +-- .../gandiva/evaluator/FilterProjectTest.java | 2 -- .../arrow/gandiva/evaluator/FilterTest.java | 8 ----- .../evaluator/ProjectorDecimalTest.java | 11 ------- .../gandiva/evaluator/ProjectorTest.java | 1 - 7 files changed, 19 insertions(+), 41 deletions(-) diff --git a/.env b/.env index 5398249b24..d3e1c1d63a 100644 --- a/.env +++ b/.env @@ -54,4 +54,4 @@ MAVEN=3.9.9 # Keep in sync with apache/arrow ARROW_REPO_ROOT=./arrow PYTHON=3.9 -VCPKG="943c5ef1c8f6b5e6ced092b242c8299caae2ff01" # 2024.04.26 Release +VCPKG="f7423ee180c4b7f40d43402c2feb3859161ef625" # 2024.06.15 Release diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index faad59832b..4f240e6714 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -107,12 +107,12 @@ jobs: - name: Extract source archive run: | tar -xf apache-arrow-java-*.tar.gz --strip-components=1 - - name: Download the latest Apache Arrow C++ - if: github.event_name != 'schedule' - run: | - ci/scripts/download_cpp.sh + # - name: Download the latest Apache Arrow C++ + # if: github.event_name != 'schedule' + # run: | + # ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ - if: github.event_name == 'schedule' + # if: github.event_name == 'schedule' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: apache/arrow @@ -174,12 +174,12 @@ jobs: - name: Extract source archive run: | tar -xf apache-arrow-java-*.tar.gz --strip-components=1 - - name: Download the latest Apache Arrow C++ - if: github.event_name != 'schedule' - run: | - ci/scripts/download_cpp.sh + # - name: Download the latest Apache Arrow C++ + # if: github.event_name != 'schedule' + # run: | + # ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ - if: github.event_name == 'schedule' + # if: github.event_name == 'schedule' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: apache/arrow @@ -299,13 +299,13 @@ jobs: shell: bash run: | tar -xf apache-arrow-java-*.tar.gz --strip-components=1 - - name: Download the latest Apache Arrow C++ - if: github.event_name != 'schedule' - shell: bash - run: | - ci/scripts/download_cpp.sh + # - name: Download the latest Apache Arrow C++ + # if: github.event_name != 'schedule' + # shell: bash + # run: | + # ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ - if: github.event_name == 'schedule' + # if: github.event_name == 'schedule' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: apache/arrow diff --git a/compose.yaml b/compose.yaml index 58a19676a1..b125c3c983 100644 --- a/compose.yaml +++ b/compose.yaml @@ -92,12 +92,12 @@ services: # Usage: # docker compose build vcpkg-jni # docker compose run vcpkg-jni - image: ${REPO}:${ARCH}-vcpkg-jni + image: ${REPO}:${ARCH}-vcpkg-jni-${VCPKG} build: context: . dockerfile: ci/docker/vcpkg-jni.dockerfile cache_from: - - ${REPO}:${ARCH}-vcpkg-jni + - ${REPO}:${ARCH}-vcpkg-jni-${VCPKG} args: base: ${ARROW_REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2014-vcpkg-${VCPKG} volumes: diff --git a/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/FilterProjectTest.java b/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/FilterProjectTest.java index 75169a37a9..80427de0f0 100644 --- a/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/FilterProjectTest.java +++ b/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/FilterProjectTest.java @@ -34,13 +34,11 @@ import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class FilterProjectTest extends BaseEvaluatorTest { @Test - @Disabled("GH-43576 - Fix and enable this test") public void testSimpleSV16() throws GandivaException, Exception { Field a = Field.nullable("a", int32); Field b = Field.nullable("b", int32); diff --git a/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/FilterTest.java b/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/FilterTest.java index a98a7cb6b5..7563465f37 100644 --- a/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/FilterTest.java +++ b/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/FilterTest.java @@ -34,7 +34,6 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class FilterTest extends BaseEvaluatorTest { @@ -73,7 +72,6 @@ List stringBufs(String[] strings) { } @Test - @Disabled("GH-43576 - Fix and enable this test") public void testSimpleInString() throws GandivaException, Exception { Field c1 = Field.nullable("c1", new ArrowType.Utf8()); TreeNode l1 = TreeBuilder.makeLiteral(1L); @@ -137,7 +135,6 @@ public void testSimpleInString() throws GandivaException, Exception { } @Test - @Disabled("GH-43576 - Fix and enable this test") public void testSimpleInInt() throws GandivaException, Exception { Field c1 = Field.nullable("c1", int32); @@ -181,7 +178,6 @@ public void testSimpleInInt() throws GandivaException, Exception { } @Test - @Disabled("GH-43576 - Fix and enable this test") public void testSimpleSV16() throws GandivaException, Exception { Field a = Field.nullable("a", int32); Field b = Field.nullable("b", int32); @@ -203,7 +199,6 @@ public void testSimpleSV16() throws GandivaException, Exception { } @Test - @Disabled("GH-43576 - Fix and enable this test") public void testSimpleSV16_AllMatched() throws GandivaException, Exception { Field a = Field.nullable("a", int32); Field b = Field.nullable("b", int32); @@ -233,7 +228,6 @@ public void testSimpleSV16_AllMatched() throws GandivaException, Exception { } @Test - @Disabled("GH-43576 - Fix and enable this test") public void testSimpleSV16_GreaterThan64Recs() throws GandivaException, Exception { Field a = Field.nullable("a", int32); Field b = Field.nullable("b", int32); @@ -265,7 +259,6 @@ public void testSimpleSV16_GreaterThan64Recs() throws GandivaException, Exceptio } @Test - @Disabled("GH-43576 - Fix and enable this test") public void testSimpleSV32() throws GandivaException, Exception { Field a = Field.nullable("a", int32); Field b = Field.nullable("b", int32); @@ -287,7 +280,6 @@ public void testSimpleSV32() throws GandivaException, Exception { } @Test - @Disabled("GH-43576 - Fix and enable this test") public void testSimpleFilterWithNoOptimisation() throws GandivaException, Exception { Field a = Field.nullable("a", int32); Field b = Field.nullable("b", int32); diff --git a/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorDecimalTest.java b/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorDecimalTest.java index 74180c0f35..3916051224 100644 --- a/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorDecimalTest.java +++ b/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorDecimalTest.java @@ -42,13 +42,11 @@ import org.apache.arrow.vector.types.pojo.ArrowType.Decimal; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class ProjectorDecimalTest extends org.apache.arrow.gandiva.evaluator.BaseEvaluatorTest { @Test - @Disabled("GH-43576 - Fix and enable this test") public void test_add() throws GandivaException { int precision = 38; int scale = 8; @@ -116,7 +114,6 @@ public void test_add() throws GandivaException { } @Test - @Disabled("GH-43576 - Fix and enable this test") public void test_add_literal() throws GandivaException { int precision = 2; int scale = 0; @@ -178,7 +175,6 @@ public void test_add_literal() throws GandivaException { } @Test - @Disabled("GH-43576 - Fix and enable this test") public void test_multiply() throws GandivaException { int precision = 38; int scale = 8; @@ -248,7 +244,6 @@ public void test_multiply() throws GandivaException { } @Test - @Disabled("GH-43576 - Fix and enable this test") public void testCompare() throws GandivaException { Decimal aType = new Decimal(38, 3, 128); Decimal bType = new Decimal(38, 2, 128); @@ -343,7 +338,6 @@ public void testCompare() throws GandivaException { } @Test - @Disabled("GH-43576 - Fix and enable this test") public void testRound() throws GandivaException { Decimal aType = new Decimal(38, 2, 128); Decimal aWithScaleZero = new Decimal(38, 0, 128); @@ -486,7 +480,6 @@ public void testRound() throws GandivaException { } @Test - @Disabled("GH-43576 - Fix and enable this test") public void testCastToDecimal() throws GandivaException { Decimal decimalType = new Decimal(38, 2, 128); Decimal decimalWithScaleOne = new Decimal(38, 1, 128); @@ -613,7 +606,6 @@ public void testCastToDecimal() throws GandivaException { } @Test - @Disabled("GH-43576 - Fix and enable this test") public void testCastToLong() throws GandivaException { Decimal decimalType = new Decimal(38, 2, 128); Field dec = Field.nullable("dec", decimalType); @@ -666,7 +658,6 @@ public void testCastToLong() throws GandivaException { } @Test - @Disabled("GH-43576 - Fix and enable this test") public void testCastToDouble() throws GandivaException { Decimal decimalType = new Decimal(38, 2, 128); Field dec = Field.nullable("dec", decimalType); @@ -721,7 +712,6 @@ public void testCastToDouble() throws GandivaException { } @Test - @Disabled("GH-43576 - Fix and enable this test") public void testCastToString() throws GandivaException { Decimal decimalType = new Decimal(38, 2, 128); Field dec = Field.nullable("dec", decimalType); @@ -783,7 +773,6 @@ public void testCastToString() throws GandivaException { } @Test - @Disabled("GH-43576 - Fix and enable this test") public void testCastStringToDecimal() throws GandivaException { Decimal decimalType = new Decimal(4, 2, 128); Field dec = Field.nullable("dec", decimalType); diff --git a/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorTest.java b/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorTest.java index 0d86bd9e72..f2590226b1 100644 --- a/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorTest.java +++ b/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorTest.java @@ -62,7 +62,6 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -@Disabled("Disabled until GH-43981 is solved") public class ProjectorTest extends BaseEvaluatorTest { private Charset utf8Charset = Charset.forName("UTF-8"); From 2dc4646c0087c8981adc3326ba1b8f27346a92d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Thu, 20 Feb 2025 00:33:05 +0100 Subject: [PATCH 084/408] GH-620: Upgrade to Google flatbuffers 25.2.10 (#622) Closes #620 --- format/src/main/java/org/apache/arrow/flatbuf/Binary.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/BinaryView.java | 2 +- .../src/main/java/org/apache/arrow/flatbuf/BodyCompression.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Bool.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Date.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Decimal.java | 2 +- .../src/main/java/org/apache/arrow/flatbuf/DictionaryBatch.java | 2 +- .../main/java/org/apache/arrow/flatbuf/DictionaryEncoding.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Duration.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Field.java | 2 +- .../src/main/java/org/apache/arrow/flatbuf/FixedSizeBinary.java | 2 +- .../src/main/java/org/apache/arrow/flatbuf/FixedSizeList.java | 2 +- .../src/main/java/org/apache/arrow/flatbuf/FloatingPoint.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Footer.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Int.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Interval.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/KeyValue.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/LargeBinary.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/LargeList.java | 2 +- .../src/main/java/org/apache/arrow/flatbuf/LargeListView.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/LargeUtf8.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/List.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/ListView.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Map.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Message.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Null.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/RecordBatch.java | 2 +- .../src/main/java/org/apache/arrow/flatbuf/RunEndEncoded.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Schema.java | 2 +- .../java/org/apache/arrow/flatbuf/SparseMatrixIndexCSX.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/SparseTensor.java | 2 +- .../java/org/apache/arrow/flatbuf/SparseTensorIndexCOO.java | 2 +- .../java/org/apache/arrow/flatbuf/SparseTensorIndexCSF.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Struct_.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Tensor.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/TensorDim.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Time.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Timestamp.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Union.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Utf8.java | 2 +- format/src/main/java/org/apache/arrow/flatbuf/Utf8View.java | 2 +- pom.xml | 2 +- 42 files changed, 42 insertions(+), 42 deletions(-) diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Binary.java b/format/src/main/java/org/apache/arrow/flatbuf/Binary.java index 2c90e97754..62938e9cd0 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Binary.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Binary.java @@ -38,7 +38,7 @@ */ @SuppressWarnings("unused") public final class Binary extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static Binary getRootAsBinary(ByteBuffer _bb) { return getRootAsBinary(_bb, new Binary()); } public static Binary getRootAsBinary(ByteBuffer _bb, Binary obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/BinaryView.java b/format/src/main/java/org/apache/arrow/flatbuf/BinaryView.java index 3b5212f60b..4efcec0eb4 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/BinaryView.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/BinaryView.java @@ -44,7 +44,7 @@ */ @SuppressWarnings("unused") public final class BinaryView extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static BinaryView getRootAsBinaryView(ByteBuffer _bb) { return getRootAsBinaryView(_bb, new BinaryView()); } public static BinaryView getRootAsBinaryView(ByteBuffer _bb, BinaryView obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/BodyCompression.java b/format/src/main/java/org/apache/arrow/flatbuf/BodyCompression.java index e94c108f35..4b6fcd2c0b 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/BodyCompression.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/BodyCompression.java @@ -40,7 +40,7 @@ */ @SuppressWarnings("unused") public final class BodyCompression extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static BodyCompression getRootAsBodyCompression(ByteBuffer _bb) { return getRootAsBodyCompression(_bb, new BodyCompression()); } public static BodyCompression getRootAsBodyCompression(ByteBuffer _bb, BodyCompression obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Bool.java b/format/src/main/java/org/apache/arrow/flatbuf/Bool.java index 173af2eaa2..0c9c4224f4 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Bool.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Bool.java @@ -35,7 +35,7 @@ @SuppressWarnings("unused") public final class Bool extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static Bool getRootAsBool(ByteBuffer _bb) { return getRootAsBool(_bb, new Bool()); } public static Bool getRootAsBool(ByteBuffer _bb, Bool obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Date.java b/format/src/main/java/org/apache/arrow/flatbuf/Date.java index 413419535e..42a518de06 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Date.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Date.java @@ -43,7 +43,7 @@ */ @SuppressWarnings("unused") public final class Date extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static Date getRootAsDate(ByteBuffer _bb) { return getRootAsDate(_bb, new Date()); } public static Date getRootAsDate(ByteBuffer _bb, Date obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Decimal.java b/format/src/main/java/org/apache/arrow/flatbuf/Decimal.java index 0cc2cbda26..5eaadf234c 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Decimal.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Decimal.java @@ -41,7 +41,7 @@ */ @SuppressWarnings("unused") public final class Decimal extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static Decimal getRootAsDecimal(ByteBuffer _bb) { return getRootAsDecimal(_bb, new Decimal()); } public static Decimal getRootAsDecimal(ByteBuffer _bb, Decimal obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/DictionaryBatch.java b/format/src/main/java/org/apache/arrow/flatbuf/DictionaryBatch.java index 52cc347093..2bee8853c7 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/DictionaryBatch.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/DictionaryBatch.java @@ -43,7 +43,7 @@ */ @SuppressWarnings("unused") public final class DictionaryBatch extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static DictionaryBatch getRootAsDictionaryBatch(ByteBuffer _bb) { return getRootAsDictionaryBatch(_bb, new DictionaryBatch()); } public static DictionaryBatch getRootAsDictionaryBatch(ByteBuffer _bb, DictionaryBatch obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/DictionaryEncoding.java b/format/src/main/java/org/apache/arrow/flatbuf/DictionaryEncoding.java index 3d797f7028..ad1f74f833 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/DictionaryEncoding.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/DictionaryEncoding.java @@ -35,7 +35,7 @@ @SuppressWarnings("unused") public final class DictionaryEncoding extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static DictionaryEncoding getRootAsDictionaryEncoding(ByteBuffer _bb) { return getRootAsDictionaryEncoding(_bb, new DictionaryEncoding()); } public static DictionaryEncoding getRootAsDictionaryEncoding(ByteBuffer _bb, DictionaryEncoding obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Duration.java b/format/src/main/java/org/apache/arrow/flatbuf/Duration.java index a77c7b77cd..bef934cef3 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Duration.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Duration.java @@ -35,7 +35,7 @@ @SuppressWarnings("unused") public final class Duration extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static Duration getRootAsDuration(ByteBuffer _bb) { return getRootAsDuration(_bb, new Duration()); } public static Duration getRootAsDuration(ByteBuffer _bb, Duration obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Field.java b/format/src/main/java/org/apache/arrow/flatbuf/Field.java index 7ea73e8b6a..de7b25b8db 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Field.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Field.java @@ -40,7 +40,7 @@ */ @SuppressWarnings("unused") public final class Field extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static Field getRootAsField(ByteBuffer _bb) { return getRootAsField(_bb, new Field()); } public static Field getRootAsField(ByteBuffer _bb, Field obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/FixedSizeBinary.java b/format/src/main/java/org/apache/arrow/flatbuf/FixedSizeBinary.java index 54bf8395e1..2e3e06d580 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/FixedSizeBinary.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/FixedSizeBinary.java @@ -35,7 +35,7 @@ @SuppressWarnings("unused") public final class FixedSizeBinary extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static FixedSizeBinary getRootAsFixedSizeBinary(ByteBuffer _bb) { return getRootAsFixedSizeBinary(_bb, new FixedSizeBinary()); } public static FixedSizeBinary getRootAsFixedSizeBinary(ByteBuffer _bb, FixedSizeBinary obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/FixedSizeList.java b/format/src/main/java/org/apache/arrow/flatbuf/FixedSizeList.java index e3264eff30..c6e2e56f22 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/FixedSizeList.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/FixedSizeList.java @@ -35,7 +35,7 @@ @SuppressWarnings("unused") public final class FixedSizeList extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static FixedSizeList getRootAsFixedSizeList(ByteBuffer _bb) { return getRootAsFixedSizeList(_bb, new FixedSizeList()); } public static FixedSizeList getRootAsFixedSizeList(ByteBuffer _bb, FixedSizeList obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/FloatingPoint.java b/format/src/main/java/org/apache/arrow/flatbuf/FloatingPoint.java index 8bf8839649..8f552b8f78 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/FloatingPoint.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/FloatingPoint.java @@ -35,7 +35,7 @@ @SuppressWarnings("unused") public final class FloatingPoint extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static FloatingPoint getRootAsFloatingPoint(ByteBuffer _bb) { return getRootAsFloatingPoint(_bb, new FloatingPoint()); } public static FloatingPoint getRootAsFloatingPoint(ByteBuffer _bb, FloatingPoint obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Footer.java b/format/src/main/java/org/apache/arrow/flatbuf/Footer.java index b5a4bbe449..61b6accd2a 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Footer.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Footer.java @@ -40,7 +40,7 @@ */ @SuppressWarnings("unused") public final class Footer extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static Footer getRootAsFooter(ByteBuffer _bb) { return getRootAsFooter(_bb, new Footer()); } public static Footer getRootAsFooter(ByteBuffer _bb, Footer obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Int.java b/format/src/main/java/org/apache/arrow/flatbuf/Int.java index b307770e32..378c0125eb 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Int.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Int.java @@ -35,7 +35,7 @@ @SuppressWarnings("unused") public final class Int extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static Int getRootAsInt(ByteBuffer _bb) { return getRootAsInt(_bb, new Int()); } public static Int getRootAsInt(ByteBuffer _bb, Int obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Interval.java b/format/src/main/java/org/apache/arrow/flatbuf/Interval.java index c91de4210f..a9fe92f3e0 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Interval.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Interval.java @@ -35,7 +35,7 @@ @SuppressWarnings("unused") public final class Interval extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static Interval getRootAsInterval(ByteBuffer _bb) { return getRootAsInterval(_bb, new Interval()); } public static Interval getRootAsInterval(ByteBuffer _bb, Interval obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/KeyValue.java b/format/src/main/java/org/apache/arrow/flatbuf/KeyValue.java index 05b26faace..fb35b6640a 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/KeyValue.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/KeyValue.java @@ -40,7 +40,7 @@ */ @SuppressWarnings("unused") public final class KeyValue extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static KeyValue getRootAsKeyValue(ByteBuffer _bb) { return getRootAsKeyValue(_bb, new KeyValue()); } public static KeyValue getRootAsKeyValue(ByteBuffer _bb, KeyValue obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/LargeBinary.java b/format/src/main/java/org/apache/arrow/flatbuf/LargeBinary.java index 60d7cf33f8..e74276a37c 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/LargeBinary.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/LargeBinary.java @@ -39,7 +39,7 @@ */ @SuppressWarnings("unused") public final class LargeBinary extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static LargeBinary getRootAsLargeBinary(ByteBuffer _bb) { return getRootAsLargeBinary(_bb, new LargeBinary()); } public static LargeBinary getRootAsLargeBinary(ByteBuffer _bb, LargeBinary obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/LargeList.java b/format/src/main/java/org/apache/arrow/flatbuf/LargeList.java index 9ef8f7b287..fe91784c82 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/LargeList.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/LargeList.java @@ -39,7 +39,7 @@ */ @SuppressWarnings("unused") public final class LargeList extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static LargeList getRootAsLargeList(ByteBuffer _bb) { return getRootAsLargeList(_bb, new LargeList()); } public static LargeList getRootAsLargeList(ByteBuffer _bb, LargeList obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/LargeListView.java b/format/src/main/java/org/apache/arrow/flatbuf/LargeListView.java index 558c8e31d3..b062dc67b3 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/LargeListView.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/LargeListView.java @@ -39,7 +39,7 @@ */ @SuppressWarnings("unused") public final class LargeListView extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static LargeListView getRootAsLargeListView(ByteBuffer _bb) { return getRootAsLargeListView(_bb, new LargeListView()); } public static LargeListView getRootAsLargeListView(ByteBuffer _bb, LargeListView obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/LargeUtf8.java b/format/src/main/java/org/apache/arrow/flatbuf/LargeUtf8.java index 6451124c4f..f2b0f8bb3e 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/LargeUtf8.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/LargeUtf8.java @@ -39,7 +39,7 @@ */ @SuppressWarnings("unused") public final class LargeUtf8 extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static LargeUtf8 getRootAsLargeUtf8(ByteBuffer _bb) { return getRootAsLargeUtf8(_bb, new LargeUtf8()); } public static LargeUtf8 getRootAsLargeUtf8(ByteBuffer _bb, LargeUtf8 obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/List.java b/format/src/main/java/org/apache/arrow/flatbuf/List.java index af88b8db91..eede1d4c1b 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/List.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/List.java @@ -35,7 +35,7 @@ @SuppressWarnings("unused") public final class List extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static List getRootAsList(ByteBuffer _bb) { return getRootAsList(_bb, new List()); } public static List getRootAsList(ByteBuffer _bb, List obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/ListView.java b/format/src/main/java/org/apache/arrow/flatbuf/ListView.java index 6a0c1f4883..19415a17d9 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/ListView.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/ListView.java @@ -40,7 +40,7 @@ */ @SuppressWarnings("unused") public final class ListView extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static ListView getRootAsListView(ByteBuffer _bb) { return getRootAsListView(_bb, new ListView()); } public static ListView getRootAsListView(ByteBuffer _bb, ListView obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Map.java b/format/src/main/java/org/apache/arrow/flatbuf/Map.java index 096ae530f3..8f996bea45 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Map.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Map.java @@ -62,7 +62,7 @@ */ @SuppressWarnings("unused") public final class Map extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static Map getRootAsMap(ByteBuffer _bb) { return getRootAsMap(_bb, new Map()); } public static Map getRootAsMap(ByteBuffer _bb, Map obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Message.java b/format/src/main/java/org/apache/arrow/flatbuf/Message.java index c5eb9321ba..45066eb965 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Message.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Message.java @@ -35,7 +35,7 @@ @SuppressWarnings("unused") public final class Message extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static Message getRootAsMessage(ByteBuffer _bb) { return getRootAsMessage(_bb, new Message()); } public static Message getRootAsMessage(ByteBuffer _bb, Message obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Null.java b/format/src/main/java/org/apache/arrow/flatbuf/Null.java index 2ef526d040..88a0c31f5a 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Null.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Null.java @@ -38,7 +38,7 @@ */ @SuppressWarnings("unused") public final class Null extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static Null getRootAsNull(ByteBuffer _bb) { return getRootAsNull(_bb, new Null()); } public static Null getRootAsNull(ByteBuffer _bb, Null obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/RecordBatch.java b/format/src/main/java/org/apache/arrow/flatbuf/RecordBatch.java index f3004cca23..db68dc2150 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/RecordBatch.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/RecordBatch.java @@ -40,7 +40,7 @@ */ @SuppressWarnings("unused") public final class RecordBatch extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static RecordBatch getRootAsRecordBatch(ByteBuffer _bb) { return getRootAsRecordBatch(_bb, new RecordBatch()); } public static RecordBatch getRootAsRecordBatch(ByteBuffer _bb, RecordBatch obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/RunEndEncoded.java b/format/src/main/java/org/apache/arrow/flatbuf/RunEndEncoded.java index 87a968f197..a1b7699597 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/RunEndEncoded.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/RunEndEncoded.java @@ -42,7 +42,7 @@ */ @SuppressWarnings("unused") public final class RunEndEncoded extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static RunEndEncoded getRootAsRunEndEncoded(ByteBuffer _bb) { return getRootAsRunEndEncoded(_bb, new RunEndEncoded()); } public static RunEndEncoded getRootAsRunEndEncoded(ByteBuffer _bb, RunEndEncoded obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Schema.java b/format/src/main/java/org/apache/arrow/flatbuf/Schema.java index 0b2d1880e5..0fd03b5a67 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Schema.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Schema.java @@ -39,7 +39,7 @@ */ @SuppressWarnings("unused") public final class Schema extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static Schema getRootAsSchema(ByteBuffer _bb) { return getRootAsSchema(_bb, new Schema()); } public static Schema getRootAsSchema(ByteBuffer _bb, Schema obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/SparseMatrixIndexCSX.java b/format/src/main/java/org/apache/arrow/flatbuf/SparseMatrixIndexCSX.java index ad8d6ffad7..b2c3e6d800 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/SparseMatrixIndexCSX.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/SparseMatrixIndexCSX.java @@ -38,7 +38,7 @@ */ @SuppressWarnings("unused") public final class SparseMatrixIndexCSX extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static SparseMatrixIndexCSX getRootAsSparseMatrixIndexCSX(ByteBuffer _bb) { return getRootAsSparseMatrixIndexCSX(_bb, new SparseMatrixIndexCSX()); } public static SparseMatrixIndexCSX getRootAsSparseMatrixIndexCSX(ByteBuffer _bb, SparseMatrixIndexCSX obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/SparseTensor.java b/format/src/main/java/org/apache/arrow/flatbuf/SparseTensor.java index fb75d872a1..4cf10e03a2 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/SparseTensor.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/SparseTensor.java @@ -35,7 +35,7 @@ @SuppressWarnings("unused") public final class SparseTensor extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static SparseTensor getRootAsSparseTensor(ByteBuffer _bb) { return getRootAsSparseTensor(_bb, new SparseTensor()); } public static SparseTensor getRootAsSparseTensor(ByteBuffer _bb, SparseTensor obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/SparseTensorIndexCOO.java b/format/src/main/java/org/apache/arrow/flatbuf/SparseTensorIndexCOO.java index 204d6fa2b3..e2321ae412 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/SparseTensorIndexCOO.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/SparseTensorIndexCOO.java @@ -69,7 +69,7 @@ */ @SuppressWarnings("unused") public final class SparseTensorIndexCOO extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static SparseTensorIndexCOO getRootAsSparseTensorIndexCOO(ByteBuffer _bb) { return getRootAsSparseTensorIndexCOO(_bb, new SparseTensorIndexCOO()); } public static SparseTensorIndexCOO getRootAsSparseTensorIndexCOO(ByteBuffer _bb, SparseTensorIndexCOO obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/SparseTensorIndexCSF.java b/format/src/main/java/org/apache/arrow/flatbuf/SparseTensorIndexCSF.java index a67eddd074..f2d42eb656 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/SparseTensorIndexCSF.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/SparseTensorIndexCSF.java @@ -38,7 +38,7 @@ */ @SuppressWarnings("unused") public final class SparseTensorIndexCSF extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static SparseTensorIndexCSF getRootAsSparseTensorIndexCSF(ByteBuffer _bb) { return getRootAsSparseTensorIndexCSF(_bb, new SparseTensorIndexCSF()); } public static SparseTensorIndexCSF getRootAsSparseTensorIndexCSF(ByteBuffer _bb, SparseTensorIndexCSF obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Struct_.java b/format/src/main/java/org/apache/arrow/flatbuf/Struct_.java index d1758b7014..4eb4bbcc7b 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Struct_.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Struct_.java @@ -40,7 +40,7 @@ */ @SuppressWarnings("unused") public final class Struct_ extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static Struct_ getRootAsStruct_(ByteBuffer _bb) { return getRootAsStruct_(_bb, new Struct_()); } public static Struct_ getRootAsStruct_(ByteBuffer _bb, Struct_ obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Tensor.java b/format/src/main/java/org/apache/arrow/flatbuf/Tensor.java index 7fe869cfe5..4bcf93d9a0 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Tensor.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Tensor.java @@ -35,7 +35,7 @@ @SuppressWarnings("unused") public final class Tensor extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static Tensor getRootAsTensor(ByteBuffer _bb) { return getRootAsTensor(_bb, new Tensor()); } public static Tensor getRootAsTensor(ByteBuffer _bb, Tensor obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/TensorDim.java b/format/src/main/java/org/apache/arrow/flatbuf/TensorDim.java index c7d215811b..36459e742a 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/TensorDim.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/TensorDim.java @@ -40,7 +40,7 @@ */ @SuppressWarnings("unused") public final class TensorDim extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static TensorDim getRootAsTensorDim(ByteBuffer _bb) { return getRootAsTensorDim(_bb, new TensorDim()); } public static TensorDim getRootAsTensorDim(ByteBuffer _bb, TensorDim obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Time.java b/format/src/main/java/org/apache/arrow/flatbuf/Time.java index 4e8e24d3ab..2d692c1006 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Time.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Time.java @@ -51,7 +51,7 @@ */ @SuppressWarnings("unused") public final class Time extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static Time getRootAsTime(ByteBuffer _bb) { return getRootAsTime(_bb, new Time()); } public static Time getRootAsTime(ByteBuffer _bb, Time obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Timestamp.java b/format/src/main/java/org/apache/arrow/flatbuf/Timestamp.java index 587c81527f..df0e79fced 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Timestamp.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Timestamp.java @@ -142,7 +142,7 @@ */ @SuppressWarnings("unused") public final class Timestamp extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static Timestamp getRootAsTimestamp(ByteBuffer _bb) { return getRootAsTimestamp(_bb, new Timestamp()); } public static Timestamp getRootAsTimestamp(ByteBuffer _bb, Timestamp obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Union.java b/format/src/main/java/org/apache/arrow/flatbuf/Union.java index 6ea670edc4..21f3b47437 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Union.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Union.java @@ -41,7 +41,7 @@ */ @SuppressWarnings("unused") public final class Union extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static Union getRootAsUnion(ByteBuffer _bb) { return getRootAsUnion(_bb, new Union()); } public static Union getRootAsUnion(ByteBuffer _bb, Union obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Utf8.java b/format/src/main/java/org/apache/arrow/flatbuf/Utf8.java index 0297f39553..1abd8ae6ce 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Utf8.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Utf8.java @@ -38,7 +38,7 @@ */ @SuppressWarnings("unused") public final class Utf8 extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static Utf8 getRootAsUtf8(ByteBuffer _bb) { return getRootAsUtf8(_bb, new Utf8()); } public static Utf8 getRootAsUtf8(ByteBuffer _bb, Utf8 obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/format/src/main/java/org/apache/arrow/flatbuf/Utf8View.java b/format/src/main/java/org/apache/arrow/flatbuf/Utf8View.java index b8f9eddd94..68b99a32fb 100644 --- a/format/src/main/java/org/apache/arrow/flatbuf/Utf8View.java +++ b/format/src/main/java/org/apache/arrow/flatbuf/Utf8View.java @@ -44,7 +44,7 @@ */ @SuppressWarnings("unused") public final class Utf8View extends Table { - public static void ValidateVersion() { Constants.FLATBUFFERS_25_1_24(); } + public static void ValidateVersion() { Constants.FLATBUFFERS_25_2_10(); } public static Utf8View getRootAsUtf8View(ByteBuffer _bb) { return getRootAsUtf8View(_bb, new Utf8View()); } public static Utf8View getRootAsUtf8View(ByteBuffer _bb, Utf8View obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } diff --git a/pom.xml b/pom.xml index eca403363c..6daa8ecb13 100644 --- a/pom.xml +++ b/pom.xml @@ -101,7 +101,7 @@ under the License. 3.25.5 2.18.2 3.4.1 - 25.1.24 + 25.2.10 1.12.0 2 From 17ab467c3bdf29fa28ee9ccf66c71ad9f10d2280 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 19 Feb 2025 18:34:58 -0500 Subject: [PATCH 085/408] MINOR: Bump io.netty:netty-bom from 4.1.117.Final to 4.1.118.Final (#616) Bumps [io.netty:netty-bom](https://github.com/netty/netty) from 4.1.117.Final to 4.1.118.Final. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6daa8ecb13..52e383ea0f 100644 --- a/pom.xml +++ b/pom.xml @@ -96,7 +96,7 @@ under the License. 5.11.4 2.0.16 33.4.0-jre - 4.1.117.Final + 4.1.118.Final 1.70.0 3.25.5 2.18.2 From 81a5c2d96be13796873db48dc592e38396d28730 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Feb 2025 06:22:59 -0500 Subject: [PATCH 086/408] MINOR: Bump com.google.api.grpc:proto-google-common-protos from 2.51.0 to 2.52.0 (#617) Bumps [com.google.api.grpc:proto-google-common-protos](https://github.com/googleapis/sdk-platform-java) from 2.51.0 to 2.52.0. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml index 7484349730..b516e71bda 100644 --- a/flight/flight-core/pom.xml +++ b/flight/flight-core/pom.xml @@ -134,7 +134,7 @@ under the License. com.google.api.grpc proto-google-common-protos - 2.51.0 + 2.52.0 test From 799d9fd45e36d962c7a7bae0f3903bb414062554 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 21 Feb 2025 05:43:16 +0900 Subject: [PATCH 087/408] GH-623: Cleanup enabling Gandiva tests change (#624) ## What's Changed * Add a comment when we can remove this workaround. * Remove a needless workaround for Windows. Closes #623. --- .github/workflows/rc.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 4f240e6714..49a39102f4 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -107,6 +107,12 @@ jobs: - name: Extract source archive run: | tar -xf apache-arrow-java-*.tar.gz --strip-components=1 + # We always use the main branch for apache/arrow for now. + # Because we want to use + # https://github.com/apache/arrow/pull/45114 in + # apache/arrow-java. We can revert this workaround once Apache + # Arrow 20.0.0 that includes the change released. + # # - name: Download the latest Apache Arrow C++ # if: github.event_name != 'schedule' # run: | @@ -299,13 +305,13 @@ jobs: shell: bash run: | tar -xf apache-arrow-java-*.tar.gz --strip-components=1 - # - name: Download the latest Apache Arrow C++ - # if: github.event_name != 'schedule' - # shell: bash - # run: | - # ci/scripts/download_cpp.sh + - name: Download the latest Apache Arrow C++ + if: github.event_name != 'schedule' + shell: bash + run: | + ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ - # if: github.event_name == 'schedule' + if: github.event_name == 'schedule' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: apache/arrow From 370031e620da91bda2f00f29b5b8ea962aec9f98 Mon Sep 17 00:00:00 2001 From: lriggs Date: Thu, 20 Feb 2025 16:05:08 -0800 Subject: [PATCH 088/408] GH-601: [Gandiva] Synchronize some methods on the Projector (#602) ### Rationale for this change Multiple threads can attempt to create the same llvm expression in Gandiva. This isn't allowed with the new JIT compiler, so synchronizing will prevent this scenario. ### What changes are included in this PR? Synchronize some methods to avoid adding duplicate llvm expressions. ### Are these changes tested? Yes, through unit tests in Gandiva. ### Are there any user-facing changes? No. Closes GH-601 --- .../java/org/apache/arrow/gandiva/evaluator/Projector.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Projector.java b/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Projector.java index 5c16c46e5e..9c5b22d659 100644 --- a/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Projector.java +++ b/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Projector.java @@ -188,7 +188,7 @@ public static Projector make( * @param configurationId Custom configuration created through config builder. * @return A native evaluator object that can be used to invoke these projections on a RecordBatch */ - public static Projector make( + public static synchronized Projector make( Schema schema, List exprs, SelectionVectorType selectionVectorType, @@ -314,7 +314,7 @@ public void evaluate( outColumns); } - private void evaluate( + private synchronized void evaluate( int numRows, List buffers, List buffersLayout, From 6564b471fcd8dfac3701660a157464feb5312220 Mon Sep 17 00:00:00 2001 From: David Li Date: Thu, 20 Feb 2025 21:12:05 -0500 Subject: [PATCH 089/408] MINOR: Fix typo in dev_pr.js (#626) ## What's Changed Fix a typo in the dev_pr script affecting MINOR PRs. --- .github/workflows/dev_pr.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev_pr.js b/.github/workflows/dev_pr.js index 19fd6e5695..13acc946e1 100644 --- a/.github/workflows/dev_pr.js +++ b/.github/workflows/dev_pr.js @@ -121,7 +121,7 @@ module.exports = { check_title_format: function({core, github, context}) { const title = context.payload.pull_request.title; if (title.startsWith("MINOR: ")) { - context.log("PR is a minor PR"); + console.log("PR is a minor PR"); return {"issue": null}; } From 149e97a8f70cee9eff0487874df01d7f50eefc0f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Feb 2025 21:25:55 -0500 Subject: [PATCH 090/408] MINOR: Bump checker.framework.version from 3.48.4 to 3.49.0 (#598) Bumps `checker.framework.version` from 3.48.4 to 3.49.0. Updates `org.checkerframework:checker-qual` from 3.48.4 to 3.49.0 Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 52e383ea0f..d944109bef 100644 --- a/pom.xml +++ b/pom.xml @@ -109,7 +109,7 @@ under the License. true 2.31.0 5.15.2 - 3.48.4 + 3.49.0 1.5.16 none -Xdoclint:none From 1cdbe834f89f4519e121673d2f1a37dff6bba5dc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Feb 2025 05:54:03 -0500 Subject: [PATCH 091/408] MINOR: Bump org.bouncycastle:bcpkix-jdk18on from 1.79 to 1.80 (#597) Bumps [org.bouncycastle:bcpkix-jdk18on](https://github.com/bcgit/bc-java) from 1.79 to 1.80. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 204e17efe8..1daa023778 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -121,7 +121,7 @@ under the License. org.bouncycastle bcpkix-jdk18on - 1.79 + 1.80 From cdef9d33fefc94d0b71163fa077f9d66ea45c60f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Feb 2025 09:05:53 -0500 Subject: [PATCH 092/408] MINOR: Bump com.github.luben:zstd-jni from 1.5.6-9 to 1.5.7-1 (#628) Bumps [com.github.luben:zstd-jni](https://github.com/luben/zstd-jni) from 1.5.6-9 to 1.5.7-1. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- compression/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compression/pom.xml b/compression/pom.xml index 3c339d557a..8d398fe0d4 100644 --- a/compression/pom.xml +++ b/compression/pom.xml @@ -55,7 +55,7 @@ under the License. com.github.luben zstd-jni - 1.5.6-9 + 1.5.7-1 From f8660a84ed3893b347ed9557a8e827e446432859 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 24 Feb 2025 19:18:42 -0500 Subject: [PATCH 093/408] GH-629: Add post-release tasks to dev/release/README.md (#630) ## What's Changed Add instructions for undocumented post-release tasks. Closes #629. --- dev/release/README.md | 68 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/dev/release/README.md b/dev/release/README.md index f62befba03..9069a4fdaf 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -172,6 +172,74 @@ $ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/bump_version.sh 19.0.0-SNAPSHOT It creates a feature branch and adds a commit that bumps version. This opens a pull request from the feature branch by `gh pr create`. +### Close the GitHub milestone + +Close the milestone here, then open a new milestone for the next release: + +https://github.com/apache/arrow-java/milestones + +The milestone should be named after the version (e.g. "18.4.0"). + +### Publish the release blog post + +Open a pull request on +[apache/arrow-site](https://github.com/apache/arrow-site) announcing the new +release and summarizing the changes. Do not include the full changelog, just +important entries (breaking changes, new features, major bug fixes) and a link +to the full changelog. See an [example +PR](https://github.com/apache/arrow-site/pull/594). + +### Announce the new release on the mailing list + +Send an email to "announce@apache.org" from your Apache email, CC'ing +dev@arrow.apache.org/user@arrow.apache.org. See an [example +post](https://lists.apache.org/thread/bxpt0r8kw0ltgywnylqdroskkt6966z4). + +``` +To: announce@apache.org +CC: dev@arrow.apache.org, user@arrow.apache.org +Subject: [ANNOUNCE] Apache Arrow Java 18.2.0 released + +The Apache Arrow community is pleased to announce the Arrow Java 18.2.0 release. + +The release is available now from our website: + https://arrow.apache.org/install/ +and + https://www.apache.org/dyn/closer.cgi/arrow/apache-arrow-java-18.2.0/ + +Read about what's new in the release at: + https://arrow.apache.org/blog/2025/02/19/arrow-java-18.2.0/ + +Read the full changelog: + https://github.com/apache/arrow-java/commits/v18.2.0 + +What is Apache Arrow? +------------------------------- + +Apache Arrow is a universal columnar format and multi-language toolbox +for fast data interchange and in-memory analytics. It houses a set of +canonical in-memory representations of flat and hierarchical data +along with multiple language-bindings for structure manipulation. It +also provides low-overhead streaming and batch messaging, zero-copy +interprocess communication (IPC), and vectorized in-memory analytics +libraries. + +Please report any feedback to the mailing lists: + https://lists.apache.org/list.html?dev@arrow.apache.org + +Regards, +The Apache Arrow community. +``` + +### Announce the new release on social media + +Make a post on our [BlueSky](https://bsky.app/profile/arrow.apache.org) and +[LinkedIn](https://www.linkedin.com/company/apache-arrow/) accounts. (Ask +your fellow PMC members for access if need be, or ask a PMC member to make the +post on your behalf.) The post should link to the blog post. See [example +BlueSky post](https://bsky.app/profile/arrow.apache.org/post/3lioi6ov5h22d) +and [example LinkedIn post](https://www.linkedin.com/posts/apache-arrow_apache-arrow-java-1820-release-activity-7298633716522758144-L71x). + ## Verify We have a script to verify a RC. From af5b85ca2290c80cff4f84eb5e7d6e3782af1430 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2025 19:22:55 -0500 Subject: [PATCH 094/408] MINOR: Bump dep.junit.jupiter.version from 5.11.4 to 5.12.0 (#631) Bumps `dep.junit.jupiter.version` from 5.11.4 to 5.12.0. Updates `org.junit.jupiter:junit-jupiter-engine` from 5.11.4 to 5.12.0 Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d944109bef..95cca18c6c 100644 --- a/pom.xml +++ b/pom.xml @@ -93,7 +93,7 @@ under the License. ${project.build.directory}/generated-sources 1.9.0 - 5.11.4 + 5.12.0 2.0.16 33.4.0-jre 4.1.118.Final From fad445a83325b2b0c011418cf43bcac18186c12e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2025 19:23:46 -0500 Subject: [PATCH 095/408] MINOR: Bump com.puppycrawl.tools:checkstyle from 10.21.2 to 10.21.3 (#633) Bumps [com.puppycrawl.tools:checkstyle](https://github.com/checkstyle/checkstyle) from 10.21.2 to 10.21.3. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 95cca18c6c..44eed9ae03 100644 --- a/pom.xml +++ b/pom.xml @@ -105,7 +105,7 @@ under the License. 1.12.0 2 - 10.21.2 + 10.21.3 true 2.31.0 5.15.2 From 91b17f63c36ee01dc3fb2d4ab64fc8588426f1e8 Mon Sep 17 00:00:00 2001 From: wsuppiger <20688370+wsuppiger@users.noreply.github.com> Date: Tue, 25 Feb 2025 07:20:44 -0500 Subject: [PATCH 096/408] GH-625: Map MinorType getNewFieldWriter returns UnionMapWriter (#627) ## What's Changed Closes #625 --------- Co-authored-by: William Suppiger --- .../org/apache/arrow/vector/types/Types.java | 3 ++- .../apache/arrow/vector/TestMapVector.java | 22 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/vector/src/main/java/org/apache/arrow/vector/types/Types.java index e9b963b62c..17503f98c8 100644 --- a/vector/src/main/java/org/apache/arrow/vector/types/Types.java +++ b/vector/src/main/java/org/apache/arrow/vector/types/Types.java @@ -116,6 +116,7 @@ import org.apache.arrow.vector.complex.impl.UnionLargeListViewWriter; import org.apache.arrow.vector.complex.impl.UnionLargeListWriter; import org.apache.arrow.vector.complex.impl.UnionListWriter; +import org.apache.arrow.vector.complex.impl.UnionMapWriter; import org.apache.arrow.vector.complex.impl.UnionWriter; import org.apache.arrow.vector.complex.impl.VarBinaryWriterImpl; import org.apache.arrow.vector.complex.impl.VarCharWriterImpl; @@ -721,7 +722,7 @@ public FieldVector getNewVector( @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new UnionListWriter((MapVector) vector); + return new UnionMapWriter((MapVector) vector); } }, TIMESTAMPSECTZ(null) { diff --git a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java index a4197c50b5..313d83ec91 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java @@ -35,6 +35,7 @@ import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; +import org.apache.arrow.vector.complex.writer.FieldWriter; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; @@ -1241,4 +1242,25 @@ public void testMakeTransferPairPreserveNullability() { assertEquals(intField, vec.getField().getChildren().get(0)); assertEquals(intField, res.getField().getChildren().get(0)); } + + @Test + public void testMapTypeReturnsSupportedMapWriter() { + try (final MapVector vector = MapVector.empty("map", allocator, false)) { + vector.allocateNew(); + FieldWriter mapWriter = MinorType.MAP.getNewFieldWriter(vector); + + mapWriter.startMap(); + mapWriter.startEntry(); + mapWriter.key().bigInt().writeBigInt(1); + mapWriter.value().integer().writeInt(11); + mapWriter.endEntry(); + mapWriter.endMap(); + + Object result = vector.getObject(0); + ArrayList resultSet = (ArrayList) result; + Map resultStruct = (Map) resultSet.get(0); + assertEquals(1L, getResultKey(resultStruct)); + assertEquals(11, getResultValue(resultStruct)); + } + } } From a194e1a3b28f8a9fb2a67be22cda588d3c4e247b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Feb 2025 19:35:49 -0500 Subject: [PATCH 097/408] MINOR: [CI] Bump actions/upload-artifact from 4.6.0 to 4.6.1 (#635) Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 4.6.0 to 4.6.1. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 49a39102f4..ab1cbfcc9a 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -71,7 +71,7 @@ jobs: run: | dev/release/run_rat.sh "${TAR_GZ}" - name: Upload source archive - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 with: name: release-source path: | @@ -154,7 +154,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf jni-linux-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 with: name: jni-linux-${{ matrix.platform.arch }} path: jni-linux-${{ matrix.platform.arch }}.tar.gz @@ -280,7 +280,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf jni-macos-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 with: name: jni-macos-${{ matrix.platform.arch }} path: jni-macos-${{ matrix.platform.arch }}.tar.gz @@ -358,7 +358,7 @@ jobs: shell: bash run: tar -cvzf jni-windows-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 with: name: jni-windows-${{ matrix.platform.arch }} path: jni-windows-${{ matrix.platform.arch }}.tar.gz @@ -430,12 +430,12 @@ jobs: cp -a target/site/apidocs reference tar -cvzf reference.tar.gz reference - name: Upload binaries - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 with: name: release-binaries path: binaries/* - name: Upload docs - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 with: name: reference path: reference.tar.gz @@ -473,7 +473,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf html.tar.gz -C docs/build html - name: Upload artifacts - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 with: name: release-html path: html.tar.gz From 045249ad47ee480e859338d81ab75b4d1af91ed5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Feb 2025 21:00:24 -0500 Subject: [PATCH 098/408] MINOR: Bump org.apache.calcite.avatica:avatica from 1.25.0 to 1.26.0 (#634) Bumps [org.apache.calcite.avatica:avatica](https://github.com/apache/calcite-avatica) from 1.25.0 to 1.26.0. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 1daa023778..237d450eec 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -115,7 +115,7 @@ under the License. org.apache.calcite.avatica avatica - 1.25.0 + 1.26.0 From f1d09e925e75dd4ecdb16395ede92b6c73861df9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Mar 2025 06:00:03 -0500 Subject: [PATCH 099/408] MINOR: Bump com.fasterxml.jackson:jackson-bom from 2.18.2 to 2.18.3 (#645) Bumps [com.fasterxml.jackson:jackson-bom](https://github.com/FasterXML/jackson-bom) from 2.18.2 to 2.18.3. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 44eed9ae03..e22cc8daef 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,7 @@ under the License. 4.1.118.Final 1.70.0 3.25.5 - 2.18.2 + 2.18.3 3.4.1 25.2.10 1.12.0 From 432d1bcbaf8f8de73c61eba7f675a1df8d97045e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Mar 2025 06:00:38 -0500 Subject: [PATCH 100/408] MINOR: [CI] Bump actions/download-artifact from 4.1.8 to 4.1.9 (#646) Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 4.1.8 to 4.1.9. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index ab1cbfcc9a..f62b44bc23 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -101,7 +101,7 @@ jobs: packages: write steps: - name: Download source archive - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9 with: name: release-source - name: Extract source archive @@ -174,7 +174,7 @@ jobs: MACOSX_DEPLOYMENT_TARGET: "14.0" steps: - name: Download source archive - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9 with: name: release-source - name: Extract source archive @@ -298,7 +298,7 @@ jobs: arch: "x86_64" steps: - name: Download source archive - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9 with: name: release-source - name: Extract source archive @@ -371,7 +371,7 @@ jobs: - jni-windows steps: - name: Download artifacts - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9 with: path: artifacts - name: Decompress artifacts @@ -452,11 +452,11 @@ jobs: with: cache: 'pip' - name: Download source archive - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9 with: name: release-source - name: Download Javadocs - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9 with: name: reference - name: Extract source archive @@ -521,7 +521,7 @@ jobs: cp ../.asf.yaml ./ git add .nojekyll .asf.yaml - name: Download - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9 with: name: release-html - name: Extract @@ -557,7 +557,7 @@ jobs: - ubuntu-latest steps: - name: Download release artifacts - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9 with: pattern: release-* - name: Verify @@ -591,7 +591,7 @@ jobs: contents: write steps: - name: Download release artifacts - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9 with: pattern: release-* path: artifacts From 49e237b2591b8506be0e909e1e78a71d28a46478 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Mar 2025 07:03:37 +0900 Subject: [PATCH 101/408] MINOR: Bump logback.version from 1.5.16 to 1.5.17 (#641) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps `logback.version` from 1.5.16 to 1.5.17. Updates `ch.qos.logback:logback-classic` from 1.5.16 to 1.5.17
Release notes

Sourced from ch.qos.logback:logback-classic's releases.

Logback 1.5.17

2025-02-25 Release of logback version 1.5.17

• Fixed Jansi 2.4.0 color-coded output not working on Windows CMD.exe console when the default terminal application is set to "Windows Console Host". This problem was reported in issues/753 by Michael Lyubkin.

• Fixed race condition occurring in case MDC class is initialized while org.slf4j.LoggerFactory is initializing logback-classic's LoggerContext. When this race conditions occurs, the MDCAdapter instance used by MDC does not match the instance used by logback-classic. This issue was reported in SLF4J issues/450. While logback-classic version 1.5.17 remains compatible with SLF4J versions in the 2.0.x series, fixing this particular MDC issue requires SLF4J version 2.0.17.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit 10358724ed723b3745c010aa40cb02a2dfed4593 associated with the tag v_1.5.17. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits
  • 1035872 prepare release 1.5.17
  • 2e6984d bump to slf4j version 2.0.17
  • 1009952 use a new LoggerContert instance when running LogbackListenerTest. This shoul...
  • a3bb4b0 Merge branch 'master' of github.com:qos-ch/logback
  • b507297 Fixed race condition occurring in case MDC class is initialized while org.slf...
  • f5b3bc5 add warning about the deprecation of SerializedModelConfigurator if activated
  • 5bc0998 Update README.md
  • 5610c96 correct relocation address
  • f3d100b update logback-access evaluator examples
  • 51e3903 fix issues/753 for the second time
  • Additional commits viewable in compare view

Updates `ch.qos.logback:logback-core` from 1.5.16 to 1.5.17
Release notes

Sourced from ch.qos.logback:logback-core's releases.

Logback 1.5.17

2025-02-25 Release of logback version 1.5.17

• Fixed Jansi 2.4.0 color-coded output not working on Windows CMD.exe console when the default terminal application is set to "Windows Console Host". This problem was reported in issues/753 by Michael Lyubkin.

• Fixed race condition occurring in case MDC class is initialized while org.slf4j.LoggerFactory is initializing logback-classic's LoggerContext. When this race conditions occurs, the MDCAdapter instance used by MDC does not match the instance used by logback-classic. This issue was reported in SLF4J issues/450. While logback-classic version 1.5.17 remains compatible with SLF4J versions in the 2.0.x series, fixing this particular MDC issue requires SLF4J version 2.0.17.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit 10358724ed723b3745c010aa40cb02a2dfed4593 associated with the tag v_1.5.17. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits
  • 1035872 prepare release 1.5.17
  • 2e6984d bump to slf4j version 2.0.17
  • 1009952 use a new LoggerContert instance when running LogbackListenerTest. This shoul...
  • a3bb4b0 Merge branch 'master' of github.com:qos-ch/logback
  • b507297 Fixed race condition occurring in case MDC class is initialized while org.slf...
  • f5b3bc5 add warning about the deprecation of SerializedModelConfigurator if activated
  • 5bc0998 Update README.md
  • 5610c96 correct relocation address
  • f3d100b update logback-access evaluator examples
  • 51e3903 fix issues/753 for the second time
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e22cc8daef..3cc706e752 100644 --- a/pom.xml +++ b/pom.xml @@ -110,7 +110,7 @@ under the License. 2.31.0 5.15.2 3.49.0 - 1.5.16 + 1.5.17 none -Xdoclint:none From 21d82e8e6a54142fe0e79a96ddeac9cfccdeed3a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Mar 2025 09:14:45 +0900 Subject: [PATCH 102/408] MINOR: Bump com.google.api.grpc:proto-google-common-protos from 2.52.0 to 2.53.0 (#644) Bumps [com.google.api.grpc:proto-google-common-protos](https://github.com/googleapis/sdk-platform-java) from 2.52.0 to 2.53.0.
Release notes

Sourced from com.google.api.grpc:proto-google-common-protos's releases.

v2.53.0

2.53.0 (2025-02-10)

Features

  • enable DirectPath bound token in InstantiatingGrpcChannelProvider (#3572) (5080495)
  • Enable MTLS_S2A bound token by default for gRPC S2A enabled flows (#3591) (81e21f2)
  • migrate away from deprecated graal-sdk dependency to use nativeimage (#2706) (757801a)

Bug Fixes

  • Avoid creating message string prematurely for streaming calls (#3622) (f805e70)

Dependencies

  • update dependency com.google.code.gson:gson to v2.12.0 (#3595) (1f1b119)
  • update dependency com.google.code.gson:gson to v2.12.0 (#3596) (af62f53)
  • update dependency com.google.code.gson:gson to v2.12.1 (#3599) (18917ee)
  • update dependency com.google.code.gson:gson to v2.12.1 (#3600) (3f82836)
  • update dependency commons-codec:commons-codec to v1.18.0 (#3590) (cd46ba5)
  • update dependency io.netty:netty-tcnative-boringssl-static to v2.0.70.final (#3623) (a4d1f95)
  • update dependency lxml to v5.3.1 (#3624) (5407646)
  • update dependency net.bytebuddy:byte-buddy to v1.17.0 (#3582) (54d99e9)
  • update dependency org.checkerframework:checker-qual to v3.49.0 (#3604) (390cffa)
  • update dependency org.graalvm.sdk:nativeimage to v24.1.2 (#3597) (9d151c4)
  • update docker.io/library/maven:3.9.9-eclipse-temurin-11-alpine docker digest to 456f60c (#3607) (c2d2768)
  • update docker.io/library/maven:3.9.9-eclipse-temurin-11-alpine docker digest to d323c2b (#3601) (ed35c23)
  • update docker.io/library/python docker tag to v3.13.2 (#3615) (ba007c2)
  • update docker.io/library/python:3.13.1-alpine3.20 docker digest to 7788ec8 (#3586) (a24d1ba)
  • update google api dependencies (#3584) (08f2b7b)
  • update google auth library dependencies to v1.32.0 (#3611) (9436eb0)
  • update google auth library dependencies to v1.32.1 (#3618) (88c78e2)
  • update google http client dependencies to v1.46.1 (#3616) (2462105)
  • update googleapis/java-cloud-bom digest to 47ad868 (#3608) (2bcf9e0)
  • update googleapis/java-cloud-bom digest to 514a644 (#3602) (172d4da)
  • update googleapis/java-cloud-bom digest to 7752ecd (#3603) (06be924)
  • update netty dependencies to v4.1.117.final (#3581) (2734dc0)
  • update netty dependencies to v4.1.118.final (#3625) (16ff6bd)
  • update netty dependencies to v4.1.118.final (#3626) (316c425)
  • Update OpenTelemetry semantic convention packages in the shared dependencies (#3402) (0e69784)
  • update opentelemetry-java monorepo to v1.46.0 (#3585) (ac214be)
  • update opentelemetry-java monorepo to v1.47.0 (#3619) (66901df)
  • update repo-automation-bots digest to 35eff2c (#3609) (b962a01)
  • update repo-automation-bots digest to 3a68a9c (#3620) (1d79552)
Changelog

Sourced from com.google.api.grpc:proto-google-common-protos's changelog.

2.53.0 (2025-02-10)

Features

  • enable DirectPath bound token in InstantiatingGrpcChannelProvider (#3572) (5080495)
  • Enable MTLS_S2A bound token by default for gRPC S2A enabled flows (#3591) (81e21f2)
  • migrate away from deprecated graal-sdk dependency to use nativeimage (#2706) (757801a)

Bug Fixes

  • Avoid creating message string prematurely for streaming calls (#3622) (f805e70)

Dependencies

  • update dependency com.google.code.gson:gson to v2.12.0 (#3595) (1f1b119)
  • update dependency com.google.code.gson:gson to v2.12.0 (#3596) (af62f53)
  • update dependency com.google.code.gson:gson to v2.12.1 (#3599) (18917ee)
  • update dependency com.google.code.gson:gson to v2.12.1 (#3600) (3f82836)
  • update dependency commons-codec:commons-codec to v1.18.0 (#3590) (cd46ba5)
  • update dependency io.netty:netty-tcnative-boringssl-static to v2.0.70.final (#3623) (a4d1f95)
  • update dependency lxml to v5.3.1 (#3624) (5407646)
  • update dependency net.bytebuddy:byte-buddy to v1.17.0 (#3582) (54d99e9)
  • update dependency org.checkerframework:checker-qual to v3.49.0 (#3604) (390cffa)
  • update dependency org.graalvm.sdk:nativeimage to v24.1.2 (#3597) (9d151c4)
  • update docker.io/library/maven:3.9.9-eclipse-temurin-11-alpine docker digest to 456f60c (#3607) (c2d2768)
  • update docker.io/library/maven:3.9.9-eclipse-temurin-11-alpine docker digest to d323c2b (#3601) (ed35c23)
  • update docker.io/library/python docker tag to v3.13.2 (#3615) (ba007c2)
  • update docker.io/library/python:3.13.1-alpine3.20 docker digest to 7788ec8 (#3586) (a24d1ba)
  • update google api dependencies (#3584) (08f2b7b)
  • update google auth library dependencies to v1.32.0 (#3611) (9436eb0)
  • update google auth library dependencies to v1.32.1 (#3618) (88c78e2)
  • update google http client dependencies to v1.46.1 (#3616) (2462105)
  • update googleapis/java-cloud-bom digest to 47ad868 (#3608) (2bcf9e0)
  • update googleapis/java-cloud-bom digest to 514a644 (#3602) (172d4da)
  • update googleapis/java-cloud-bom digest to 7752ecd (#3603) (06be924)
  • update netty dependencies to v4.1.117.final (#3581) (2734dc0)
  • update netty dependencies to v4.1.118.final (#3625) (16ff6bd)
  • update netty dependencies to v4.1.118.final (#3626) (316c425)
  • Update OpenTelemetry semantic convention packages in the shared dependencies (#3402) (0e69784)
  • update opentelemetry-java monorepo to v1.46.0 (#3585) (ac214be)
  • update opentelemetry-java monorepo to v1.47.0 (#3619) (66901df)
  • update repo-automation-bots digest to 35eff2c (#3609) (b962a01)
  • update repo-automation-bots digest to 3a68a9c (#3620) (1d79552)
Commits
  • 0bc9cce chore(main): release 2.53.0 (#3594)
  • f805e70 fix: Avoid creating message string prematurely for streaming calls (#3622)
  • c11360e chore: adjust native-image config for spring-cloud-gcp (#3617)
  • 5080495 feat: enable DirectPath bound token in InstantiatingGrpcChannelProvider (#3572)
  • 316c425 deps: update netty dependencies to v4.1.118.final (#3626)
  • 16ff6bd deps: update netty dependencies to v4.1.118.final (#3625)
  • a4d1f95 deps: update dependency io.netty:netty-tcnative-boringssl-static to v2.0.70.f...
  • 88c78e2 deps: update google auth library dependencies to v1.32.1 (#3618)
  • 5407646 deps: update dependency lxml to v5.3.1 (#3624)
  • 1d79552 deps: update repo-automation-bots digest to 3a68a9c (#3620)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.api.grpc:proto-google-common-protos&package-manager=maven&previous-version=2.52.0&new-version=2.53.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml index b516e71bda..aab685abc9 100644 --- a/flight/flight-core/pom.xml +++ b/flight/flight-core/pom.xml @@ -134,7 +134,7 @@ under the License. com.google.api.grpc proto-google-common-protos - 2.52.0 + 2.53.0 test From a22a38845f1cecc2d5e96fa130f198d6da344861 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Mar 2025 20:03:40 -0500 Subject: [PATCH 103/408] MINOR: Bump dep.slf4j.version from 2.0.16 to 2.0.17 (#642) Bumps `dep.slf4j.version` from 2.0.16 to 2.0.17. Updates `org.slf4j:slf4j-api` from 2.0.16 to 2.0.17 Updates `org.slf4j:slf4j-jdk14` from 2.0.16 to 2.0.17 Updates `org.slf4j:jul-to-slf4j` from 2.0.16 to 2.0.17 Updates `org.slf4j:jcl-over-slf4j` from 2.0.16 to 2.0.17 Updates `org.slf4j:log4j-over-slf4j` from 2.0.16 to 2.0.17 Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3cc706e752..8d0307b330 100644 --- a/pom.xml +++ b/pom.xml @@ -94,7 +94,7 @@ under the License. ${project.build.directory}/generated-sources 1.9.0 5.12.0 - 2.0.16 + 2.0.17 33.4.0-jre 4.1.118.Final 1.70.0 From 82a5b714acc1ee0036b549f245374666bf9ab834 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Mar 2025 07:26:30 -0500 Subject: [PATCH 104/408] MINOR: Bump io.netty:netty-bom from 4.1.118.Final to 4.1.119.Final (#643) Bumps [io.netty:netty-bom](https://github.com/netty/netty) from 4.1.118.Final to 4.1.119.Final. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8d0307b330..fc725ca4f4 100644 --- a/pom.xml +++ b/pom.xml @@ -96,7 +96,7 @@ under the License. 5.12.0 2.0.17 33.4.0-jre - 4.1.118.Final + 4.1.119.Final 1.70.0 3.25.5 2.18.3 From 26e14f08f1c2e4672d9525e8ffdac7ca17970f7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Tue, 4 Mar 2025 15:29:59 +0100 Subject: [PATCH 105/408] MINOR: Upgrade to GitHub actions/cache 4.2.2 (#650) Upgrade to GitHub actions/cache 4.2.2 as previous versions are deprecated. --- .github/workflows/dev.yml | 2 +- .github/workflows/rc.yml | 8 ++++---- .github/workflows/test.yml | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 2ac230f635..80d334bb78 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -42,7 +42,7 @@ jobs: with: python-version: '3.x' - name: pre-commit (cache) - uses: actions/cache@v4 + uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 with: path: ~/.cache/pre-commit key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index f62b44bc23..5d37252a93 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -139,7 +139,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Cache - uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 with: path: .docker key: jni-linux-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} @@ -266,7 +266,7 @@ jobs: run: | echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} - name: Cache ccache - uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 with: path: ccache key: jni-macos-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} @@ -342,7 +342,7 @@ jobs: run: | echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} - name: Cache ccache - uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 with: path: ccache key: jni-windows-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} @@ -416,7 +416,7 @@ jobs: repository: apache/arrow-testing path: testing - name: Cache ~/.m2 - uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 with: path: ~/.m2 key: binaries-build-${{ hashFiles('**/*.java', '**/pom.xml') }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3ea5d8a4b6..2fd12a02bf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,7 +63,7 @@ jobs: fetch-depth: 0 submodules: recursive - name: Cache Docker Volumes - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 with: path: .docker key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('compose.yaml', '**/pom.xml', '**/*.java') }} @@ -180,7 +180,7 @@ jobs: run: | ci/scripts/util_free_space.sh - name: Cache Docker Volumes - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 with: path: .docker key: integration-conda-${{ hashFiles('cpp/**') }} From 5e71a2a67ebb215099f3d8d532df54f6cb1c537c Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Tue, 4 Mar 2025 09:42:57 -0800 Subject: [PATCH 106/408] MINOR: Add explicit exception when no more buffer can be read when loading buffers (#649) ## What's Changed When `VectorLoader` tries to load buffers (i.e., `loadBuffers`), it has detect on the error case that some buffers are not consumed. But another error that the number of buffers is less than expected is not handled for now. Once it is happened, users will get `java.util.NoSuchElementException` which is not easy to understand. This patch adds an explicit exception for such case. See more discussion at #648. --- .../src/main/java/org/apache/arrow/vector/VectorLoader.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java b/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java index ecd3fb9124..9b9a890346 100644 --- a/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java +++ b/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java @@ -122,6 +122,10 @@ private void loadBuffers( (int) (variadicBufferLayoutCount + TypeLayout.getTypeBufferCount(field.getType())); List ownBuffers = new ArrayList<>(bufferLayoutCount); for (int j = 0; j < bufferLayoutCount; j++) { + if (!buffers.hasNext()) { + throw new IllegalArgumentException( + "no more buffers for field " + field + ". Expected " + bufferLayoutCount); + } ArrowBuf nextBuf = buffers.next(); // for vectors without nulls, the buffer is empty, so there is no need to decompress it. ArrowBuf bufferToAdd = From cf8b2521ca43c862784b67d32af94861dea1d388 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Wed, 5 Mar 2025 01:53:12 +0100 Subject: [PATCH 107/408] MINOR: Upgrade to Spotless 2.44.3 and Google Java Format 1.17 (supporting JDK23) (#637) --- .../arrow/adapter/avro/AvroToArrowConfig.java | 1 + .../arrow/adapter/avro/AvroToArrowUtils.java | 65 +++++++++++-------- .../adapter/jdbc/JdbcParameterBinder.java | 2 +- .../arrow/adapter/jdbc/JdbcToArrowConfig.java | 1 + .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 2 +- .../adapter/jdbc/JdbcParameterBinderTest.java | 6 +- .../adapter/jdbc/UnreliableMetaDataTest.java | 5 +- bom/pom.xml | 2 +- .../arrow/c/BufferImportTypeVisitor.java | 6 +- .../java/org/apache/arrow/c/StreamTest.java | 5 +- .../compression/Lz4CompressionCodec.java | 4 +- .../compression/ZstdCompressionCodec.java | 7 +- .../TestArrowReaderWriterWithCompression.java | 2 +- .../org/apache/arrow/flight/FlightClient.java | 1 + .../arrow/flight/TestBasicOperation.java | 3 +- .../apache/arrow/flight/TestLargeMessage.java | 3 +- .../arrow/memory/AllocationManager.java | 3 +- .../apache/arrow/memory/BaseAllocator.java | 2 +- .../apache/arrow/memory/BufferAllocator.java | 3 +- .../apache/arrow/memory/util/MemoryUtil.java | 1 + .../org/apache/arrow/util/AutoCloseables.java | 2 +- pom.xml | 4 +- .../apache/arrow/vector/Decimal256Vector.java | 2 +- .../arrow/vector/compare/VectorVisitor.java | 3 +- .../dictionary/ListSubfieldEncoder.java | 2 +- .../dictionary/StructSubfieldEncoder.java | 4 +- .../org/apache/arrow/vector/table/Row.java | 2 + .../arrow/vector/TestDenseUnionVector.java | 2 +- .../arrow/vector/TestDictionaryVector.java | 30 ++++----- .../arrow/vector/TestFixedSizeListVector.java | 30 ++++----- .../TestIntervalMonthDayNanoVector.java | 10 +-- .../apache/arrow/vector/TestUnionVector.java | 5 +- .../arrow/vector/TestVectorSchemaRoot.java | 4 +- .../vector/ipc/MessageSerializerTest.java | 2 +- .../vector/ipc/TestArrowReaderWriter.java | 8 +-- 35 files changed, 127 insertions(+), 107 deletions(-) diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowConfig.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowConfig.java index 290d1a77d9..bd70c2b8ba 100644 --- a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowConfig.java +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowConfig.java @@ -25,6 +25,7 @@ public class AvroToArrowConfig { private final BufferAllocator allocator; + /** * The maximum rowCount to read each time when partially convert data. Default value is 1024 and * -1 means read all data into one vector. diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java index b39121cfd1..ed7642aabd 100644 --- a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java @@ -185,7 +185,7 @@ private static Consumer createConsumer( break; case STRING: arrowType = new ArrowType.Utf8(); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); + fieldType = new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroStringConsumer((VarCharVector) vector); break; @@ -195,7 +195,7 @@ private static Consumer createConsumer( arrowType = createDecimalArrowType((LogicalTypes.Decimal) logicalType); fieldType = new FieldType( - nullable, arrowType, /*dictionary=*/ null, getMetaData(schema, extProps)); + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, extProps)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroDecimalConsumer.FixedDecimalConsumer( @@ -204,7 +204,7 @@ private static Consumer createConsumer( arrowType = new ArrowType.FixedSizeBinary(schema.getFixedSize()); fieldType = new FieldType( - nullable, arrowType, /*dictionary=*/ null, getMetaData(schema, extProps)); + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, extProps)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroFixedConsumer((FixedSizeBinaryVector) vector, schema.getFixedSize()); } @@ -212,79 +212,88 @@ private static Consumer createConsumer( case INT: if (logicalType instanceof LogicalTypes.Date) { arrowType = new ArrowType.Date(DateUnit.DAY); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); + fieldType = + new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroDateConsumer((DateDayVector) vector); } else if (logicalType instanceof LogicalTypes.TimeMillis) { arrowType = new ArrowType.Time(TimeUnit.MILLISECOND, 32); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); + fieldType = + new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroTimeMillisConsumer((TimeMilliVector) vector); } else { - arrowType = new ArrowType.Int(32, /*isSigned=*/ true); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); + arrowType = new ArrowType.Int(32, /* isSigned= */ true); + fieldType = + new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroIntConsumer((IntVector) vector); } break; case BOOLEAN: arrowType = new ArrowType.Bool(); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); + fieldType = new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroBooleanConsumer((BitVector) vector); break; case LONG: if (logicalType instanceof LogicalTypes.TimeMicros) { arrowType = new ArrowType.Time(TimeUnit.MICROSECOND, 64); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); + fieldType = + new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroTimeMicroConsumer((TimeMicroVector) vector); } else if (logicalType instanceof LogicalTypes.TimestampMillis) { arrowType = new ArrowType.Timestamp(TimeUnit.MILLISECOND, null); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); + fieldType = + new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroTimestampMillisConsumer((TimeStampMilliVector) vector); } else if (logicalType instanceof LogicalTypes.TimestampMicros) { arrowType = new ArrowType.Timestamp(TimeUnit.MICROSECOND, null); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); + fieldType = + new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroTimestampMicrosConsumer((TimeStampMicroVector) vector); } else { - arrowType = new ArrowType.Int(64, /*isSigned=*/ true); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); + arrowType = new ArrowType.Int(64, /* isSigned= */ true); + fieldType = + new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroLongConsumer((BigIntVector) vector); } break; case FLOAT: arrowType = new ArrowType.FloatingPoint(SINGLE); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); + fieldType = new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroFloatConsumer((Float4Vector) vector); break; case DOUBLE: arrowType = new ArrowType.FloatingPoint(DOUBLE); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); + fieldType = new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroDoubleConsumer((Float8Vector) vector); break; case BYTES: if (logicalType instanceof LogicalTypes.Decimal) { arrowType = createDecimalArrowType((LogicalTypes.Decimal) logicalType); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); + fieldType = + new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroDecimalConsumer.BytesDecimalConsumer((DecimalVector) vector); } else { arrowType = new ArrowType.Binary(); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); + fieldType = + new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroBytesConsumer((VarBinaryVector) vector); } break; case NULL: arrowType = new ArrowType.Null(); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); - vector = fieldType.createNewSingleVector(name, allocator, /*schemaCallBack=*/ null); + fieldType = new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); + vector = fieldType.createNewSingleVector(name, allocator, /* schemaCallBack= */ null); consumer = new AvroNullConsumer((NullVector) vector); break; default: @@ -462,17 +471,17 @@ private static Field avroSchemaToField( case MAP: // MapVector internal struct field and key field should be non-nullable FieldType keyFieldType = - new FieldType(/*nullable=*/ false, new ArrowType.Utf8(), /*dictionary=*/ null); - Field keyField = new Field("key", keyFieldType, /*children=*/ null); + new FieldType(/* nullable= */ false, new ArrowType.Utf8(), /* dictionary= */ null); + Field keyField = new Field("key", keyFieldType, /* children= */ null); Field valueField = avroSchemaToField(schema.getValueType(), "value", config); FieldType structFieldType = - new FieldType(false, new ArrowType.Struct(), /*dictionary=*/ null); + new FieldType(false, new ArrowType.Struct(), /* dictionary= */ null); Field structField = new Field("internal", structFieldType, Arrays.asList(keyField, valueField)); children.add(structField); fieldType = - createFieldType(new ArrowType.Map(/*keysSorted=*/ false), schema, externalProps); + createFieldType(new ArrowType.Map(/* keysSorted= */ false), schema, externalProps); break; case RECORD: final Set skipFieldNames = config.getSkipFieldNames(); @@ -506,7 +515,7 @@ private static Field avroSchemaToField( indexType, schema, externalProps, - new DictionaryEncoding(current, /*ordered=*/ false, /*indexType=*/ indexType)); + new DictionaryEncoding(current, /* ordered= */ false, /* indexType= */ indexType)); break; case STRING: @@ -528,7 +537,7 @@ private static Field avroSchemaToField( } else if (logicalType instanceof LogicalTypes.TimeMillis) { intArrowType = new ArrowType.Time(TimeUnit.MILLISECOND, 32); } else { - intArrowType = new ArrowType.Int(32, /*isSigned=*/ true); + intArrowType = new ArrowType.Int(32, /* isSigned= */ true); } fieldType = createFieldType(intArrowType, schema, externalProps); break; @@ -544,7 +553,7 @@ private static Field avroSchemaToField( } else if (logicalType instanceof LogicalTypes.TimestampMicros) { longArrowType = new ArrowType.Timestamp(TimeUnit.MICROSECOND, null); } else { - longArrowType = new ArrowType.Int(64, /*isSigned=*/ true); + longArrowType = new ArrowType.Int(64, /* isSigned= */ true); } fieldType = createFieldType(longArrowType, schema, externalProps); break; @@ -806,7 +815,7 @@ private static Map createExternalProps(Schema schema) { private static FieldType createFieldType( ArrowType arrowType, Schema schema, Map externalProps) { - return createFieldType(arrowType, schema, externalProps, /*dictionary=*/ null); + return createFieldType(arrowType, schema, externalProps, /* dictionary= */ null); } private static FieldType createFieldType( @@ -816,7 +825,7 @@ private static FieldType createFieldType( DictionaryEncoding dictionary) { return new FieldType( - /*nullable=*/ false, arrowType, dictionary, getMetaData(schema, externalProps)); + /* nullable= */ false, arrowType, dictionary, getMetaData(schema, externalProps)); } private static String convertAliases(Set aliases) { diff --git a/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcParameterBinder.java b/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcParameterBinder.java index fd4721bcd9..d41dcb91bd 100644 --- a/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcParameterBinder.java +++ b/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcParameterBinder.java @@ -116,7 +116,7 @@ public static class Builder { /** Bind each column to the corresponding parameter in order. */ public Builder bindAll() { for (int i = 0; i < root.getFieldVectors().size(); i++) { - bind(/*parameterIndex=*/ i + 1, /*columnIndex=*/ i); + bind(/* parameterIndex= */ i + 1, /* columnIndex= */ i); } return this; } diff --git a/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfig.java b/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfig.java index 1bfcfc8fe0..2992bdb251 100644 --- a/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfig.java +++ b/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfig.java @@ -61,6 +61,7 @@ public final class JdbcToArrowConfig { private final Map schemaMetadata; private final Map> columnMetadataByColumnIndex; private final RoundingMode bigDecimalRoundingMode; + /** * The maximum rowCount to read each time when partially convert data. Default value is 1024 and * -1 means disable partial read. default is -1 which means disable partial read. Note that this diff --git a/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index aecb734a8b..1edb6261d6 100644 --- a/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -145,7 +145,7 @@ public static Schema jdbcToArrowSchema( final int scale = parameterMetaData.getScale(parameterCounter); final ArrowType arrowType = getArrowTypeFromJdbcType(new JdbcFieldInfo(jdbcDataType, precision, scale), calendar); - final FieldType fieldType = new FieldType(arrowIsNullable, arrowType, /*dictionary=*/ null); + final FieldType fieldType = new FieldType(arrowIsNullable, arrowType, /* dictionary= */ null); parameterFields.add(new Field(null, fieldType, null)); } diff --git a/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcParameterBinderTest.java b/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcParameterBinderTest.java index a05130f18e..2c9473f3c3 100644 --- a/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcParameterBinderTest.java +++ b/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcParameterBinderTest.java @@ -107,8 +107,8 @@ void bindOrder() throws SQLException { final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { final JdbcParameterBinder binder = JdbcParameterBinder.builder(statement, root) - .bind(/*parameterIndex=*/ 1, /*columnIndex=*/ 2) - .bind(/*parameterIndex=*/ 2, /*columnIndex=*/ 0) + .bind(/* parameterIndex= */ 1, /* columnIndex= */ 2) + .bind(/* parameterIndex= */ 2, /* columnIndex= */ 0) .build(); assertThat(binder.next()).isFalse(); @@ -166,7 +166,7 @@ void customBinder() throws SQLException { final JdbcParameterBinder binder = JdbcParameterBinder.builder(statement, root) .bind( - /*parameterIndex=*/ 1, + /* parameterIndex= */ 1, new ColumnBinder() { private final IntVector vector = (IntVector) root.getVector(0); diff --git a/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/UnreliableMetaDataTest.java b/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/UnreliableMetaDataTest.java index 4993a8b1ae..1708c8c12c 100644 --- a/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/UnreliableMetaDataTest.java +++ b/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/UnreliableMetaDataTest.java @@ -187,10 +187,11 @@ public void testIncorrectNullability(boolean reuseVectorSchemaRoot) throws Excep final Schema notNullSchema = new Schema( Collections.singletonList( - Field.notNullable(/*name=*/ null, new ArrowType.Int(32, true)))); + Field.notNullable(/* name= */ null, new ArrowType.Int(32, true)))); final Schema nullSchema = new Schema( - Collections.singletonList(Field.nullable(/*name=*/ null, new ArrowType.Int(32, true)))); + Collections.singletonList( + Field.nullable(/* name= */ null, new ArrowType.Int(32, true)))); try (final ResultSet rs = resultSetBuilder.build()) { JdbcToArrowConfig config = diff --git a/bom/pom.xml b/bom/pom.xml index 61441048ff..171d0bc5e9 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -203,7 +203,7 @@ under the License. com.diffplug.spotless spotless-maven-plugin - 2.30.0 + 2.44.3 org.codehaus.mojo diff --git a/c/src/main/java/org/apache/arrow/c/BufferImportTypeVisitor.java b/c/src/main/java/org/apache/arrow/c/BufferImportTypeVisitor.java index 2661c12cda..5ca398c9f9 100644 --- a/c/src/main/java/org/apache/arrow/c/BufferImportTypeVisitor.java +++ b/c/src/main/java/org/apache/arrow/c/BufferImportTypeVisitor.java @@ -136,7 +136,7 @@ private ArrowBuf maybeImportBitmap(ArrowType type) { if (buffers[0] == NULL) { return null; } - return importFixedBits(type, 0, /*bitsPerSlot=*/ 1); + return importFixedBits(type, 0, /* bitsPerSlot= */ 1); } @Override @@ -205,7 +205,7 @@ public List visit(ArrowType.FloatingPoint type) { switch (type.getPrecision()) { case HALF: return Arrays.asList( - maybeImportBitmap(type), importFixedBytes(type, 1, /*bytesPerSlot=*/ 2)); + maybeImportBitmap(type), importFixedBytes(type, 1, /* bytesPerSlot= */ 2)); case SINGLE: return Arrays.asList( maybeImportBitmap(type), importFixedBytes(type, 1, Float4Vector.TYPE_WIDTH)); @@ -333,7 +333,7 @@ public List visit(ArrowType.FixedSizeBinary type) { @Override public List visit(ArrowType.Bool type) { - return Arrays.asList(maybeImportBitmap(type), importFixedBits(type, 1, /*bitsPerSlot=*/ 1)); + return Arrays.asList(maybeImportBitmap(type), importFixedBits(type, 1, /* bitsPerSlot= */ 1)); } @Override diff --git a/c/src/test/java/org/apache/arrow/c/StreamTest.java b/c/src/test/java/org/apache/arrow/c/StreamTest.java index 95363fcc32..3dc370c424 100644 --- a/c/src/test/java/org/apache/arrow/c/StreamTest.java +++ b/c/src/test/java/org/apache/arrow/c/StreamTest.java @@ -229,7 +229,7 @@ public void roundtripDictionary() throws Exception { Collections.singletonList( new Field( "dict", - new FieldType(/*nullable=*/ true, indexType, encoding), + new FieldType(/* nullable= */ true, indexType, encoding), Collections.emptyList()))); final List batches = new ArrayList<>(); try (final CDataDictionaryProvider provider = new CDataDictionaryProvider(); @@ -362,7 +362,8 @@ void roundtrip(Schema schema, List batches) throws Exception { private static void assertVectorsEqual(FieldVector expected, FieldVector actual) { assertThat(actual.getField().getType()).isEqualTo(expected.getField().getType()); assertThat(actual.getValueCount()).isEqualTo(expected.getValueCount()); - final Range range = new Range(/*leftStart=*/ 0, /*rightStart=*/ 0, expected.getValueCount()); + final Range range = + new Range(/* leftStart= */ 0, /* rightStart= */ 0, expected.getValueCount()); assertThat(new RangeEqualsVisitor(expected, actual).rangeEquals(range)) .as("Vectors were not equal.\nExpected: %s\nGot: %s", expected, actual) .isTrue(); diff --git a/compression/src/main/java/org/apache/arrow/compression/Lz4CompressionCodec.java b/compression/src/main/java/org/apache/arrow/compression/Lz4CompressionCodec.java index edd52604bc..91cefc2a9e 100644 --- a/compression/src/main/java/org/apache/arrow/compression/Lz4CompressionCodec.java +++ b/compression/src/main/java/org/apache/arrow/compression/Lz4CompressionCodec.java @@ -41,7 +41,7 @@ protected ArrowBuf doCompress(BufferAllocator allocator, ArrowBuf uncompressedBu Integer.MAX_VALUE); byte[] inBytes = new byte[(int) uncompressedBuffer.writerIndex()]; - uncompressedBuffer.getBytes(/*index=*/ 0, inBytes); + uncompressedBuffer.getBytes(/* index= */ 0, inBytes); ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (InputStream in = new ByteArrayInputStream(inBytes); OutputStream out = new FramedLZ4CompressorOutputStream(baos)) { @@ -81,7 +81,7 @@ protected ArrowBuf doDecompress(BufferAllocator allocator, ArrowBuf compressedBu byte[] outBytes = out.toByteArray(); ArrowBuf decompressedBuffer = allocator.buffer(outBytes.length); - decompressedBuffer.setBytes(/*index=*/ 0, outBytes); + decompressedBuffer.setBytes(/* index= */ 0, outBytes); decompressedBuffer.writerIndex(decompressedLength); return decompressedBuffer; } diff --git a/compression/src/main/java/org/apache/arrow/compression/ZstdCompressionCodec.java b/compression/src/main/java/org/apache/arrow/compression/ZstdCompressionCodec.java index 46b1e223a4..290723608d 100644 --- a/compression/src/main/java/org/apache/arrow/compression/ZstdCompressionCodec.java +++ b/compression/src/main/java/org/apache/arrow/compression/ZstdCompressionCodec.java @@ -46,8 +46,8 @@ protected ArrowBuf doCompress(BufferAllocator allocator, ArrowBuf uncompressedBu compressedBuffer.memoryAddress() + CompressionUtil.SIZE_OF_UNCOMPRESSED_LENGTH, dstSize, /*src*/ uncompressedBuffer.memoryAddress(), - /*srcSize=*/ uncompressedBuffer.writerIndex(), - /*level=*/ this.compressionLevel); + /* srcSize= */ uncompressedBuffer.writerIndex(), + /* level= */ this.compressionLevel); if (Zstd.isError(bytesWritten)) { compressedBuffer.close(); throw new RuntimeException("Error compressing: " + Zstd.getErrorName(bytesWritten)); @@ -64,7 +64,8 @@ protected ArrowBuf doDecompress(BufferAllocator allocator, ArrowBuf compressedBu Zstd.decompressUnsafe( uncompressedBuffer.memoryAddress(), decompressedLength, - /*src=*/ compressedBuffer.memoryAddress() + CompressionUtil.SIZE_OF_UNCOMPRESSED_LENGTH, + /* src= */ compressedBuffer.memoryAddress() + + CompressionUtil.SIZE_OF_UNCOMPRESSED_LENGTH, compressedBuffer.writerIndex() - CompressionUtil.SIZE_OF_UNCOMPRESSED_LENGTH); if (Zstd.isError(decompressedSize)) { uncompressedBuffer.close(); diff --git a/compression/src/test/java/org/apache/arrow/compression/TestArrowReaderWriterWithCompression.java b/compression/src/test/java/org/apache/arrow/compression/TestArrowReaderWriterWithCompression.java index d7318e306c..03446e5316 100644 --- a/compression/src/test/java/org/apache/arrow/compression/TestArrowReaderWriterWithCompression.java +++ b/compression/src/test/java/org/apache/arrow/compression/TestArrowReaderWriterWithCompression.java @@ -141,7 +141,7 @@ private Dictionary createDictionary(VarCharVector dictionaryVector) { return new Dictionary( dictionaryVector, - new DictionaryEncoding(/*id=*/ 1L, /*ordered=*/ false, /*indexType=*/ null)); + new DictionaryEncoding(/* id= */ 1L, /* ordered= */ false, /* indexType= */ null)); } @Test diff --git a/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightClient.java b/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightClient.java index a15c3049aa..b12ae5ec8c 100644 --- a/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightClient.java +++ b/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightClient.java @@ -72,6 +72,7 @@ /** Client for Flight services. */ public class FlightClient implements AutoCloseable { private static final int PENDING_REQUESTS = 5; + /** * The maximum number of trace events to keep on the gRPC Channel. This value disables channel * tracing. diff --git a/flight/flight-core/src/test/java/org/apache/arrow/flight/TestBasicOperation.java b/flight/flight-core/src/test/java/org/apache/arrow/flight/TestBasicOperation.java index 5e818e6f5d..039bc14641 100644 --- a/flight/flight-core/src/test/java/org/apache/arrow/flight/TestBasicOperation.java +++ b/flight/flight-core/src/test/java/org/apache/arrow/flight/TestBasicOperation.java @@ -486,8 +486,7 @@ public void testProtobufSchemaCompatibility() throws Exception { // Should have no body buffers assertFalse(message.getBufs().iterator().hasNext()); final Flight.FlightData protobufData = - arrowMessageToProtobuf(marshaller, message) - .toBuilder() + arrowMessageToProtobuf(marshaller, message).toBuilder() .setDataBody(ByteString.EMPTY) .build(); assertEquals(0, protobufData.getDataBody().size()); diff --git a/flight/flight-core/src/test/java/org/apache/arrow/flight/TestLargeMessage.java b/flight/flight-core/src/test/java/org/apache/arrow/flight/TestLargeMessage.java index 9362e1f552..e5c5d707c2 100644 --- a/flight/flight-core/src/test/java/org/apache/arrow/flight/TestLargeMessage.java +++ b/flight/flight-core/src/test/java/org/apache/arrow/flight/TestLargeMessage.java @@ -134,7 +134,8 @@ public Runnable acceptPut( CallContext context, FlightStream flightStream, StreamListener ackStream) { return () -> { try (VectorSchemaRoot root = flightStream.getRoot()) { - while (flightStream.next()) {; + while (flightStream.next()) { + ; } } }; diff --git a/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationManager.java b/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationManager.java index e9dd8cb9d2..22f9202008 100644 --- a/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationManager.java +++ b/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationManager.java @@ -74,8 +74,7 @@ protected AllocationManager(BufferAllocator accountingAllocator) { this.owningLedger = associate(accountingAllocator, false); } - @Nullable - BufferLedger getOwningLedger() { + @Nullable BufferLedger getOwningLedger() { return owningLedger; } diff --git a/memory/memory-core/src/main/java/org/apache/arrow/memory/BaseAllocator.java b/memory/memory-core/src/main/java/org/apache/arrow/memory/BaseAllocator.java index 20a89d0b7b..72946e7fb7 100644 --- a/memory/memory-core/src/main/java/org/apache/arrow/memory/BaseAllocator.java +++ b/memory/memory-core/src/main/java/org/apache/arrow/memory/BaseAllocator.java @@ -272,7 +272,7 @@ public ArrowBuf wrapForeignAllocation(ForeignAllocation allocation) { final AllocationManager manager = new ForeignAllocationManager(this, allocation); final BufferLedger ledger = manager.associate(this); final ArrowBuf buf = - new ArrowBuf(ledger, /*bufferManager=*/ null, size, allocation.memoryAddress()); + new ArrowBuf(ledger, /* bufferManager= */ null, size, allocation.memoryAddress()); buf.writerIndex(size); listener.onAllocation(size); return buf; diff --git a/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferAllocator.java b/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferAllocator.java index a4db99f619..4f9d3c61c6 100644 --- a/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferAllocator.java +++ b/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferAllocator.java @@ -157,8 +157,7 @@ BufferAllocator newChildAllocator( * * @return parent allocator */ - @Nullable - BufferAllocator getParentAllocator(); + @Nullable BufferAllocator getParentAllocator(); /** * Returns the set of child allocators. diff --git a/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java b/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java index acf77547fb..91bd7cd905 100644 --- a/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java +++ b/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java @@ -32,6 +32,7 @@ public class MemoryUtil { org.slf4j.LoggerFactory.getLogger(MemoryUtil.class); private static final @Nullable Constructor DIRECT_BUFFER_CONSTRUCTOR; + /** The unsafe object from which to access the off-heap memory. */ private static final Unsafe UNSAFE; diff --git a/memory/memory-core/src/main/java/org/apache/arrow/util/AutoCloseables.java b/memory/memory-core/src/main/java/org/apache/arrow/util/AutoCloseables.java index 3796fb94bc..a39004a9d0 100644 --- a/memory/memory-core/src/main/java/org/apache/arrow/util/AutoCloseables.java +++ b/memory/memory-core/src/main/java/org/apache/arrow/util/AutoCloseables.java @@ -128,7 +128,7 @@ public Iterator iterator() { .flatMap( (Iterable i) -> StreamSupport.stream( - ((Iterable) i).spliterator(), /*parallel=*/ false)) + ((Iterable) i).spliterator(), /* parallel= */ false)) .iterator(); } }; diff --git a/pom.xml b/pom.xml index fc725ca4f4..16e1ae36f4 100644 --- a/pom.xml +++ b/pom.xml @@ -486,7 +486,7 @@ under the License. com.diffplug.spotless spotless-maven-plugin - 2.30.0 + 2.44.3 org.codehaus.mojo @@ -732,7 +732,7 @@ under the License. - 1.7 + 1.17.0 diff --git a/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java b/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java index 42ad741c85..90f67798dd 100644 --- a/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java +++ b/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java @@ -58,7 +58,7 @@ public final class Decimal256Vector extends BaseFixedWidthVector public Decimal256Vector(String name, BufferAllocator allocator, int precision, int scale) { this( name, - FieldType.nullable(new ArrowType.Decimal(precision, scale, /*bitWidth=*/ TYPE_WIDTH * 8)), + FieldType.nullable(new ArrowType.Decimal(precision, scale, /* bitWidth= */ TYPE_WIDTH * 8)), allocator); } diff --git a/vector/src/main/java/org/apache/arrow/vector/compare/VectorVisitor.java b/vector/src/main/java/org/apache/arrow/vector/compare/VectorVisitor.java index 989c57a0c9..a95cde5275 100644 --- a/vector/src/main/java/org/apache/arrow/vector/compare/VectorVisitor.java +++ b/vector/src/main/java/org/apache/arrow/vector/compare/VectorVisitor.java @@ -76,5 +76,6 @@ default OUT visit(LargeListViewVector left, IN value) { default OUT visit(RunEndEncodedVector left, IN value) { throw new UnsupportedOperationException( "VectorVisitor for LargeListViewVector is not supported."); - }; + } + ; } diff --git a/vector/src/main/java/org/apache/arrow/vector/dictionary/ListSubfieldEncoder.java b/vector/src/main/java/org/apache/arrow/vector/dictionary/ListSubfieldEncoder.java index 62b9628967..f56d819885 100644 --- a/vector/src/main/java/org/apache/arrow/vector/dictionary/ListSubfieldEncoder.java +++ b/vector/src/main/java/org/apache/arrow/vector/dictionary/ListSubfieldEncoder.java @@ -60,7 +60,7 @@ private static BaseListVector cloneVector(BaseListVector vector, BufferAllocator BaseListVector cloned = (BaseListVector) fieldType.createNewSingleVector( - vector.getField().getName(), allocator, /*schemaCallBack=*/ null); + vector.getField().getName(), allocator, /* schemaCallBack= */ null); final ArrowFieldNode fieldNode = new ArrowFieldNode(vector.getValueCount(), vector.getNullCount()); diff --git a/vector/src/main/java/org/apache/arrow/vector/dictionary/StructSubfieldEncoder.java b/vector/src/main/java/org/apache/arrow/vector/dictionary/StructSubfieldEncoder.java index dc25bc3268..8ff152fb1c 100644 --- a/vector/src/main/java/org/apache/arrow/vector/dictionary/StructSubfieldEncoder.java +++ b/vector/src/main/java/org/apache/arrow/vector/dictionary/StructSubfieldEncoder.java @@ -80,7 +80,7 @@ private static StructVector cloneVector(StructVector vector, BufferAllocator all StructVector cloned = (StructVector) fieldType.createNewSingleVector( - vector.getField().getName(), allocator, /*schemaCallback=*/ null); + vector.getField().getName(), allocator, /* schemaCallback= */ null); final ArrowFieldNode fieldNode = new ArrowFieldNode(vector.getValueCount(), vector.getNullCount()); @@ -120,7 +120,7 @@ public StructVector encode(StructVector vector, Map columnToDicti dictionary.getEncoding().getIndexType(), dictionary.getEncoding()); childrenFields.add( - new Field(childVector.getField().getName(), indexFieldType, /*children=*/ null)); + new Field(childVector.getField().getName(), indexFieldType, /* children= */ null)); } } diff --git a/vector/src/main/java/org/apache/arrow/vector/table/Row.java b/vector/src/main/java/org/apache/arrow/vector/table/Row.java index b89159b5ee..8d21f36627 100644 --- a/vector/src/main/java/org/apache/arrow/vector/table/Row.java +++ b/vector/src/main/java/org/apache/arrow/vector/table/Row.java @@ -118,8 +118,10 @@ public class Row implements Iterator { /** The table we're enumerating. */ protected final BaseTable table; + /** the current row number. */ protected int rowNumber = -1; + /** Indicates whether the next non-deleted row has been determined yet. */ private boolean nextRowSet; diff --git a/vector/src/test/java/org/apache/arrow/vector/TestDenseUnionVector.java b/vector/src/test/java/org/apache/arrow/vector/TestDenseUnionVector.java index 9cd89d57ff..9ac30730c4 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestDenseUnionVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestDenseUnionVector.java @@ -408,7 +408,7 @@ public void testGetFieldTypeInfo() throws Exception { final FieldType fieldType = new FieldType( - false, new ArrowType.Union(UnionMode.Dense, typeIds), /*dictionary=*/ null, metadata); + false, new ArrowType.Union(UnionMode.Dense, typeIds), /* dictionary= */ null, metadata); final Field field = new Field("union", fieldType, children); MinorType minorType = MinorType.DENSEUNION; diff --git a/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java b/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java index d65047efb1..0945919b91 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java @@ -1107,9 +1107,9 @@ public void testDictionaryUInt1() { new Dictionary( dictionaryVector, new DictionaryEncoding( - /*id=*/ 10L, - /*ordered=*/ false, - /*indexType=*/ new ArrowType.Int(/*bitWidth*/ 8, /*isSigned*/ false))); + /* id= */ 10L, + /* ordered= */ false, + /* indexType= */ new ArrowType.Int(/*bitWidth*/ 8, /*isSigned*/ false))); testDictionary(dictionary1, (vector, index) -> ((UInt1Vector) vector).get(index)); } } @@ -1122,9 +1122,9 @@ public void testDictionaryUInt2() { new Dictionary( dictionaryVector, new DictionaryEncoding( - /*id=*/ 20L, - /*ordered=*/ false, - /*indexType=*/ new ArrowType.Int(/*bitWidth=*/ 16, /*isSigned*/ false))); + /* id= */ 20L, + /* ordered= */ false, + /* indexType= */ new ArrowType.Int(/* bitWidth= */ 16, /*isSigned*/ false))); testDictionary(dictionary2, (vector, index) -> ((UInt2Vector) vector).get(index)); } } @@ -1137,9 +1137,9 @@ public void testDictionaryUInt4() { new Dictionary( dictionaryVector, new DictionaryEncoding( - /*id=*/ 30L, - /*ordered=*/ false, - /*indexType=*/ new ArrowType.Int(/*bitWidth=*/ 32, /*isSigned*/ false))); + /* id= */ 30L, + /* ordered= */ false, + /* indexType= */ new ArrowType.Int(/* bitWidth= */ 32, /*isSigned*/ false))); testDictionary(dictionary4, (vector, index) -> ((UInt4Vector) vector).get(index)); } } @@ -1152,9 +1152,9 @@ public void testDictionaryUInt8() { new Dictionary( dictionaryVector, new DictionaryEncoding( - /*id=*/ 40L, - /*ordered=*/ false, - /*indexType=*/ new ArrowType.Int(/*bitWidth=*/ 64, /*isSigned*/ false))); + /* id= */ 40L, + /* ordered= */ false, + /* indexType= */ new ArrowType.Int(/* bitWidth= */ 64, /*isSigned*/ false))); testDictionary(dictionary8, (vector, index) -> (int) ((UInt8Vector) vector).get(index)); } } @@ -1174,9 +1174,9 @@ public void testDictionaryUIntOverflow() { new Dictionary( dictionaryVector, new DictionaryEncoding( - /*id=*/ 10L, - /*ordered=*/ false, - /*indexType=*/ new ArrowType.Int(/*bitWidth=*/ 8, /*isSigned*/ false))); + /* id= */ 10L, + /* ordered= */ false, + /* indexType= */ new ArrowType.Int(/* bitWidth= */ 8, /*isSigned*/ false))); try (VarCharVector vector = new VarCharVector("vector", allocator)) { setVector(vector, "255"); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java index f582406de6..73a88b3a1e 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java @@ -61,7 +61,7 @@ public void terminate() throws Exception { @Test public void testIntType() { - try (FixedSizeListVector vector = FixedSizeListVector.empty("list", /*size=*/ 2, allocator)) { + try (FixedSizeListVector vector = FixedSizeListVector.empty("list", /* size= */ 2, allocator)) { IntVector nested = (IntVector) vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector(); @@ -90,7 +90,7 @@ public void testIntType() { @Test public void testFloatTypeNullable() { - try (FixedSizeListVector vector = FixedSizeListVector.empty("list", /*size=*/ 2, allocator)) { + try (FixedSizeListVector vector = FixedSizeListVector.empty("list", /* size= */ 2, allocator)) { Float4Vector nested = (Float4Vector) vector.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector(); @@ -275,7 +275,7 @@ public void testTransferEmptyVector() throws Exception { @Test public void testConsistentChildName() throws Exception { try (FixedSizeListVector listVector = - FixedSizeListVector.empty("sourceVector", /*size=*/ 2, allocator)) { + FixedSizeListVector.empty("sourceVector", /* size= */ 2, allocator)) { String emptyListStr = listVector.getField().toString(); assertTrue(emptyListStr.contains(ListVector.DATA_VECTOR_NAME)); @@ -292,7 +292,7 @@ public void testUnionFixedSizeListWriterWithNulls() throws Exception { * Read and verify */ try (final FixedSizeListVector vector = - FixedSizeListVector.empty("vector", /*size=*/ 3, allocator)) { + FixedSizeListVector.empty("vector", /* size= */ 3, allocator)) { UnionFixedSizeListWriter writer = vector.getWriter(); writer.allocate(); @@ -321,7 +321,7 @@ public void testUnionFixedSizeListWriterWithNulls() throws Exception { @Test public void testUnionFixedSizeListWriter() throws Exception { try (final FixedSizeListVector vector1 = - FixedSizeListVector.empty("vector", /*size=*/ 3, allocator)) { + FixedSizeListVector.empty("vector", /* size= */ 3, allocator)) { UnionFixedSizeListWriter writer1 = vector1.getWriter(); writer1.allocate(); @@ -350,7 +350,7 @@ public void testUnionFixedSizeListWriter() throws Exception { @Test public void testWriteDecimal() throws Exception { try (final FixedSizeListVector vector = - FixedSizeListVector.empty("vector", /*size=*/ 3, allocator)) { + FixedSizeListVector.empty("vector", /* size= */ 3, allocator)) { UnionFixedSizeListWriter writer = vector.getWriter(); writer.allocate(); @@ -379,7 +379,7 @@ public void testWriteDecimal() throws Exception { @Test public void testDecimalIndexCheck() throws Exception { try (final FixedSizeListVector vector = - FixedSizeListVector.empty("vector", /*size=*/ 3, allocator)) { + FixedSizeListVector.empty("vector", /* size= */ 3, allocator)) { UnionFixedSizeListWriter writer = vector.getWriter(); writer.allocate(); @@ -405,7 +405,7 @@ public void testWriteIllegalData() throws Exception { IllegalStateException.class, () -> { try (final FixedSizeListVector vector1 = - FixedSizeListVector.empty("vector", /*size=*/ 3, allocator)) { + FixedSizeListVector.empty("vector", /* size= */ 3, allocator)) { UnionFixedSizeListWriter writer1 = vector1.getWriter(); writer1.allocate(); @@ -430,7 +430,7 @@ public void testWriteIllegalData() throws Exception { @Test public void testSplitAndTransfer() throws Exception { try (final FixedSizeListVector vector1 = - FixedSizeListVector.empty("vector", /*size=*/ 3, allocator)) { + FixedSizeListVector.empty("vector", /* size= */ 3, allocator)) { UnionFixedSizeListWriter writer1 = vector1.getWriter(); writer1.allocate(); @@ -462,7 +462,7 @@ public void testSplitAndTransfer() throws Exception { @Test public void testZeroWidthVector() { try (final FixedSizeListVector vector1 = - FixedSizeListVector.empty("vector", /*size=*/ 0, allocator)) { + FixedSizeListVector.empty("vector", /* size= */ 0, allocator)) { UnionFixedSizeListWriter writer1 = vector1.getWriter(); writer1.allocate(); @@ -494,7 +494,7 @@ public void testZeroWidthVector() { @Test public void testVectorWithNulls() { try (final FixedSizeListVector vector1 = - FixedSizeListVector.empty("vector", /*size=*/ 4, allocator)) { + FixedSizeListVector.empty("vector", /* size= */ 4, allocator)) { UnionFixedSizeListWriter writer1 = vector1.getWriter(); writer1.allocate(); @@ -527,7 +527,7 @@ public void testVectorWithNulls() { @Test public void testWriteVarCharHelpers() throws Exception { try (final FixedSizeListVector vector = - FixedSizeListVector.empty("vector", /*size=*/ 4, allocator)) { + FixedSizeListVector.empty("vector", /* size= */ 4, allocator)) { UnionFixedSizeListWriter writer = vector.getWriter(); writer.allocate(); @@ -547,7 +547,7 @@ public void testWriteVarCharHelpers() throws Exception { @Test public void testWriteLargeVarCharHelpers() throws Exception { try (final FixedSizeListVector vector = - FixedSizeListVector.empty("vector", /*size=*/ 4, allocator)) { + FixedSizeListVector.empty("vector", /* size= */ 4, allocator)) { UnionFixedSizeListWriter writer = vector.getWriter(); writer.allocate(); @@ -567,7 +567,7 @@ public void testWriteLargeVarCharHelpers() throws Exception { @Test public void testWriteVarBinaryHelpers() throws Exception { try (final FixedSizeListVector vector = - FixedSizeListVector.empty("vector", /*size=*/ 4, allocator)) { + FixedSizeListVector.empty("vector", /* size= */ 4, allocator)) { UnionFixedSizeListWriter writer = vector.getWriter(); writer.allocate(); @@ -599,7 +599,7 @@ public void testWriteVarBinaryHelpers() throws Exception { @Test public void testWriteLargeVarBinaryHelpers() throws Exception { try (final FixedSizeListVector vector = - FixedSizeListVector.empty("vector", /*size=*/ 4, allocator)) { + FixedSizeListVector.empty("vector", /* size= */ 4, allocator)) { UnionFixedSizeListWriter writer = vector.getWriter(); writer.allocate(); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestIntervalMonthDayNanoVector.java b/vector/src/test/java/org/apache/arrow/vector/TestIntervalMonthDayNanoVector.java index 2b39db3cd4..b576c145fe 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestIntervalMonthDayNanoVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestIntervalMonthDayNanoVector.java @@ -47,7 +47,7 @@ public void terminate() throws Exception { @Test public void testBasics() { try (final IntervalMonthDayNanoVector vector = - new IntervalMonthDayNanoVector(/*name=*/ "", allocator)) { + new IntervalMonthDayNanoVector(/* name= */ "", allocator)) { int valueCount = 100; vector.setInitialCapacity(valueCount); vector.allocateNew(); @@ -61,12 +61,12 @@ public void testBasics() { holder.days = Integer.MIN_VALUE; holder.nanoseconds = Long.MIN_VALUE; - vector.set(0, /*months=*/ 1, /*days=*/ 2, /*nanoseconds=*/ -2); - vector.setSafe(2, /*months=*/ 1, /*days=*/ 2, /*nanoseconds=*/ -3); - vector.setSafe(/*index=*/ 4, nullableHolder); + vector.set(0, /* months= */ 1, /* days= */ 2, /* nanoseconds= */ -2); + vector.setSafe(2, /* months= */ 1, /* days= */ 2, /* nanoseconds= */ -3); + vector.setSafe(/* index= */ 4, nullableHolder); vector.set(3, holder); nullableHolder.isSet = 0; - vector.setSafe(/*index=*/ 5, nullableHolder); + vector.setSafe(/* index= */ 5, nullableHolder); vector.setValueCount(5); assertEquals("P1M2D PT-0.000000002S ", vector.getAsStringBuilder(0).toString()); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java b/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java index 6c05073c16..40c05f9b11 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java @@ -395,7 +395,10 @@ public void testGetFieldTypeInfo() throws Exception { final FieldType fieldType = new FieldType( - false, new ArrowType.Union(UnionMode.Sparse, typeIds), /*dictionary=*/ null, metadata); + false, + new ArrowType.Union(UnionMode.Sparse, typeIds), + /* dictionary= */ null, + metadata); final Field field = new Field("union", fieldType, children); MinorType minorType = MinorType.UNION; diff --git a/vector/src/test/java/org/apache/arrow/vector/TestVectorSchemaRoot.java b/vector/src/test/java/org/apache/arrow/vector/TestVectorSchemaRoot.java index 50f61d311e..c121d94892 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestVectorSchemaRoot.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestVectorSchemaRoot.java @@ -91,8 +91,8 @@ private void checkCount(BitVector vec1, IntVector vec2, VectorSchemaRoot vsr, in } private VectorSchemaRoot createBatch() { - FieldType varCharType = new FieldType(true, new ArrowType.Utf8(), /*dictionary=*/ null); - FieldType listType = new FieldType(true, new ArrowType.List(), /*dictionary=*/ null); + FieldType varCharType = new FieldType(true, new ArrowType.Utf8(), /* dictionary= */ null); + FieldType listType = new FieldType(true, new ArrowType.List(), /* dictionary= */ null); // create the schema List schemaFields = new ArrayList<>(); diff --git a/vector/src/test/java/org/apache/arrow/vector/ipc/MessageSerializerTest.java b/vector/src/test/java/org/apache/arrow/vector/ipc/MessageSerializerTest.java index 0a41b6c599..b529ca645a 100644 --- a/vector/src/test/java/org/apache/arrow/vector/ipc/MessageSerializerTest.java +++ b/vector/src/test/java/org/apache/arrow/vector/ipc/MessageSerializerTest.java @@ -208,7 +208,7 @@ public void testSerializeRecordBatchV5() throws Exception { { byte[] validBytes = out.toByteArray(); - byte[] missingBytes = Arrays.copyOfRange(validBytes, /*from=*/ 0, validBytes.length - 1); + byte[] missingBytes = Arrays.copyOfRange(validBytes, /* from= */ 0, validBytes.length - 1); ByteArrayInputStream in = new ByteArrayInputStream(missingBytes); ReadChannel channel = new ReadChannel(Channels.newChannel(in)); diff --git a/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java b/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java index 74ff95d41d..dc3613df72 100644 --- a/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java @@ -144,19 +144,19 @@ public void init() { dictionary1 = new Dictionary( dictionaryVector1, - new DictionaryEncoding(/*id=*/ 1L, /*ordered=*/ false, /*indexType=*/ null)); + new DictionaryEncoding(/* id= */ 1L, /* ordered= */ false, /* indexType= */ null)); dictionary2 = new Dictionary( dictionaryVector2, - new DictionaryEncoding(/*id=*/ 2L, /*ordered=*/ false, /*indexType=*/ null)); + new DictionaryEncoding(/* id= */ 2L, /* ordered= */ false, /* indexType= */ null)); dictionary3 = new Dictionary( dictionaryVector3, - new DictionaryEncoding(/*id=*/ 1L, /*ordered=*/ false, /*indexType=*/ null)); + new DictionaryEncoding(/* id= */ 1L, /* ordered= */ false, /* indexType= */ null)); dictionary4 = new Dictionary( dictionaryVector4, - new DictionaryEncoding(/*id=*/ 3L, /*ordered=*/ false, /*indexType=*/ null)); + new DictionaryEncoding(/* id= */ 3L, /* ordered= */ false, /* indexType= */ null)); } @AfterEach From e8d86a1f18230d3f85a2c96ec1ba906563238e3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Thu, 6 Mar 2025 01:48:20 +0100 Subject: [PATCH 108/408] MINOR: Update versions in LICENSE/NOTICE (#652) Align versions in `LICENSE`/`NOTICE` in regards of the recent dependency updates. --- .../flight-integration-tests/src/shade/LICENSE.txt | 8 ++++---- .../flight-integration-tests/src/shade/NOTICE.txt | 2 +- .../flight-sql-jdbc-driver/src/shade/LICENSE.txt | 14 +++++++------- flight/flight-sql-jdbc-driver/src/shade/NOTICE.txt | 2 +- tools/src/shade/LICENSE.txt | 10 +++++----- tools/src/shade/NOTICE.txt | 2 +- vector/src/shade/LICENSE.txt | 2 +- 7 files changed, 20 insertions(+), 20 deletions(-) diff --git a/flight/flight-integration-tests/src/shade/LICENSE.txt b/flight/flight-integration-tests/src/shade/LICENSE.txt index 499a3993fd..1d49e4e670 100644 --- a/flight/flight-integration-tests/src/shade/LICENSE.txt +++ b/flight/flight-integration-tests/src/shade/LICENSE.txt @@ -203,7 +203,7 @@ -------------------------------------------------------------------------------- -This binary artifact contains Jackson 2.18.2. +This binary artifact contains Jackson 2.18.3. Home page: https://github.com/FasterXML/jackson License: https://www.apache.org/licenses/LICENSE-2.0 @@ -218,14 +218,14 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- -This binary artifact contains Google Flatbuffers 25.1.24. +This binary artifact contains Google Flatbuffers 25.2.10. Home page: https://flatbuffers.dev/ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- -This binary artifact contains Netty 4.1.117.Final. +This binary artifact contains Netty 4.1.119.Final. Copyright: Copyright 2014 The Netty Project Home page: https://netty.io/ @@ -1097,7 +1097,7 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- -This binary artifact contains Slf4j 2.0.16. +This binary artifact contains Slf4j 2.0.17. Copyright: Copyright (c) 2004-2022 QOS.ch Sarl (Switzerland) Home page: http://www.slf4j.org/ diff --git a/flight/flight-integration-tests/src/shade/NOTICE.txt b/flight/flight-integration-tests/src/shade/NOTICE.txt index f8cd4979c0..6ec75cd470 100644 --- a/flight/flight-integration-tests/src/shade/NOTICE.txt +++ b/flight/flight-integration-tests/src/shade/NOTICE.txt @@ -6,7 +6,7 @@ The Apache Software Foundation (http://www.apache.org/). --------------------------------------------------------- -This product includes Netty 4.1.117.Final, with the following in its NOTICE: +This product includes Netty 4.1.119.Final, with the following in its NOTICE: | The Netty Project | ================= diff --git a/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt b/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt index 3363121bb9..fa445fb055 100644 --- a/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt +++ b/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt @@ -211,7 +211,7 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- -This binary artifact contains Netty 4.1.117.Final. +This binary artifact contains Netty 4.1.119.Final. Copyright: Copyright 2014 The Netty Project Home page: https://netty.io/ @@ -234,7 +234,7 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- -This binary artifact contains Protobuf 3.25.3. +This binary artifact contains Protobuf 3.25.5. Copyright: Copyright 2008 Google Inc. All rights reserved. Home page: https://protobuf.dev/ @@ -276,7 +276,7 @@ License text: -------------------------------------------------------------------------------- -This binary artifact contains Jackson 2.18.2. +This binary artifact contains Jackson 2.18.3. Home page: https://github.com/FasterXML/jackson License: https://www.apache.org/licenses/LICENSE-2.0 @@ -291,14 +291,14 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- -This binary artifact contains Google Flatbuffers 25.1.24. +This binary artifact contains Google Flatbuffers 25.2.10. Home page: https://flatbuffers.dev/ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- -This binary artifact contains Slf4j 2.0.16. +This binary artifact contains Slf4j 2.0.17. Copyright: Copyright (c) 2004-2022 QOS.ch Sarl (Switzerland) Home page: http://www.slf4j.org/ @@ -329,7 +329,7 @@ License text: -------------------------------------------------------------------------------- -This binary artifact contains Apache Calcite Avatica 1.25.0. +This binary artifact contains Apache Calcite Avatica 1.26.0. Copyright: Copyright 2012-2024 The Apache Software Foundation Home page: https://calcite.apache.org/avatica/ @@ -345,7 +345,7 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- -This binary artifact contains Bouncycastle 1.79. +This binary artifact contains Bouncycastle 1.80. Copyright: Copyright (c) 2000-2024 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org). Home page: https://www.bouncycastle.org/ diff --git a/flight/flight-sql-jdbc-driver/src/shade/NOTICE.txt b/flight/flight-sql-jdbc-driver/src/shade/NOTICE.txt index fcc90b2ed8..0e833bb1c8 100644 --- a/flight/flight-sql-jdbc-driver/src/shade/NOTICE.txt +++ b/flight/flight-sql-jdbc-driver/src/shade/NOTICE.txt @@ -73,7 +73,7 @@ This product includes gRPC 1.70.0, with the following in its NOTICE: --------------------------------------------------- -This product includes Netty 4.1.117.Final, with the following in its NOTICE: +This product includes Netty 4.1.119.Final, with the following in its NOTICE: | The Netty Project | ================= diff --git a/tools/src/shade/LICENSE.txt b/tools/src/shade/LICENSE.txt index 61fbdf256f..2130dc4bf7 100644 --- a/tools/src/shade/LICENSE.txt +++ b/tools/src/shade/LICENSE.txt @@ -203,7 +203,7 @@ -------------------------------------------------------------------------------- -This binary artifact contains Netty 4.1.117.Final. +This binary artifact contains Netty 4.1.119.Final. Copyright: Copyright 2014 The Netty Project Home page: https://netty.io/ @@ -211,7 +211,7 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- -This binary artifact contains Jackson 2.18.2. +This binary artifact contains Jackson 2.18.3. Home page: https://github.com/FasterXML/jackson License: https://www.apache.org/licenses/LICENSE-2.0 @@ -250,7 +250,7 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- -This binary artifact contains Luben Zstd 1.5.6-9. +This binary artifact contains Luben Zstd 1.5.7-1. Copyright: Copyright (c) 2015-present, Luben Karavelov/ All rights reserved. Home page: https://github.com/luben/zstd-jni @@ -294,14 +294,14 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- -This binary artifact contains Google Flatbuffers 25.1.24. +This binary artifact contains Google Flatbuffers 25.2.10. Home page: https://flatbuffers.dev/ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- -This binary artifact contains Slf4j 2.0.16. +This binary artifact contains Slf4j 2.0.17. Copyright: Copyright (c) 2004-2022 QOS.ch Sarl (Switzerland) Home page: http://www.slf4j.org/ diff --git a/tools/src/shade/NOTICE.txt b/tools/src/shade/NOTICE.txt index 7c5b36d714..1f709a5336 100644 --- a/tools/src/shade/NOTICE.txt +++ b/tools/src/shade/NOTICE.txt @@ -6,7 +6,7 @@ The Apache Software Foundation (http://www.apache.org/). --------------------------------------------------- -This product includes Netty 4.1.117.Final, with the following in its NOTICE: +This product includes Netty 4.1.119.Final, with the following in its NOTICE: | The Netty Project | ================= diff --git a/vector/src/shade/LICENSE.txt b/vector/src/shade/LICENSE.txt index d14211db2b..147a99ee2f 100644 --- a/vector/src/shade/LICENSE.txt +++ b/vector/src/shade/LICENSE.txt @@ -203,7 +203,7 @@ -------------------------------------------------------------------------------- -This binary artifact contains Google Flatbuffers 25.1.24. +This binary artifact contains Google Flatbuffers 25.2.10. Home page: https://flatbuffers.dev/ License: https://www.apache.org/licenses/LICENSE-2.0 From d304da5f872615346557e947ca5b04e9e23e8b87 Mon Sep 17 00:00:00 2001 From: lriggs Date: Wed, 5 Mar 2025 17:09:10 -0800 Subject: [PATCH 109/408] GH-653: Nullify fieldReader when invalidating parent object (#654) ## What's Changed Set the fieldReader object to null when invalidating the parent reader. This will prevent odd behavior if the object is reused. Closes #653 --- .../main/java/org/apache/arrow/vector/complex/ListVector.java | 1 + 1 file changed, 1 insertion(+) diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 76682c28fe..3daeb6d77b 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -769,6 +769,7 @@ public UnionVector promoteToUnion() { } protected void invalidateReader() { + fieldReader = null; reader = null; } From 8d3fe93d493188f0d3df59028cfd852be2085820 Mon Sep 17 00:00:00 2001 From: David Li Date: Fri, 7 Mar 2025 07:25:49 -0500 Subject: [PATCH 110/408] GH-81: [Flight] Expose gRPC in Flight client builder (#660) ## What's Changed Expose the internal gRPC channel builder in the FlightClient so that applications can build on top of it without having to replicate all the special Flight logic. This makes it much easier to apply some gRPC tweaks without duplicating Arrow code. Closes #81. --- .../org/apache/arrow/flight/FlightClient.java | 158 ++---------- .../apache/arrow/flight/FlightGrpcUtils.java | 14 ++ .../arrow/flight/grpc/NettyClientBuilder.java | 232 ++++++++++++++++++ 3 files changed, 267 insertions(+), 137 deletions(-) create mode 100644 flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/NettyClientBuilder.java diff --git a/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightClient.java b/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightClient.java index b12ae5ec8c..fd6e498d13 100644 --- a/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightClient.java +++ b/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightClient.java @@ -23,19 +23,13 @@ import io.grpc.ManagedChannel; import io.grpc.MethodDescriptor; import io.grpc.StatusRuntimeException; -import io.grpc.netty.GrpcSslContexts; import io.grpc.netty.NettyChannelBuilder; import io.grpc.stub.ClientCallStreamObserver; import io.grpc.stub.ClientCalls; import io.grpc.stub.ClientResponseObserver; import io.grpc.stub.StreamObserver; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.ServerChannel; -import io.netty.handler.ssl.SslContextBuilder; -import io.netty.handler.ssl.util.InsecureTrustManagerFactory; import java.io.IOException; import java.io.InputStream; -import java.lang.reflect.InvocationTargetException; import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -45,7 +39,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.function.BooleanSupplier; -import javax.net.ssl.SSLException; import org.apache.arrow.flight.FlightProducer.StreamListener; import org.apache.arrow.flight.auth.BasicClientAuthHandler; import org.apache.arrow.flight.auth.ClientAuthHandler; @@ -57,6 +50,7 @@ import org.apache.arrow.flight.auth2.ClientIncomingAuthHeaderMiddleware; import org.apache.arrow.flight.grpc.ClientInterceptorAdapter; import org.apache.arrow.flight.grpc.CredentialCallOption; +import org.apache.arrow.flight.grpc.NettyClientBuilder; import org.apache.arrow.flight.grpc.StatusUtils; import org.apache.arrow.flight.impl.Flight; import org.apache.arrow.flight.impl.Flight.Empty; @@ -73,12 +67,6 @@ public class FlightClient implements AutoCloseable { private static final int PENDING_REQUESTS = 5; - /** - * The maximum number of trace events to keep on the gRPC Channel. This value disables channel - * tracing. - */ - private static final int MAX_CHANNEL_TRACE_EVENTS = 0; - private final BufferAllocator allocator; private final ManagedChannel channel; @@ -97,11 +85,12 @@ public class FlightClient implements AutoCloseable { List middleware) { this.allocator = incomingAllocator.newChildAllocator("flight-client", 0, Long.MAX_VALUE); this.channel = channel; - this.middleware = middleware; + // We need a mutable copy (shared between this class and ClientInterceptorAdapter) + this.middleware = new ArrayList<>(middleware); final ClientInterceptor[] interceptors; interceptors = - new ClientInterceptor[] {authInterceptor, new ClientInterceptorAdapter(middleware)}; + new ClientInterceptor[] {authInterceptor, new ClientInterceptorAdapter(this.middleware)}; // Create a channel with interceptors pre-applied for DoGet and DoPut Channel interceptedChannel = ClientInterceptors.intercept(channel, interceptors); @@ -772,176 +761,71 @@ public static Builder builder(BufferAllocator allocator, Location location) { /** A builder for Flight clients. */ public static final class Builder { - private BufferAllocator allocator; - private Location location; - private boolean forceTls = false; - private int maxInboundMessageSize = FlightServer.MAX_GRPC_MESSAGE_SIZE; - private InputStream trustedCertificates = null; - private InputStream clientCertificate = null; - private InputStream clientKey = null; - private String overrideHostname = null; - private List middleware = new ArrayList<>(); - private boolean verifyServer = true; - - private Builder() {} + private final NettyClientBuilder builder; + + private Builder() { + this.builder = new NettyClientBuilder(); + } private Builder(BufferAllocator allocator, Location location) { - this.allocator = Preconditions.checkNotNull(allocator); - this.location = Preconditions.checkNotNull(location); + this.builder = new NettyClientBuilder(allocator, location); } /** Force the client to connect over TLS. */ public Builder useTls() { - this.forceTls = true; + builder.useTls(); return this; } /** Override the hostname checked for TLS. Use with caution in production. */ public Builder overrideHostname(final String hostname) { - this.overrideHostname = hostname; + builder.overrideHostname(hostname); return this; } /** Set the maximum inbound message size. */ public Builder maxInboundMessageSize(int maxSize) { - Preconditions.checkArgument(maxSize > 0); - this.maxInboundMessageSize = maxSize; + builder.maxInboundMessageSize(maxSize); return this; } /** Set the trusted TLS certificates. */ public Builder trustedCertificates(final InputStream stream) { - this.trustedCertificates = Preconditions.checkNotNull(stream); + builder.trustedCertificates(stream); return this; } /** Set the trusted TLS certificates. */ public Builder clientCertificate( final InputStream clientCertificate, final InputStream clientKey) { - Preconditions.checkNotNull(clientKey); - this.clientCertificate = Preconditions.checkNotNull(clientCertificate); - this.clientKey = Preconditions.checkNotNull(clientKey); + builder.clientCertificate(clientCertificate, clientKey); return this; } public Builder allocator(BufferAllocator allocator) { - this.allocator = Preconditions.checkNotNull(allocator); + builder.allocator(allocator); return this; } public Builder location(Location location) { - this.location = Preconditions.checkNotNull(location); + builder.location(location); return this; } public Builder intercept(FlightClientMiddleware.Factory factory) { - middleware.add(factory); + builder.intercept(factory); return this; } public Builder verifyServer(boolean verifyServer) { - this.verifyServer = verifyServer; + builder.verifyServer(verifyServer); return this; } /** Create the client from this builder. */ public FlightClient build() { - final NettyChannelBuilder builder; - - switch (location.getUri().getScheme()) { - case LocationSchemes.GRPC: - case LocationSchemes.GRPC_INSECURE: - case LocationSchemes.GRPC_TLS: - { - builder = NettyChannelBuilder.forAddress(location.toSocketAddress()); - break; - } - case LocationSchemes.GRPC_DOMAIN_SOCKET: - { - // The implementation is platform-specific, so we have to find the classes at runtime - builder = NettyChannelBuilder.forAddress(location.toSocketAddress()); - try { - try { - // Linux - builder.channelType( - Class.forName("io.netty.channel.epoll.EpollDomainSocketChannel") - .asSubclass(ServerChannel.class)); - final EventLoopGroup elg = - Class.forName("io.netty.channel.epoll.EpollEventLoopGroup") - .asSubclass(EventLoopGroup.class) - .getDeclaredConstructor() - .newInstance(); - builder.eventLoopGroup(elg); - } catch (ClassNotFoundException e) { - // BSD - builder.channelType( - Class.forName("io.netty.channel.kqueue.KQueueDomainSocketChannel") - .asSubclass(ServerChannel.class)); - final EventLoopGroup elg = - Class.forName("io.netty.channel.kqueue.KQueueEventLoopGroup") - .asSubclass(EventLoopGroup.class) - .getDeclaredConstructor() - .newInstance(); - builder.eventLoopGroup(elg); - } - } catch (ClassNotFoundException - | InstantiationException - | IllegalAccessException - | NoSuchMethodException - | InvocationTargetException e) { - throw new UnsupportedOperationException( - "Could not find suitable Netty native transport implementation for domain socket address."); - } - break; - } - default: - throw new IllegalArgumentException( - "Scheme is not supported: " + location.getUri().getScheme()); - } - - if (this.forceTls || LocationSchemes.GRPC_TLS.equals(location.getUri().getScheme())) { - builder.useTransportSecurity(); - - final boolean hasTrustedCerts = this.trustedCertificates != null; - final boolean hasKeyCertPair = this.clientCertificate != null && this.clientKey != null; - if (!this.verifyServer && (hasTrustedCerts || hasKeyCertPair)) { - throw new IllegalArgumentException( - "FlightClient has been configured to disable server verification, " - + "but certificate options have been specified."); - } - - final SslContextBuilder sslContextBuilder = GrpcSslContexts.forClient(); - - if (!this.verifyServer) { - sslContextBuilder.trustManager(InsecureTrustManagerFactory.INSTANCE); - } else if (this.trustedCertificates != null - || this.clientCertificate != null - || this.clientKey != null) { - if (this.trustedCertificates != null) { - sslContextBuilder.trustManager(this.trustedCertificates); - } - if (this.clientCertificate != null && this.clientKey != null) { - sslContextBuilder.keyManager(this.clientCertificate, this.clientKey); - } - } - try { - builder.sslContext(sslContextBuilder.build()); - } catch (SSLException e) { - throw new RuntimeException(e); - } - - if (this.overrideHostname != null) { - builder.overrideAuthority(this.overrideHostname); - } - } else { - builder.usePlaintext(); - } - - builder - .maxTraceEvents(MAX_CHANNEL_TRACE_EVENTS) - .maxInboundMessageSize(maxInboundMessageSize) - .maxInboundMetadataSize(maxInboundMessageSize); - return new FlightClient(allocator, builder.build(), middleware); + final NettyChannelBuilder channelBuilder = builder.build(); + return new FlightClient(builder.allocator(), channelBuilder.build(), builder.middleware()); } } diff --git a/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightGrpcUtils.java b/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightGrpcUtils.java index 13e4f2f215..df5e29741b 100644 --- a/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightGrpcUtils.java +++ b/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightGrpcUtils.java @@ -23,6 +23,7 @@ import io.grpc.ManagedChannel; import io.grpc.MethodDescriptor; import java.util.Collections; +import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import org.apache.arrow.flight.auth.ServerAuthHandler; @@ -151,6 +152,19 @@ public static FlightClient createFlightClient( return new FlightClient(incomingAllocator, channel, Collections.emptyList()); } + /** + * Creates a Flight client. + * + * @param incomingAllocator Memory allocator + * @param channel provides a connection to a gRPC server. + */ + public static FlightClient createFlightClient( + BufferAllocator incomingAllocator, + ManagedChannel channel, + List middleware) { + return new FlightClient(incomingAllocator, channel, middleware); + } + /** * Creates a Flight client. * diff --git a/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/NettyClientBuilder.java b/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/NettyClientBuilder.java new file mode 100644 index 0000000000..42cdaac016 --- /dev/null +++ b/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/NettyClientBuilder.java @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.flight.grpc; + +import io.grpc.ManagedChannel; +import io.grpc.netty.GrpcSslContexts; +import io.grpc.netty.NettyChannelBuilder; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.ServerChannel; +import io.netty.handler.ssl.SslContextBuilder; +import io.netty.handler.ssl.util.InsecureTrustManagerFactory; +import java.io.InputStream; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import javax.net.ssl.SSLException; +import org.apache.arrow.flight.FlightClientMiddleware; +import org.apache.arrow.flight.Location; +import org.apache.arrow.flight.LocationSchemes; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.util.Preconditions; + +/** + * A wrapper around gRPC's Netty builder. + * + *

It is recommended to use the Netty channel builder directly with {@link + * org.apache.arrow.flight.FlightGrpcUtils#createFlightClient(BufferAllocator, ManagedChannel)}. + * However, this class provides an adapter that implements the existing Flight-specific builder + * interface but allows usage of the Netty builder as well. + */ +public class NettyClientBuilder { + /** + * The maximum number of trace events to keep on the gRPC Channel. This value disables channel + * tracing. + */ + private static final int MAX_CHANNEL_TRACE_EVENTS = 0; + + protected BufferAllocator allocator; + protected Location location; + protected boolean forceTls = false; + protected int maxInboundMessageSize = Integer.MAX_VALUE; + protected InputStream trustedCertificates = null; + protected InputStream clientCertificate = null; + protected InputStream clientKey = null; + protected String overrideHostname = null; + protected List middleware = new ArrayList<>(); + protected boolean verifyServer = true; + + public NettyClientBuilder() {} + + public NettyClientBuilder(BufferAllocator allocator, Location location) { + this.allocator = Preconditions.checkNotNull(allocator); + this.location = Preconditions.checkNotNull(location); + } + + /** Force the client to connect over TLS. */ + public NettyClientBuilder useTls() { + this.forceTls = true; + return this; + } + + /** Override the hostname checked for TLS. Use with caution in production. */ + public NettyClientBuilder overrideHostname(final String hostname) { + this.overrideHostname = hostname; + return this; + } + + /** Set the maximum inbound message size. */ + public NettyClientBuilder maxInboundMessageSize(int maxSize) { + Preconditions.checkArgument(maxSize > 0); + this.maxInboundMessageSize = maxSize; + return this; + } + + /** Set the trusted TLS certificates. */ + public NettyClientBuilder trustedCertificates(final InputStream stream) { + this.trustedCertificates = Preconditions.checkNotNull(stream); + return this; + } + + /** Set the trusted TLS certificates. */ + public NettyClientBuilder clientCertificate( + final InputStream clientCertificate, final InputStream clientKey) { + Preconditions.checkNotNull(clientKey); + this.clientCertificate = Preconditions.checkNotNull(clientCertificate); + this.clientKey = Preconditions.checkNotNull(clientKey); + return this; + } + + public BufferAllocator allocator() { + return allocator; + } + + public NettyClientBuilder allocator(BufferAllocator allocator) { + this.allocator = Preconditions.checkNotNull(allocator); + return this; + } + + public NettyClientBuilder location(Location location) { + this.location = Preconditions.checkNotNull(location); + return this; + } + + public List middleware() { + return Collections.unmodifiableList(middleware); + } + + public NettyClientBuilder intercept(FlightClientMiddleware.Factory factory) { + middleware.add(factory); + return this; + } + + public NettyClientBuilder verifyServer(boolean verifyServer) { + this.verifyServer = verifyServer; + return this; + } + + /** Create the client from this builder. */ + public NettyChannelBuilder build() { + final NettyChannelBuilder builder; + + switch (location.getUri().getScheme()) { + case LocationSchemes.GRPC: + case LocationSchemes.GRPC_INSECURE: + case LocationSchemes.GRPC_TLS: + { + builder = NettyChannelBuilder.forAddress(location.toSocketAddress()); + break; + } + case LocationSchemes.GRPC_DOMAIN_SOCKET: + { + // The implementation is platform-specific, so we have to find the classes at runtime + builder = NettyChannelBuilder.forAddress(location.toSocketAddress()); + try { + try { + // Linux + builder.channelType( + Class.forName("io.netty.channel.epoll.EpollDomainSocketChannel") + .asSubclass(ServerChannel.class)); + final EventLoopGroup elg = + Class.forName("io.netty.channel.epoll.EpollEventLoopGroup") + .asSubclass(EventLoopGroup.class) + .getDeclaredConstructor() + .newInstance(); + builder.eventLoopGroup(elg); + } catch (ClassNotFoundException e) { + // BSD + builder.channelType( + Class.forName("io.netty.channel.kqueue.KQueueDomainSocketChannel") + .asSubclass(ServerChannel.class)); + final EventLoopGroup elg = + Class.forName("io.netty.channel.kqueue.KQueueEventLoopGroup") + .asSubclass(EventLoopGroup.class) + .getDeclaredConstructor() + .newInstance(); + builder.eventLoopGroup(elg); + } + } catch (ClassNotFoundException + | InstantiationException + | IllegalAccessException + | NoSuchMethodException + | InvocationTargetException e) { + throw new UnsupportedOperationException( + "Could not find suitable Netty native transport implementation for domain socket address."); + } + break; + } + default: + throw new IllegalArgumentException( + "Scheme is not supported: " + location.getUri().getScheme()); + } + + if (this.forceTls || LocationSchemes.GRPC_TLS.equals(location.getUri().getScheme())) { + builder.useTransportSecurity(); + + final boolean hasTrustedCerts = this.trustedCertificates != null; + final boolean hasKeyCertPair = this.clientCertificate != null && this.clientKey != null; + if (!this.verifyServer && (hasTrustedCerts || hasKeyCertPair)) { + throw new IllegalArgumentException( + "FlightClient has been configured to disable server verification, " + + "but certificate options have been specified."); + } + + final SslContextBuilder sslContextBuilder = GrpcSslContexts.forClient(); + + if (!this.verifyServer) { + sslContextBuilder.trustManager(InsecureTrustManagerFactory.INSTANCE); + } else if (this.trustedCertificates != null + || this.clientCertificate != null + || this.clientKey != null) { + if (this.trustedCertificates != null) { + sslContextBuilder.trustManager(this.trustedCertificates); + } + if (this.clientCertificate != null && this.clientKey != null) { + sslContextBuilder.keyManager(this.clientCertificate, this.clientKey); + } + } + try { + builder.sslContext(sslContextBuilder.build()); + } catch (SSLException e) { + throw new RuntimeException(e); + } + + if (this.overrideHostname != null) { + builder.overrideAuthority(this.overrideHostname); + } + } else { + builder.usePlaintext(); + } + + builder + .maxTraceEvents(MAX_CHANNEL_TRACE_EVENTS) + .maxInboundMessageSize(maxInboundMessageSize) + .maxInboundMetadataSize(maxInboundMessageSize); + return builder; + } +} From bec96a7f7956fac18020a8bac8e43b0904bb7683 Mon Sep 17 00:00:00 2001 From: David Li Date: Sun, 9 Mar 2025 21:50:45 -0400 Subject: [PATCH 111/408] GH-456: Add action to assign milestone to PR/issue (#659) ## What's Changed Derived from the ADBC one. Fixes #456. --------- Co-authored-by: Sutou Kouhei --- .github/workflows/dev_pr.yml | 7 +++ .github/workflows/dev_pr_milestone.sh | 70 +++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 .github/workflows/dev_pr_milestone.sh diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml index 8da3ab5fd5..34b3363c50 100644 --- a/.github/workflows/dev_pr.yml +++ b/.github/workflows/dev_pr.yml @@ -75,3 +75,10 @@ jobs: script: | const scripts = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr.js`); await scripts.check_linked_issue({core, github, context, issue: ${{ steps.title-format.outputs.result }}}); + + - name: Assign milestone + if: '! github.event.pull_request.draft' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + ./.github/workflows/dev_pr_milestone.sh "${GITHUB_REPOSITORY}" ${{ github.event.number }} diff --git a/.github/workflows/dev_pr_milestone.sh b/.github/workflows/dev_pr_milestone.sh new file mode 100644 index 0000000000..b6876b4b08 --- /dev/null +++ b/.github/workflows/dev_pr_milestone.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Assign a milestone to the given PR based on the open milestones and known +# releases. + +set -euo pipefail + +main() { + local -r repo="${1}" + local -r pr_number="${2}" + echo "On ${repo} pull ${pr_number}" + + local -r existing_milestone=$(gh pr view "${pr_number}" \ + --json milestone \ + -t '{{if .milestone}}{{.milestone.title}}{{end}}') + + if [[ -n "${existing_milestone}" ]]; then + echo "PR has milestone: ${existing_milestone}" + local -r milestone="${existing_milestone}" + else + local -r milestone=$( + gh api "/repos/${repo}/milestones" | + jq --raw-output '.[] | .title' | + grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' + head -n1 + ) + + echo "Assigning milestone: ${milestone}" + gh pr edit "${pr_number}" -m "${milestone}" + fi + + local -r repo_owner=$(echo "${repo}" | cut -d'/' -f1) + local -r repo_name=$(echo "${repo}" | cut -d'/' -f2) + local -r graphql_query="{ + repository(owner: \"${repo_owner}\", name: \"${repo_name}\") { + pullRequest(number: ${pr_number}) { + closingIssuesReferences(first: 5) { + edges { + node { + number + } + } + } + } + } + }" + local -r linked_issues=$(gh api graphql -f query="${graphql_query}" | jq -r '.data.repository.pullRequest.closingIssuesReferences.edges | .[].node.number') + for issue in ${linked_issues}; do + echo "Linked issue: ${issue}" + gh issue edit "${issue}" --milestone "${milestone}" + done +} + +main "$@" From 6f27dc92b1216f84747faf95c8d2cc2ab140c3f5 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 10 Mar 2025 04:01:12 -0400 Subject: [PATCH 112/408] MINOR: Add check-shebang-scripts-are-executable (#670) ## What's Changed Make sure that new scripts committed are actually executable, because I always forget. --- .github/workflows/dev_pr_milestone.sh | 0 .pre-commit-config.yaml | 1 + 2 files changed, 1 insertion(+) mode change 100644 => 100755 .github/workflows/dev_pr_milestone.sh diff --git a/.github/workflows/dev_pr_milestone.sh b/.github/workflows/dev_pr_milestone.sh old mode 100644 new mode 100755 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ca3cacec3b..8e8faf3cc0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,6 +25,7 @@ repos: hooks: - id: trailing-whitespace - id: end-of-file-fixer + - id: check-shebang-scripts-are-executable - id: check-yaml - id: check-added-large-files - id: file-contents-sorter From 0225334e3aa563dd531328332cf9584302e145b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Mon, 10 Mar 2025 10:12:53 +0100 Subject: [PATCH 113/408] Add jbonofre in CODEOWNERS and remove from collaborators list (#671) As jbonofre has been elected as committers, collaborators entry is not required anymore. And happy to be in `CODEOWNERS` to help on review. --- .asf.yaml | 2 -- .github/CODEOWNERS | 50 +++++++++++++++++++++++----------------------- 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index 05a8663e3c..ead2581149 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -33,8 +33,6 @@ github: protected_branches: main: required_linear_history: true - collaborators: # Note: the number of collaborators is limited to 10 - - jbonofre notifications: commits: commits@arrow.apache.org issues_status: issues@arrow.apache.org diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9e8acb8d85..8201a5a915 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -23,34 +23,34 @@ # documentation about the syntax: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners # modules -/adapter/ @lidavidm @laurentgo @wgtmac -/algorithm/ @lidavidm @laurentgo @wgtmac -/arrow-format/ @lidavidm @laurentgo @wgtmac -/c/ @lidavidm @laurentgo @wgtmac -/compression/ @lidavidm @laurentgo @wgtmac -/dataset/ @lidavidm @laurentgo @wgtmac -/flight/ @lidavidm @laurentgo @wgtmac -/format/ @lidavidm @laurentgo @wgtmac -/gandiva/ @lidavidm @laurentgo @wgtmac -/memory/ @lidavidm @laurentgo @wgtmac -/performance/ @lidavidm @laurentgo @wgtmac -/tools/ @lidavidm @laurentgo @wgtmac -/vector/ @lidavidm @laurentgo @wgtmac -CMakeLists.txt @lidavidm @laurentgo @wgtmac +/adapter/ @lidavidm @laurentgo @wgtmac @jbonofre +/algorithm/ @lidavidm @laurentgo @wgtmac @jbonofre +/arrow-format/ @lidavidm @laurentgo @wgtmac @jbonofre +/c/ @lidavidm @laurentgo @wgtmac @jbonofre +/compression/ @lidavidm @laurentgo @wgtmac @jbonofre +/dataset/ @lidavidm @laurentgo @wgtmac @jbonofre +/flight/ @lidavidm @laurentgo @wgtmac @jbonofre +/format/ @lidavidm @laurentgo @wgtmac @jbonofre +/gandiva/ @lidavidm @laurentgo @wgtmac @jbonofre +/memory/ @lidavidm @laurentgo @wgtmac @jbonofre +/performance/ @lidavidm @laurentgo @wgtmac @jbonofre +/tools/ @lidavidm @laurentgo @wgtmac @jbonofre +/vector/ @lidavidm @laurentgo @wgtmac @jbonofre +CMakeLists.txt @lidavidm @laurentgo @wgtmac @jbonofre # release scripts -/ci/ @lidavidm @laurentgo @wgtmac @kou -/dev/ @lidavidm @laurentgo @wgtmac @kou +/ci/ @lidavidm @laurentgo @wgtmac @kou @jbonofre +/dev/ @lidavidm @laurentgo @wgtmac @kou @jbonofre # PR CI and repository files -/.github/ @lidavidm @laurentgo @wgtmac @kou -/.asf.yaml @lidavidm @laurentgo @wgtmac @kou -/.env @lidavidm @laurentgo @wgtmac -/.pre-commit-config.yaml @lidavidm @laurentgo @wgtmac -/Brewfile @lidavidm @laurentgo @wgtmac -/docker-compose.yaml @lidavidm @laurentgo @wgtmac +/.github/ @lidavidm @laurentgo @wgtmac @kou @jbonofre +/.asf.yaml @lidavidm @laurentgo @wgtmac @kou @jbonofre +/.env @lidavidm @laurentgo @wgtmac @jbonofre +/.pre-commit-config.yaml @lidavidm @laurentgo @wgtmac @jbonofre +/Brewfile @lidavidm @laurentgo @wgtmac @jbonofre +/docker-compose.yaml @lidavidm @laurentgo @wgtmac @jbonofre # Java specific -/.mvn/ @lidavidm @laurentgo @wgtmac -/bom/ @lidavidm @laurentgo @wgtmac -pom.xml @lidavidm @laurentgo @wgtmac +/.mvn/ @lidavidm @laurentgo @wgtmac @jbonofre +/bom/ @lidavidm @laurentgo @wgtmac @jbonofre +pom.xml @lidavidm @laurentgo @wgtmac @jbonofre From d8c86d191dcb369428d80afdf98223480cad4794 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Mar 2025 11:28:13 +0100 Subject: [PATCH 114/408] MINOR: Bump org.apache.orc:orc-core from 2.1.0 to 2.1.1 (#664) Bumps org.apache.orc:orc-core from 2.1.0 to 2.1.1. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.orc:orc-core&package-manager=maven&previous-version=2.1.0&new-version=2.1.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---

Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- adapter/orc/pom.xml | 2 +- dataset/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/adapter/orc/pom.xml b/adapter/orc/pom.xml index 8b45dd72c9..e3ae7d5163 100644 --- a/adapter/orc/pom.xml +++ b/adapter/orc/pom.xml @@ -61,7 +61,7 @@ under the License. org.apache.orc orc-core - 2.1.0 + 2.1.1 test diff --git a/dataset/pom.xml b/dataset/pom.xml index 4ff931d307..d671fcf1f9 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -130,7 +130,7 @@ under the License. org.apache.orc orc-core - 2.1.0 + 2.1.1 test From adfbef4a23ea145d974597be2256491fb2cc926b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Mar 2025 14:46:57 +0100 Subject: [PATCH 115/408] MINOR: Bump io.grpc:grpc-bom from 1.70.0 to 1.71.0 (#668) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [io.grpc:grpc-bom](https://github.com/grpc/grpc-java) from 1.70.0 to 1.71.0.
Release notes

Sourced from io.grpc:grpc-bom's releases.

V1.71.0

API Changes

  • xds: Enable Xds Client Fallback by default. This allows having a backup xDS server as described in gRFC A71-xds-fallback.md (#11817) (176f3eed1)
  • protobuf: Experimental API marshallerWithRecursionLimit in ProtoUtils is now stabilized (#11884) (90b1c4fe9)

Bug Fixes

  • xds: Cluster weights should be uint32 (199a7ea3e). They were previously processed as int32, although the sum of weights was checked to be positive. So this would have caused a very large weight to never be selected and to reduce the chances of immediately-following clusters to be selected. There have been no reports of control planes using such large weights
  • xds: Fix an unlikely infinite loop triggered by route update (199a7ea3e). Triggering required the old cluster to no longer be used, an RPC processing when the update arrived, and for a RPC to not match any route in the new config. There have been no reports of this actually happening
  • core: Release data frame if it is received before the headers (dc316f7fd)

Improvements

  • Replace jsr305's CheckReturnValue with Error Prone's (#11811) (7b5d0692c)
  • core: optimize number of buffer allocations for message sizes larger than 1 MB (#11879) (5a7f35053)
  • core: Update the retry backoff range from [0, 1] to [0.8, 1.2] as per the A6 redefinition (#11858) (44e92e2c2)
  • core: include last pick status in status message when wait-for-ready RPC’s deadline expires (#11851) (7585b1607). This makes it much easier to debug connectivity issues when using wait-for-ready RPCs
  • xds: Include max concurrent request limit in the error status for concurrent connections limit exceeded (#11845) (0f5503ebb)
  • netty, servlet: Remove 4096 min write buffer size because MessageFramer.flush() is being called between every message, so messages are never combined and the larger allocation just wastes memory. (4a10a3816, 7153ff852)
  • core: When ClientStreamObserver closes the response observer log the error message if this operation fails (#11880) (302342cfc)
  • bom: use gradle java-platform to build pom instead of custom xml generation (#11875) (3142928fa)
  • xds: Reuse filter interceptors on client-side across RPCs (c506190b0, b3db8c248). This was an internal refactor that should have no user-visible change
  • alts: Enhance AltsContextUtil to allow getting the AltsContext on client-side (b1bc0a9d2)
  • xds: Envoy proto sync to 2024-11-11 (#11816) (b44ebce45)

Documentation

  • examples: Update HelloWorldServer to use Executor (#11850) (16edf7ac4)
  • examples: Add README for all examples lacking it (#11676) (9e8629914)

Dependencies

  • Version upgrades (#11874) (fc8571a0e)
  • Upgrade netty-tcnative to 2.0.70 (122b68371)

Thanks to

@​benjamin @​panchenko @​harshagoo94 @​NaveenPrasannaV

Commits
  • 865c443 Bump version to 1.71.0
  • ac9d0d2 Update README etc to reference 1.71.0
  • 16edf7a Examples: Updated HelloWorldServer to use Executor (#11850)
  • 16d2672 s2a: Don't allow S2AStub to be set
  • 9e54e8e servlet: Provide Gradle a filter version number
  • c1d7035 okhttp:Use a locally specified value instead of Segment.SIZE in okhttp (#11899)
  • 57af63a kokoro: Increase gradle mem in android-interop
  • a5347b2 s2a: inject Optional<AccessTokenManager> in tests
  • 41dd0c6 xds:Cleanup to reduce test flakiness (#11895)
  • 5a7f350 optimize number of buffer allocations (#11879)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=io.grpc:grpc-bom&package-manager=maven&previous-version=1.70.0&new-version=1.71.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: JB Onofré --- flight/flight-integration-tests/src/shade/LICENSE.txt | 2 +- flight/flight-integration-tests/src/shade/NOTICE.txt | 2 +- flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt | 2 +- flight/flight-sql-jdbc-driver/src/shade/NOTICE.txt | 2 +- pom.xml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flight/flight-integration-tests/src/shade/LICENSE.txt b/flight/flight-integration-tests/src/shade/LICENSE.txt index 1d49e4e670..b4aba55f70 100644 --- a/flight/flight-integration-tests/src/shade/LICENSE.txt +++ b/flight/flight-integration-tests/src/shade/LICENSE.txt @@ -233,7 +233,7 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- -This binary artifact contains gRPC 1.70.0. +This binary artifact contains gRPC 1.71.0. Copyright: Copyright 2014 The gRPC Authors Home page: https://grpc.io/ diff --git a/flight/flight-integration-tests/src/shade/NOTICE.txt b/flight/flight-integration-tests/src/shade/NOTICE.txt index 6ec75cd470..668cfe684b 100644 --- a/flight/flight-integration-tests/src/shade/NOTICE.txt +++ b/flight/flight-integration-tests/src/shade/NOTICE.txt @@ -274,7 +274,7 @@ This product includes Netty 4.1.119.Final, with the following in its NOTICE: --------------------------------------------------------- -This product includes gRPC 1.70.0, with the following in its NOTICE: +This product includes gRPC 1.71.0, with the following in its NOTICE: | Copyright 2014 The gRPC Authors | diff --git a/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt b/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt index fa445fb055..41c80d948c 100644 --- a/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt +++ b/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt @@ -203,7 +203,7 @@ -------------------------------------------------------------------------------- -This binary artifact contains gRPC 1.70.0. +This binary artifact contains gRPC 1.71.0. Copyright: Copyright 2014 The gRPC Authors Home page: https://grpc.io/ diff --git a/flight/flight-sql-jdbc-driver/src/shade/NOTICE.txt b/flight/flight-sql-jdbc-driver/src/shade/NOTICE.txt index 0e833bb1c8..2f2f6b9e75 100644 --- a/flight/flight-sql-jdbc-driver/src/shade/NOTICE.txt +++ b/flight/flight-sql-jdbc-driver/src/shade/NOTICE.txt @@ -6,7 +6,7 @@ The Apache Software Foundation (http://www.apache.org/). --------------------------------------------------- -This product includes gRPC 1.70.0, with the following in its NOTICE: +This product includes gRPC 1.71.0, with the following in its NOTICE: | Copyright 2014 The gRPC Authors | diff --git a/pom.xml b/pom.xml index 16e1ae36f4..f27013c0ba 100644 --- a/pom.xml +++ b/pom.xml @@ -97,7 +97,7 @@ under the License. 2.0.17 33.4.0-jre 4.1.119.Final - 1.70.0 + 1.71.0 3.25.5 2.18.3 3.4.1 From 6ca25f78db90e0060ff93f068d25cb837df4d732 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Mar 2025 14:49:46 +0100 Subject: [PATCH 116/408] MINOR: Bump checker.framework.version from 3.49.0 to 3.49.1 (#663) Bumps `checker.framework.version` from 3.49.0 to 3.49.1. Updates `org.checkerframework:checker-qual` from 3.49.0 to 3.49.1
Release notes

Sourced from org.checkerframework:checker-qual's releases.

Checker Framework 3.49.1

Version 3.49.1 (March 3, 2025)

Closed issues:

#6970, #6974.

Changelog

Sourced from org.checkerframework:checker-qual's changelog.

Version 3.49.1 (March 3, 2025)

Closed issues:

#6970, #6974.

Commits

Updates `org.checkerframework:checker` from 3.49.0 to 3.49.1
Release notes

Sourced from org.checkerframework:checker's releases.

Checker Framework 3.49.1

Version 3.49.1 (March 3, 2025)

Closed issues:

#6970, #6974.

Changelog

Sourced from org.checkerframework:checker's changelog.

Version 3.49.1 (March 3, 2025)

Closed issues:

#6970, #6974.

Commits

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f27013c0ba..c569008740 100644 --- a/pom.xml +++ b/pom.xml @@ -109,7 +109,7 @@ under the License. true 2.31.0 5.15.2 - 3.49.0 + 3.49.1 1.5.17 none -Xdoclint:none From 14ca2f3f5cb7cc898db44267e62eae963f104c1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Mon, 10 Mar 2025 23:07:43 +0100 Subject: [PATCH 117/408] MINOR: Use mockito bom and upgrade to Mockito 5.16.0 (#672) Use Mockito BoM for dependencies instead of individual versions. --- flight/flight-sql-jdbc-core/pom.xml | 6 +++++- pom.xml | 15 ++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 237d450eec..15368e918d 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -97,7 +97,11 @@ under the License. org.mockito mockito-core - ${mockito.core.version} + test + + + org.mockito + mockito-junit-jupiter test diff --git a/pom.xml b/pom.xml index c569008740..c27b4d04f8 100644 --- a/pom.xml +++ b/pom.xml @@ -103,12 +103,12 @@ under the License. 3.4.1 25.2.10 1.12.0 + 5.16.0 2 10.21.3 true 2.31.0 - 5.15.2 3.49.1 1.5.17 none @@ -221,6 +221,13 @@ under the License. pom import + + org.mockito + mockito-bom + ${dep.mockito-bom.version} + pom + import + ch.qos.logback logback-classic @@ -275,12 +282,6 @@ under the License. ${dep.junit.jupiter.version} test - - org.mockito - mockito-junit-jupiter - 5.15.2 - test - ch.qos.logback logback-classic From 66cb737f77f75fd1616873b69a49c4876f4ef86e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Mar 2025 09:28:58 +0100 Subject: [PATCH 118/408] MINOR: Bump com.puppycrawl.tools:checkstyle from 10.21.3 to 10.21.4 (#667) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [com.puppycrawl.tools:checkstyle](https://github.com/checkstyle/checkstyle) from 10.21.3 to 10.21.4.
Release notes

Sourced from com.puppycrawl.tools:checkstyle's releases.

checkstyle-10.21.4

Checkstyle 10.21.4 - https://checkstyle.org/releasenotes.html#Release_10.21.4

Bug fixes:

#15161 - AnnotationOnSameLine: FalseNegative when the annotation is on the same line of the modifer of the target class #16081 - PARAMETER_DEF in FinalLocalVariable doesn't seem to work in interfaces #16165 - Regression: CyclomaticComplexity.switchBlockAsSingleDecisionPoint does not work properly anymore (10.21.1)

Commits
  • b7014bb [maven-release-plugin] prepare release checkstyle-10.21.4
  • 71e5f59 doc: release notes for 10.21.4
  • a745784 Issue #14631: Updated hash_literal to new AST format
  • 356c947 Issue #14631: Updated COLGROUP of JavadocTokenTypes
  • d4025b2 Issue #13999: Resolve pitest suppression for skipHtmlComment() method for Tag...
  • e1dc3ab Issue #15456: Violation message for SuperCloneCheck
  • cc3ada6 Issue #15161: Resolved False Negative
  • e4b625a Issue #13999: resolve pitest Suppression for JavadocStyleCheck
  • 203ce2e Issue #14631: Updated BODY_TAG_START BODY_TAG_END of JavadocTokenTypes
  • fe24819 dependency: bump org.apache.maven.plugins:maven-install-plugin
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.puppycrawl.tools:checkstyle&package-manager=maven&previous-version=10.21.3&new-version=10.21.4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: JB Onofré --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c27b4d04f8..76c7fe4f46 100644 --- a/pom.xml +++ b/pom.xml @@ -106,7 +106,7 @@ under the License. 5.16.0 2 - 10.21.3 + 10.21.4 true 2.31.0 3.49.1 From d13f0e08972da3311442bdf23f75d27f4998fc85 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Mar 2025 07:29:54 -0400 Subject: [PATCH 119/408] MINOR: Bump com.google.protobuf:protobuf-bom from 3.25.5 to 4.30.0 (#666) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [com.google.protobuf:protobuf-bom](https://github.com/protocolbuffers/protobuf) from 3.25.5 to 4.30.0. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: JB Onofré --- .../java/org/apache/arrow/flight/auth/ClientAuthWrapper.java | 4 +--- .../java/org/apache/arrow/flight/auth/ServerAuthWrapper.java | 4 +--- flight/flight-integration-tests/src/shade/LICENSE.txt | 4 ++-- flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt | 4 ++-- pom.xml | 2 +- 5 files changed, 7 insertions(+), 11 deletions(-) diff --git a/flight/flight-core/src/main/java/org/apache/arrow/flight/auth/ClientAuthWrapper.java b/flight/flight-core/src/main/java/org/apache/arrow/flight/auth/ClientAuthWrapper.java index dd62f7756e..19d4f77a03 100644 --- a/flight/flight-core/src/main/java/org/apache/arrow/flight/auth/ClientAuthWrapper.java +++ b/flight/flight-core/src/main/java/org/apache/arrow/flight/auth/ClientAuthWrapper.java @@ -76,9 +76,7 @@ public AuthObserver() { @Override public void onNext(HandshakeResponse value) { ByteString payload = value.getPayload(); - if (payload != null) { - messages.add(payload.toByteArray()); - } + messages.add(payload.toByteArray()); } private Iterator iter = diff --git a/flight/flight-core/src/main/java/org/apache/arrow/flight/auth/ServerAuthWrapper.java b/flight/flight-core/src/main/java/org/apache/arrow/flight/auth/ServerAuthWrapper.java index 879a93a73b..fba9f9baba 100644 --- a/flight/flight-core/src/main/java/org/apache/arrow/flight/auth/ServerAuthWrapper.java +++ b/flight/flight-core/src/main/java/org/apache/arrow/flight/auth/ServerAuthWrapper.java @@ -84,9 +84,7 @@ public AuthObserver(StreamObserver responseObserver) { @Override public void onNext(HandshakeRequest value) { ByteString payload = value.getPayload(); - if (payload != null) { - messages.add(payload.toByteArray()); - } + messages.add(payload.toByteArray()); } private Iterator iter = diff --git a/flight/flight-integration-tests/src/shade/LICENSE.txt b/flight/flight-integration-tests/src/shade/LICENSE.txt index b4aba55f70..6ee8645869 100644 --- a/flight/flight-integration-tests/src/shade/LICENSE.txt +++ b/flight/flight-integration-tests/src/shade/LICENSE.txt @@ -272,11 +272,11 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- -This binary artifact contains Google Protobuf 3.25.5. +This binary artifact contains Google Protobuf 4.30.0. Copyright: Copyright 2008 Google Inc. All rights reserved. Home page: https://protobuf.dev/ -License: https://github.com/protocolbuffers/protobuf/blob/v3.25.5/LICENSE (BSD) +License: https://github.com/protocolbuffers/protobuf/blob/v4.30.0/LICENSE (BSD) License text: | Copyright 2008 Google Inc. All rights reserved. diff --git a/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt b/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt index 41c80d948c..c58a03831f 100644 --- a/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt +++ b/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt @@ -234,11 +234,11 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- -This binary artifact contains Protobuf 3.25.5. +This binary artifact contains Protobuf 4.30.0. Copyright: Copyright 2008 Google Inc. All rights reserved. Home page: https://protobuf.dev/ -License: https://github.com/protocolbuffers/protobuf/blob/v3.25.3/LICENSE (BSD) +License: https://github.com/protocolbuffers/protobuf/blob/v4.30.0/LICENSE (BSD) License text: | Copyright 2008 Google Inc. All rights reserved. diff --git a/pom.xml b/pom.xml index 76c7fe4f46..4cab1dafad 100644 --- a/pom.xml +++ b/pom.xml @@ -98,7 +98,7 @@ under the License. 33.4.0-jre 4.1.119.Final 1.71.0 - 3.25.5 + 4.30.0 2.18.3 3.4.1 25.2.10 From 32e4ae4b0b844da3a06f6d8bb205b90622b9f46e Mon Sep 17 00:00:00 2001 From: lriggs Date: Wed, 12 Mar 2025 17:32:10 -0700 Subject: [PATCH 120/408] GH-655: Failure in UnionReader.read after DecimalVector promotion to UnionVector (#656) When a DecimalVector is promoted to a UnionVector via a PromotableWriter, the UnionVector will have the decimal vector in it's internal struct vector, but the decimalVector field will not be set. If UnionReader.read is then used to read from the UnionVector, it will fail when it tries to read one of the promoted decimal values, due to decimalVector being null, and the exact decimal type not being provided. This failure is unnecessary though as we have a pre-existing decimal vector, the caller just does not know the exact type - and it shouldn't be required to. The change here is to check for a pre-existing decimal vector in the internal struct when getDecimalVector() is called. If one exists, set the decimalVector field and return. Otherwise, if none exists, throw the exception. ## What's Changed Updated UnionVector to handle this case. Closes GH-655 --------- Co-authored-by: Scott Cowell --- .../main/codegen/templates/UnionVector.java | 6 ++- .../complex/impl/TestPromotableWriter.java | 47 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/vector/src/main/codegen/templates/UnionVector.java b/vector/src/main/codegen/templates/UnionVector.java index e0fd0e4644..67efdf60f7 100644 --- a/vector/src/main/codegen/templates/UnionVector.java +++ b/vector/src/main/codegen/templates/UnionVector.java @@ -23,6 +23,7 @@ import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.BaseValueVector; import org.apache.arrow.vector.BitVectorHelper; +import org.apache.arrow.vector.DecimalVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.complex.AbstractStructVector; @@ -279,7 +280,10 @@ public StructVector getStruct() { <#if minor.class?starts_with("Decimal") || is_timestamp_tz(minor.class) || minor.class == "Duration" || minor.class == "FixedSizeBinary"> public ${name}Vector get${name}Vector() { if (${uncappedName}Vector == null) { - throw new IllegalArgumentException("No ${name} present. Provide ArrowType argument to create a new vector"); + ${uncappedName}Vector = internalStruct.getChild(fieldName(MinorType.${name?upper_case}), ${name}Vector.class); + if (${uncappedName}Vector == null) { + throw new IllegalArgumentException("No ${name} present. Provide ArrowType argument to create a new vector"); + } } return ${uncappedName}Vector; } diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java index 19b26b6d0e..a791e55135 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java @@ -21,12 +21,14 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import java.math.BigDecimal; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; import java.util.Objects; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.DecimalVector; import org.apache.arrow.vector.DirtyRootAllocator; import org.apache.arrow.vector.LargeVarBinaryVector; import org.apache.arrow.vector.LargeVarCharVector; @@ -39,14 +41,18 @@ import org.apache.arrow.vector.complex.writer.BaseWriter.StructWriter; import org.apache.arrow.vector.holders.DurationHolder; import org.apache.arrow.vector.holders.FixedSizeBinaryHolder; +import org.apache.arrow.vector.holders.NullableDecimalHolder; +import org.apache.arrow.vector.holders.NullableIntHolder; import org.apache.arrow.vector.holders.NullableTimeStampMilliTZHolder; import org.apache.arrow.vector.holders.TimeStampMilliTZHolder; +import org.apache.arrow.vector.holders.UnionHolder; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.DecimalUtility; import org.apache.arrow.vector.util.Text; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -729,4 +735,45 @@ public void testPromoteLargeVarBinaryHelpersDirect() throws Exception { assertEquals("row4", new String(Objects.requireNonNull(uv.get(3)), StandardCharsets.UTF_8)); } } + + @Test + public void testPromoteToUnionFromDecimal() throws Exception { + try (final NonNullableStructVector container = + NonNullableStructVector.empty(EMPTY_SCHEMA_PATH, allocator); + final DecimalVector v = + container.addOrGet( + "dec", FieldType.nullable(new ArrowType.Decimal(38, 1, 128)), DecimalVector.class); + final PromotableWriter writer = new PromotableWriter(v, container)) { + + container.allocateNew(); + container.setValueCount(1); + + writer.setPosition(0); + writer.writeDecimal(new BigDecimal("0.1")); + writer.setPosition(1); + writer.writeInt(1); + + container.setValueCount(3); + + UnionVector unionVector = (UnionVector) container.getChild("dec"); + UnionHolder holder = new UnionHolder(); + + unionVector.get(0, holder); + NullableDecimalHolder decimalHolder = new NullableDecimalHolder(); + holder.reader.read(decimalHolder); + + assertEquals(1, decimalHolder.isSet); + assertEquals( + new BigDecimal("0.1"), + DecimalUtility.getBigDecimalFromArrowBuf( + decimalHolder.buffer, 0, decimalHolder.scale, 128)); + + unionVector.get(1, holder); + NullableIntHolder intHolder = new NullableIntHolder(); + holder.reader.read(intHolder); + + assertEquals(1, intHolder.isSet); + assertEquals(1, intHolder.value); + } + } } From d3e05d5fbff011fdee6dcae169d1039b1de3dcc1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Mar 2025 15:49:35 +0900 Subject: [PATCH 121/408] MINOR: [CI] Bump docker/login-action from 3.3.0 to 3.4.0 (#674) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [docker/login-action](https://github.com/docker/login-action) from 3.3.0 to 3.4.0.
Release notes

Sourced from docker/login-action's releases.

v3.4.0

Full Changelog: https://github.com/docker/login-action/compare/v3.3.0...v3.4.0

Commits
  • 74a5d14 Merge pull request #856 from docker/dependabot/npm_and_yarn/aws-sdk-dependenc...
  • 2f4f00e chore: update generated content
  • 67c1845 build(deps): bump the aws-sdk-dependencies group across 1 directory with 2 up...
  • 3d4cc89 Merge pull request #844 from graysonpike/master
  • 6cc823a Merge pull request #823 from docker/dependabot/npm_and_yarn/proxy-agent-depen...
  • d94e792 chore: update generated content
  • 033db0d Merge pull request #812 from docker/dependabot/github_actions/codecov/codecov...
  • 09c2ae9 build(deps): bump https-proxy-agent
  • ba56f00 ci: update deprecated input for codecov-action
  • 75bf9a7 Merge pull request #858 from docker/dependabot/npm_and_yarn/docker/actions-to...
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=docker/login-action&package-manager=github_actions&previous-version=3.3.0&new-version=3.4.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 5d37252a93..97b802d9b8 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -133,7 +133,7 @@ jobs: with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing - - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 with: registry: ghcr.io username: ${{ github.actor }} From 88eb0a333e48828bb3ec1c90f29b58263e944330 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Mar 2025 08:02:36 +0100 Subject: [PATCH 122/408] MINOR: Bump dep.junit.jupiter.version from 5.12.0 to 5.12.1 (#680) Bumps `dep.junit.jupiter.version` from 5.12.0 to 5.12.1. Updates `org.junit.jupiter:junit-jupiter-engine` from 5.12.0 to 5.12.1
Release notes

Sourced from org.junit.jupiter:junit-jupiter-engine's releases.

JUnit 5.12.1 = Platform 1.12.1 + Jupiter 5.12.1 + Vintage 5.12.1

See Release Notes.

Full Changelog: https://github.com/junit-team/junit5/compare/r5.12.0...r5.12.1

Commits
  • ba9c9ae Release 5.12.1
  • e28ad4a Finalize 5.12.1 release notes
  • 1044e2c Move entry to 5.12.1 release notes
  • bea821d Fix Javadoc formatting
  • 1a1a67d Set stable module name for the standalone JAR file
  • 062d3fa Remove internal packages from API reports
  • 74521ab Introduce ExtensionContext.getEnclosingTestClasses()
  • 1e135b9 Create initial 5.12.1 release notes from template
  • d540e4f Link to correct method
  • 31d60b7 Split API reports by module and package
  • Additional commits viewable in compare view

Updates `org.junit.jupiter:junit-jupiter-api` from 5.12.0 to 5.12.1
Release notes

Sourced from org.junit.jupiter:junit-jupiter-api's releases.

JUnit 5.12.1 = Platform 1.12.1 + Jupiter 5.12.1 + Vintage 5.12.1

See Release Notes.

Full Changelog: https://github.com/junit-team/junit5/compare/r5.12.0...r5.12.1

Commits
  • ba9c9ae Release 5.12.1
  • e28ad4a Finalize 5.12.1 release notes
  • 1044e2c Move entry to 5.12.1 release notes
  • bea821d Fix Javadoc formatting
  • 1a1a67d Set stable module name for the standalone JAR file
  • 062d3fa Remove internal packages from API reports
  • 74521ab Introduce ExtensionContext.getEnclosingTestClasses()
  • 1e135b9 Create initial 5.12.1 release notes from template
  • d540e4f Link to correct method
  • 31d60b7 Split API reports by module and package
  • Additional commits viewable in compare view

Updates `org.junit.jupiter:junit-jupiter-params` from 5.12.0 to 5.12.1
Release notes

Sourced from org.junit.jupiter:junit-jupiter-params's releases.

JUnit 5.12.1 = Platform 1.12.1 + Jupiter 5.12.1 + Vintage 5.12.1

See Release Notes.

Full Changelog: https://github.com/junit-team/junit5/compare/r5.12.0...r5.12.1

Commits
  • ba9c9ae Release 5.12.1
  • e28ad4a Finalize 5.12.1 release notes
  • 1044e2c Move entry to 5.12.1 release notes
  • bea821d Fix Javadoc formatting
  • 1a1a67d Set stable module name for the standalone JAR file
  • 062d3fa Remove internal packages from API reports
  • 74521ab Introduce ExtensionContext.getEnclosingTestClasses()
  • 1e135b9 Create initial 5.12.1 release notes from template
  • d540e4f Link to correct method
  • 31d60b7 Split API reports by module and package
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4cab1dafad..99e3f83b6a 100644 --- a/pom.xml +++ b/pom.xml @@ -93,7 +93,7 @@ under the License. ${project.build.directory}/generated-sources 1.9.0 - 5.12.0 + 5.12.1 2.0.17 33.4.0-jre 4.1.119.Final From e0735d457b3c3b8ed67860780025635658676e5e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Mar 2025 07:26:44 +0100 Subject: [PATCH 123/408] MINOR: Bump org.mockito:mockito-bom from 5.16.0 to 5.16.1 (#678) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.mockito:mockito-bom](https://github.com/mockito/mockito) from 5.16.0 to 5.16.1.
Release notes

Sourced from org.mockito:mockito-bom's releases.

v5.16.1

Changelog generated by Shipkit Changelog Gradle Plugin

5.16.1

Commits
  • d000e63 Rework of injection strategy in the context of modules (#3608)
  • 0215884 Remove Arrays.asList from critical stubbing path in GenericMetadataSupport (#...
  • d185035 Add reference to Gradle documentation on how to make task relocatable (#3606)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.mockito:mockito-bom&package-manager=maven&previous-version=5.16.0&new-version=5.16.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 99e3f83b6a..1a93166ed6 100644 --- a/pom.xml +++ b/pom.xml @@ -103,7 +103,7 @@ under the License. 3.4.1 25.2.10 1.12.0 - 5.16.0 + 5.16.1 2 10.21.4 From 8065a438288d8806d936b33362bad494150b4e8e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Mar 2025 07:27:28 +0100 Subject: [PATCH 124/408] MINOR: Bump parquet.version from 1.15.0 to 1.15.1 (#676) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps `parquet.version` from 1.15.0 to 1.15.1. Updates `org.apache.parquet:parquet-avro` from 1.15.0 to 1.15.1
Release notes

Sourced from org.apache.parquet:parquet-avro's releases.

Apache Parquet Java 1.15.1

What's Changed

Full Changelog: https://github.com/apache/parquet-java/compare/apache-parquet-1.15.0...apache-parquet-1.15.1

Commits
  • c7257b8 [maven-release-plugin] prepare release apache-parquet-1.15.1-rc0
  • 8e70eea GH-3168: Restrict trusted packages in the parquet-avro module (#3169)
  • 9d6e19f GH-3172: Do not drop blocks with some null values if DictionaryFilter is ap...
  • a19e985 GH-3133: Fix SizeStatistics to handle omitted histogram (#3135)
  • de3c2d0 Prepare for next development iteration
  • 010b067 [maven-release-plugin] prepare for next development iteration
  • See full diff in compare view

Updates `org.apache.parquet:parquet-hadoop` from 1.15.0 to 1.15.1
Release notes

Sourced from org.apache.parquet:parquet-hadoop's releases.

Apache Parquet Java 1.15.1

What's Changed

Full Changelog: https://github.com/apache/parquet-java/compare/apache-parquet-1.15.0...apache-parquet-1.15.1

Commits
  • c7257b8 [maven-release-plugin] prepare release apache-parquet-1.15.1-rc0
  • 8e70eea GH-3168: Restrict trusted packages in the parquet-avro module (#3169)
  • 9d6e19f GH-3172: Do not drop blocks with some null values if DictionaryFilter is ap...
  • a19e985 GH-3133: Fix SizeStatistics to handle omitted histogram (#3135)
  • de3c2d0 Prepare for next development iteration
  • 010b067 [maven-release-plugin] prepare for next development iteration
  • See full diff in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dataset/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataset/pom.xml b/dataset/pom.xml index d671fcf1f9..7c2da2ac21 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -32,7 +32,7 @@ under the License. ../../../cpp/release-build/ - 1.15.0 + 1.15.1 1.12.0 From 134ad66b35318807e8234bea4eb13c098d988b52 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Mar 2025 03:25:11 -0400 Subject: [PATCH 125/408] MINOR: [CI] Bump actions/cache from 4.2.2 to 4.2.3 (#687) Bumps [actions/cache](https://github.com/actions/cache) from 4.2.2 to 4.2.3. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/dev.yml | 2 +- .github/workflows/rc.yml | 8 ++++---- .github/workflows/test.yml | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 80d334bb78..37320898a6 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -42,7 +42,7 @@ jobs: with: python-version: '3.x' - name: pre-commit (cache) - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: ~/.cache/pre-commit key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 97b802d9b8..a1dbbfabe8 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -139,7 +139,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Cache - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: .docker key: jni-linux-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} @@ -266,7 +266,7 @@ jobs: run: | echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} - name: Cache ccache - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: ccache key: jni-macos-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} @@ -342,7 +342,7 @@ jobs: run: | echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} - name: Cache ccache - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: ccache key: jni-windows-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} @@ -416,7 +416,7 @@ jobs: repository: apache/arrow-testing path: testing - name: Cache ~/.m2 - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: ~/.m2 key: binaries-build-${{ hashFiles('**/*.java', '**/pom.xml') }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2fd12a02bf..5db5c988eb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,7 +63,7 @@ jobs: fetch-depth: 0 submodules: recursive - name: Cache Docker Volumes - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: .docker key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('compose.yaml', '**/pom.xml', '**/*.java') }} @@ -180,7 +180,7 @@ jobs: run: | ci/scripts/util_free_space.sh - name: Cache Docker Volumes - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: .docker key: integration-conda-${{ hashFiles('cpp/**') }} From ebe63ff48a992bc27527e6243f69cfb87e85d3c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Mon, 24 Mar 2025 08:26:01 +0100 Subject: [PATCH 126/408] MINOR: Upgrade to Apache POM 34 (#681) This PR updates to parent Apache POM 34 with new plugins versions. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1a93166ed6..7e69e143f7 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache apache - 33 + 34 org.apache.arrow From c2b2fbbc68daa4d627d8beb21fc1adbd5027af78 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Mar 2025 03:31:33 -0400 Subject: [PATCH 127/408] MINOR: [CI] Bump actions/upload-artifact from 4.6.1 to 4.6.2 (#688) Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 4.6.1 to 4.6.2. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index a1dbbfabe8..b04b958a40 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -71,7 +71,7 @@ jobs: run: | dev/release/run_rat.sh "${TAR_GZ}" - name: Upload source archive - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: release-source path: | @@ -154,7 +154,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf jni-linux-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: jni-linux-${{ matrix.platform.arch }} path: jni-linux-${{ matrix.platform.arch }}.tar.gz @@ -280,7 +280,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf jni-macos-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: jni-macos-${{ matrix.platform.arch }} path: jni-macos-${{ matrix.platform.arch }}.tar.gz @@ -358,7 +358,7 @@ jobs: shell: bash run: tar -cvzf jni-windows-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: jni-windows-${{ matrix.platform.arch }} path: jni-windows-${{ matrix.platform.arch }}.tar.gz @@ -430,12 +430,12 @@ jobs: cp -a target/site/apidocs reference tar -cvzf reference.tar.gz reference - name: Upload binaries - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: release-binaries path: binaries/* - name: Upload docs - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: reference path: reference.tar.gz @@ -473,7 +473,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf html.tar.gz -C docs/build html - name: Upload artifacts - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: release-html path: html.tar.gz From 9e07d3767496a4365e020231b2a1b4203b1f52ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Mar 2025 03:32:44 -0400 Subject: [PATCH 128/408] MINOR: [CI] Bump actions/download-artifact from 4.1.9 to 4.2.1 (#686) Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 4.1.9 to 4.2.1. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index b04b958a40..5b78cc9395 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -101,7 +101,7 @@ jobs: packages: write steps: - name: Download source archive - uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9 + uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 with: name: release-source - name: Extract source archive @@ -174,7 +174,7 @@ jobs: MACOSX_DEPLOYMENT_TARGET: "14.0" steps: - name: Download source archive - uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9 + uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 with: name: release-source - name: Extract source archive @@ -298,7 +298,7 @@ jobs: arch: "x86_64" steps: - name: Download source archive - uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9 + uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 with: name: release-source - name: Extract source archive @@ -371,7 +371,7 @@ jobs: - jni-windows steps: - name: Download artifacts - uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9 + uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 with: path: artifacts - name: Decompress artifacts @@ -452,11 +452,11 @@ jobs: with: cache: 'pip' - name: Download source archive - uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9 + uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 with: name: release-source - name: Download Javadocs - uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9 + uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 with: name: reference - name: Extract source archive @@ -521,7 +521,7 @@ jobs: cp ../.asf.yaml ./ git add .nojekyll .asf.yaml - name: Download - uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9 + uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 with: name: release-html - name: Extract @@ -557,7 +557,7 @@ jobs: - ubuntu-latest steps: - name: Download release artifacts - uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9 + uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 with: pattern: release-* - name: Verify @@ -591,7 +591,7 @@ jobs: contents: write steps: - name: Download release artifacts - uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9 + uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 with: pattern: release-* path: artifacts From 47288d95f4cd2f8206dfd59198e48ee72f70667b Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 24 Mar 2025 04:55:35 -0400 Subject: [PATCH 129/408] MINOR: Fix macOS JNI build (#689) ## What's Changed Update a sanity check in CI that's incorrectly failing the macOS JNI builds. Recent LLVM uses Z3. --- ci/scripts/jni_macos_build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/scripts/jni_macos_build.sh b/ci/scripts/jni_macos_build.sh index 1ad76266ee..65b255e2dc 100755 --- a/ci/scripts/jni_macos_build.sh +++ b/ci/scripts/jni_macos_build.sh @@ -153,6 +153,7 @@ archery linking check-dependencies \ --allow libncurses \ --allow libobjc \ --allow libz \ + --allow libz3 \ "arrow_cdata_jni/${normalized_arch}/libarrow_cdata_jni.dylib" \ "arrow_dataset_jni/${normalized_arch}/libarrow_dataset_jni.dylib" \ "arrow_orc_jni/${normalized_arch}/libarrow_orc_jni.dylib" \ From e74d9ad62ac693391315d49ffbf944cd4eddda66 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Mar 2025 11:05:36 +0100 Subject: [PATCH 130/408] MINOR: Bump com.google.protobuf:protobuf-bom from 4.30.0 to 4.30.1 (#675) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [com.google.protobuf:protobuf-bom](https://github.com/protocolbuffers/protobuf) from 4.30.0 to 4.30.1.
Commits
  • 0d815c5 Updating version.json and repo version numbers to: 30.1
  • ebb5224 Re-add system_python repo alias to MODULE.bazel (#20662)
  • 4747628 Remove Java runtime classes from kotlin release. (#20607)
  • 250c550 Fix python codegen crash when C++ features are used. (#20577)
  • 3576a1f Loosen py_proto_library check to be on the import path instead of full direct...
  • efa65c5 Merge pull request #20562 from protocolbuffers/30.x-202503042254
  • 63fc9d6 Updating version.json and repo version numbers to: 30.1-dev
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.protobuf:protobuf-bom&package-manager=maven&previous-version=4.30.0&new-version=4.30.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: JB Onofré --- flight/flight-integration-tests/src/shade/LICENSE.txt | 4 ++-- flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt | 4 ++-- pom.xml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flight/flight-integration-tests/src/shade/LICENSE.txt b/flight/flight-integration-tests/src/shade/LICENSE.txt index 6ee8645869..a23e96e9d8 100644 --- a/flight/flight-integration-tests/src/shade/LICENSE.txt +++ b/flight/flight-integration-tests/src/shade/LICENSE.txt @@ -272,11 +272,11 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- -This binary artifact contains Google Protobuf 4.30.0. +This binary artifact contains Google Protobuf 4.30.1. Copyright: Copyright 2008 Google Inc. All rights reserved. Home page: https://protobuf.dev/ -License: https://github.com/protocolbuffers/protobuf/blob/v4.30.0/LICENSE (BSD) +License: https://github.com/protocolbuffers/protobuf/blob/v4.30.1/LICENSE (BSD) License text: | Copyright 2008 Google Inc. All rights reserved. diff --git a/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt b/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt index c58a03831f..553fa802a1 100644 --- a/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt +++ b/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt @@ -234,11 +234,11 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- -This binary artifact contains Protobuf 4.30.0. +This binary artifact contains Protobuf 4.30.1. Copyright: Copyright 2008 Google Inc. All rights reserved. Home page: https://protobuf.dev/ -License: https://github.com/protocolbuffers/protobuf/blob/v4.30.0/LICENSE (BSD) +License: https://github.com/protocolbuffers/protobuf/blob/v4.30.1/LICENSE (BSD) License text: | Copyright 2008 Google Inc. All rights reserved. diff --git a/pom.xml b/pom.xml index 7e69e143f7..6a0f06e767 100644 --- a/pom.xml +++ b/pom.xml @@ -98,7 +98,7 @@ under the License. 33.4.0-jre 4.1.119.Final 1.71.0 - 4.30.0 + 4.30.1 2.18.3 3.4.1 25.2.10 From 07026f34da3d4e1eeed692850cfa931af6b9dc6a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Mar 2025 13:11:12 +0100 Subject: [PATCH 131/408] MINOR: Bump com.github.luben:zstd-jni from 1.5.7-1 to 1.5.7-2 (#679) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [com.github.luben:zstd-jni](https://github.com/luben/zstd-jni) from 1.5.7-1 to 1.5.7-2.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.github.luben:zstd-jni&package-manager=maven&previous-version=1.5.7-1&new-version=1.5.7-2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: JB Onofré --- compression/pom.xml | 2 +- tools/src/shade/LICENSE.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compression/pom.xml b/compression/pom.xml index 8d398fe0d4..6f60eb7d0a 100644 --- a/compression/pom.xml +++ b/compression/pom.xml @@ -55,7 +55,7 @@ under the License. com.github.luben zstd-jni - 1.5.7-1 + 1.5.7-2 diff --git a/tools/src/shade/LICENSE.txt b/tools/src/shade/LICENSE.txt index 2130dc4bf7..eef614e7f2 100644 --- a/tools/src/shade/LICENSE.txt +++ b/tools/src/shade/LICENSE.txt @@ -250,7 +250,7 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- -This binary artifact contains Luben Zstd 1.5.7-1. +This binary artifact contains Luben Zstd 1.5.7-2. Copyright: Copyright (c) 2015-present, Luben Karavelov/ All rights reserved. Home page: https://github.com/luben/zstd-jni From 4b9238d4e7658e9c54d9a9e1da0e6ff69744e8eb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Mar 2025 13:11:32 +0100 Subject: [PATCH 132/408] MINOR: Bump com.google.api.grpc:proto-google-common-protos from 2.53.0 to 2.54.1 (#677) Bumps [com.google.api.grpc:proto-google-common-protos](https://github.com/googleapis/sdk-platform-java) from 2.53.0 to 2.54.1.
Release notes

Sourced from com.google.api.grpc:proto-google-common-protos's releases.

v2.54.0

2.54.0 (2025-02-25)

Features

Bug Fixes

  • S2A gRPC flow creates ComputeEngineCredentials via newBuilder. (#3651) (29c061e)

Dependencies

  • update dependency ch.qos.logback:logback-core to v1.3.15 [security] (#3654) (093d867)
  • update google api dependencies (#3631) (48db2a1)
  • update google auth library dependencies to v1.33.1 (#3656) (f7877a5)
  • update google http client dependencies to v1.46.3 (#3657) (9d5b3b5)
  • update grpc to 1.70.0 (#3641) (ad26cf9)
  • update grpc to 1.70.0 (missed update) (#3658) (6ca0599)
  • Update opentelemetry-semconv to v1.29.0-alpha (#3635) (49ac09d)

Documentation

Changelog

Sourced from com.google.api.grpc:proto-google-common-protos's changelog.

Changelog

2.55.1 (2025-03-12)

Dependencies

  • revert "deps: update arrow.version to v18.2.0" (#3694) (2725744)

2.55.0 (2025-03-12)

Features

Bug Fixes

  • remove call credentials from call options if DirectPath (#3670) (5ede29c)

Dependencies

2.54.0 (2025-02-25)

Features

Bug Fixes

  • S2A gRPC flow creates ComputeEngineCredentials via newBuilder. (#3651) (29c061e)

Dependencies

  • update dependency ch.qos.logback:logback-core to v1.3.15 [security] (#3654) (093d867)
  • update google api dependencies (#3631) (48db2a1)
  • update google auth library dependencies to v1.33.1 (#3656) (f7877a5)
  • update google http client dependencies to v1.46.3 (#3657) (9d5b3b5)
  • update grpc to 1.70.0 (#3641) (ad26cf9)
  • update grpc to 1.70.0 (missed update) (#3658) (6ca0599)
  • Update opentelemetry-semconv to v1.29.0-alpha (#3635) (49ac09d)

... (truncated)

Commits
  • 95fb766 chore(main): release 2.46.1 (#3238)
  • a7781a6 deps: update dependency net.bytebuddy:byte-buddy to v1.15.2 (#3235)
  • f6e30a5 deps: update grpc dependencies to v1.68.0 (#3220)
  • ed976ff chore: update googleapis commit at Tue Sep 24 02:26:35 UTC 2024 (#3230)
  • 8b42993 deps: update junit5 monorepo to v5.11.1 (#3236)
  • 4803738 deps: update google api dependencies (#3224)
  • 69e3e38 deps: update google.cloud.opentelemetry.version to v0.32.0 (#3225)
  • 73d803b deps: update dependency black to v24.8.0 (#3234)
  • ab0976b deps: update dependency watchdog to v4.0.2 (#3232)
  • ce60bdf deps: update dependency pyyaml to v6.0.2 (#3231)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.api.grpc:proto-google-common-protos&package-manager=maven&previous-version=2.53.0&new-version=2.54.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml index aab685abc9..a32293a5e8 100644 --- a/flight/flight-core/pom.xml +++ b/flight/flight-core/pom.xml @@ -134,7 +134,7 @@ under the License. com.google.api.grpc proto-google-common-protos - 2.53.0 + 2.54.1 test From 40cc35ecb445bbfab07c256a6a5d7a262e122de4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Mar 2025 14:11:04 +0100 Subject: [PATCH 133/408] MINOR: Bump com.google.guava:guava-bom from 33.4.0-jre to 33.4.5-jre (#683) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [com.google.guava:guava-bom](https://github.com/google/guava) from 33.4.0-jre to 33.4.5-jre.
Release notes

Sourced from com.google.guava:guava-bom's releases.

33.4.5

Even if you're not upgrading from Guava 33.4.0 or earlier, still read the release notes for Guava 33.4.1. Those release notes contain information about Guava 33.4.5's effect on the module system.

Maven

<dependency>
  <groupId>com.google.guava</groupId>
  <artifactId>guava</artifactId>
  <version>33.4.5-jre</version>
  <!-- or, for Android: -->
  <version>33.4.5-android</version>
</dependency>

Jar files

Guava requires one runtime dependency, which you can download here:

Javadoc

JDiff

Changelog

  • Changed the Guava jar (plus guava-testlib and failureaccess jars) to be a modular jar. (7a71ea0bfa, 287c701a86)
  • Changed various classes to stop using sun.misc.Unsafe under Java 9+. (ee63055ddd, 80aab00dc5b7a36785f5e09b6a54397388980cde, 400af25292096746ed3f6164f0ff88209acbb19f, 71d0692d418a5dd001c9b3786275a5f1f94e1971, d1a3cd5037528a2ae990bfceed9cdd009fbc54de, b3bb29a54b8f13d6f6630b6cb929867adbf6b9a0, 1a300f6b2f7ba03ae9bc3620a80c4d4589c65b69)
    • Note that, if you use guava-android on the JVM (instead of using guava-jre), Guava will still try to use sun.misc.Unsafe. We will do further work on this in the future.
  • Belatedly updated the Public Suffix List data. (ee3b9c64382037f72b3a8341915cc64b87850b53, d25d62fc843ece1c3866859bc8639b815093eac8)

Special thanks to @​sgammon for his modularization efforts.

33.4.4

This is one of a series of releases that improve Guava's nullness annotations. For more information, including troubleshooting help, see the release notes for Guava 33.4.1. Most users can update directly to Guava 33.4.5.

Maven

</tr></table>

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.guava:guava-bom&package-manager=maven&previous-version=33.4.0-jre&new-version=33.4.5-jre)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: JB Onofré --- flight/flight-integration-tests/src/shade/LICENSE.txt | 2 +- flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt | 2 +- pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flight/flight-integration-tests/src/shade/LICENSE.txt b/flight/flight-integration-tests/src/shade/LICENSE.txt index a23e96e9d8..484542fd20 100644 --- a/flight/flight-integration-tests/src/shade/LICENSE.txt +++ b/flight/flight-integration-tests/src/shade/LICENSE.txt @@ -265,7 +265,7 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- -This binary artifact contains Guava 33.4.0-jre. +This binary artifact contains Guava 33.4.5-jre. Home page: https://github.com/google/guava License: https://www.apache.org/licenses/LICENSE-2.0 diff --git a/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt b/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt index 553fa802a1..1048fa8a8d 100644 --- a/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt +++ b/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt @@ -227,7 +227,7 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- -This binary artifact contains Guava 33.4.0-jre. +This binary artifact contains Guava 33.4.5-jre. Home page: https://github.com/google/guava License: https://www.apache.org/licenses/LICENSE-2.0 diff --git a/pom.xml b/pom.xml index 6a0f06e767..8614db035e 100644 --- a/pom.xml +++ b/pom.xml @@ -95,7 +95,7 @@ under the License. 1.9.0 5.12.1 2.0.17 - 33.4.0-jre + 33.4.5-jre 4.1.119.Final 1.71.0 4.30.1 From 48a20ba91605ebb44df845671782acd8f869bd4b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Mar 2025 15:56:39 +0100 Subject: [PATCH 134/408] MINOR: Bump logback.version from 1.5.17 to 1.5.18 (#684) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps `logback.version` from 1.5.17 to 1.5.18. Updates `ch.qos.logback:logback-classic` from 1.5.17 to 1.5.18
Release notes

Sourced from ch.qos.logback:logback-classic's releases.

Logback 1.5.18

2025-03-18 Release of logback version 1.5.18

• Added support for XZ compression for archived log files. Note that XZ compression requires Tukaani project's XZ library for Java. In case XZ compression is requested but the XZ library is missing, then logback will substitute GZ compression as a fallback. This feature was requested in issues/755.

• Removed references to java.security.AccessController class. This class has been deprecated for some time and is slated for removal in future JDK versions.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit b2a02f065379a9b1ba5ff837fc08913b744774bc associated with the tag v_1.5.18. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits
  • b2a02f0 prepare release 1.5.18
  • 991de58 remove references to AccessController marked for deletion in the JDK
  • f54ab16 If compression mode is XZ but the XZ library is missing, then fallback to GZ ...
  • fb45971 add support for XZ compression
  • 31c1f55 add xz compression support with tests
  • 8968d0f introduce strategy based compression
  • 834059c start work on 1.5.18-SNAPSHOT
  • See full diff in compare view

Updates `ch.qos.logback:logback-core` from 1.5.17 to 1.5.18
Release notes

Sourced from ch.qos.logback:logback-core's releases.

Logback 1.5.18

2025-03-18 Release of logback version 1.5.18

• Added support for XZ compression for archived log files. Note that XZ compression requires Tukaani project's XZ library for Java. In case XZ compression is requested but the XZ library is missing, then logback will substitute GZ compression as a fallback. This feature was requested in issues/755.

• Removed references to java.security.AccessController class. This class has been deprecated for some time and is slated for removal in future JDK versions.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit b2a02f065379a9b1ba5ff837fc08913b744774bc associated with the tag v_1.5.18. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits
  • b2a02f0 prepare release 1.5.18
  • 991de58 remove references to AccessController marked for deletion in the JDK
  • f54ab16 If compression mode is XZ but the XZ library is missing, then fallback to GZ ...
  • fb45971 add support for XZ compression
  • 31c1f55 add xz compression support with tests
  • 8968d0f introduce strategy based compression
  • 834059c start work on 1.5.18-SNAPSHOT
  • See full diff in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8614db035e..d74ca1e5a1 100644 --- a/pom.xml +++ b/pom.xml @@ -110,7 +110,7 @@ under the License. true 2.31.0 3.49.1 - 1.5.17 + 1.5.18 none -Xdoclint:none From dbcc81cf6e42d316aa592a6af73ad30c011937a9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Mar 2025 21:29:22 -0400 Subject: [PATCH 135/408] MINOR: Bump error_prone_core.version from 2.31.0 to 2.37.0 (#682) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps `error_prone_core.version` from 2.31.0 to 2.37.0. Updates `com.google.errorprone:error_prone_annotations` from 2.31.0 to 2.37.0 --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: JB Onofré Co-authored-by: JB Onofré --- pom.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d74ca1e5a1..afb753dcb6 100644 --- a/pom.xml +++ b/pom.xml @@ -108,7 +108,7 @@ under the License. 2 10.21.4 true - 2.31.0 + 2.37.0 3.49.1 1.5.18 none @@ -845,6 +845,7 @@ under the License. --> error-prone + [17,) !m2e.version @@ -868,6 +869,7 @@ under the License. -J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED + --should-stop=ifError=FLOW From 1e62e3bf663838d9dc77d5f34b837e22e1902622 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Mar 2025 16:34:44 +0900 Subject: [PATCH 136/408] MINOR: Bump com.puppycrawl.tools:checkstyle from 10.21.4 to 10.22.0 (#694) Bumps [com.puppycrawl.tools:checkstyle](https://github.com/checkstyle/checkstyle) from 10.21.4 to 10.22.0.
Release notes

Sourced from com.puppycrawl.tools:checkstyle's releases.

checkstyle-10.22.0

Checkstyle 10.22.0 - https://checkstyle.org/releasenotes.html#Release_10.22.0

Breaking backward compatibility:

#9280 - JavadocVariable: new property 'accessModifiers' as substitution of 'scope' and 'excludeScope' #15499 - Change default or IllegalIdentifierName

Bug fixes:

#16641 - FileContents.getJavadocBefore(): Comments should not be skipped if it is not alone in line #16385 - JavadocTagContinuationIndentation Ignore indentation check when HTML tag break line #16628 - use SLL prediction mode for fast javadoc parsing to improve performance #43 - JavadocMethod: Javadoc Not Detected Above Multiline Comments #12817 - Incorrect Indentation errors for expression switches with google_checks.xml #6637 - SuppressWarningsHolder aliasList members don't act like aliases #13043 - Make references optional for link and linkplain tags #16005 - Parse errors if ``@see spans multiple lines #14446 - Parse error when Javadoc contains `@`snippet with code example that uses Java annotation

... (truncated)

Commits
  • b20ca3b [maven-release-plugin] prepare release checkstyle-10.22.0
  • b5c86f5 doc: release notes for 10.22.0
  • 1a6caf2 Issue #14631: Updated INPUT_HTML_TAG_NAME to new AST format
  • 82c21d5 Issue #16666: Updated Broken Link - Javadoc OpenJDK 8 Report in website, 404 ...
  • 23c3c59 Issue #16641: FileContents.getJavadocBefore should skip block comments only i...
  • 9fb5504 Issue #14857: Github generate site fails to generate links with anchors
  • e1c89b9 infra: update google-java-format.yml to 1.26.0 formatter
  • 06074e5 Issue #14631: Updated LITERAL_INCLUDE in JavadocTokenTypes.java to new AST fo...
  • b5c64b2 Issue #11163: Enforce file size for javaParser
  • ae8fde4 Issue #14631: Updated TR_TAG_END in JavadocTokenTypes.java to new AST format
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.puppycrawl.tools:checkstyle&package-manager=maven&previous-version=10.21.4&new-version=10.22.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index afb753dcb6..df32105804 100644 --- a/pom.xml +++ b/pom.xml @@ -106,7 +106,7 @@ under the License. 5.16.1 2 - 10.21.4 + 10.22.0 true 2.37.0 3.49.1 From c57184a0fdd9d807ead80236be8f83008feab4b8 Mon Sep 17 00:00:00 2001 From: Martin Traverse Date: Wed, 2 Apr 2025 06:12:36 +0100 Subject: [PATCH 137/408] GH-615: Produce Avro core data types out of Arrow VSR (#638) ## What's Changed Per discussion in #615 , here is a first take on the core producers to generate Avro data from Arrow vectors. There are a few points I'd like to clarify before going further: * Nullability. Avro only understands nullable types as unions but that is not normally how they will be if the data comes from other sources. I have added a special NullableProducer to handle nullable vectors which are not unions. We will need something equivalent in the consumers and probably a setting in the AvroToArrowConfig to control it on read, defaulting to current behaviour. I have also added special handling for nullable unions, because unions can't be nested (i.e. you can't nest "type | null" as a type inside a union). I can add consumers to handle both (unions and regular types) for review, if that sounds right? At the moment the schema for nullable fields gets quite mangled on a round trip! * Arrow has a lot more types than Avro, at the level of minor / vector types. Going Avro -> Arrow we just pick the direct equivalent. Going Arrow -> Avro, we could cast silently if there is no loss of precision. E.g. TinyInt and SmallInt -> Int and so on. For types like e.g. Decimal256 and LargeVarChar we could write out safely but would need support in the consumers to read back the wider types. I could start by adding the safe conversions now and we could come back to the wide types in a later PR maybe? * I have made Producer inherit AutoClosable the same as Consumer does. Not sure if that is always what you want though - it will free the buffers in the VSR, but you might want to keep the VSR after you are finished with IO. Do we need something like detachValueVector() to go with resetValueVector()? Calling close() after detach would not affect the vector. * Type information is inferred from the list of vectors, using minor types. We'll also need to generate the Avro schema, the input for that would be a list of fields. I haven't done it yet but will do if that sounds right. * Dictionary encoding for enums not implemented yet, I'll add it if the rest looks good. Caveat is that dictionaries must be fixed before the encoding starts if we are writing out the whole file in one go (i.e. if the Avro schema is at the start of the container file). If the schema is saved separately that limitation need not apply, we could provide the schema once encoding is finished. Please do let me know if this is going in the right direction + any comments. If it is I will add the missing pieces and start the exhaustive test coverage to mirror the consumers. Once it's done this PR should get us to the point where we can round trip the contents of an individual block for most data types, but it does not address the container format. Closes #615. --------- Co-authored-by: Martin Traverse --- adapter/avro/src/main/java/module-info.java | 2 + .../arrow/adapter/avro/ArrowToAvroUtils.java | 524 +++ .../avro/producers/AvroBigIntProducer.java | 52 + .../avro/producers/AvroBooleanProducer.java | 38 + .../avro/producers/AvroBytesProducer.java | 45 + .../avro/producers/AvroEnumProducer.java | 38 + .../AvroFixedSizeBinaryProducer.java | 54 + .../producers/AvroFixedSizeListProducer.java | 78 + .../avro/producers/AvroFloat2Producer.java | 40 + .../avro/producers/AvroFloat4Producer.java | 39 + .../avro/producers/AvroFloat8Producer.java | 39 + .../avro/producers/AvroIntProducer.java | 52 + .../avro/producers/AvroListProducer.java | 73 + .../avro/producers/AvroMapProducer.java | 71 + .../avro/producers/AvroNullProducer.java | 36 + .../avro/producers/AvroNullableProducer.java | 77 + .../avro/producers/AvroSmallIntProducer.java | 39 + .../avro/producers/AvroStringProducer.java | 48 + .../avro/producers/AvroStructProducer.java | 75 + .../avro/producers/AvroTinyIntProducer.java | 39 + .../avro/producers/AvroUint1Producer.java | 38 + .../avro/producers/AvroUint2Producer.java | 38 + .../avro/producers/AvroUint4Producer.java | 40 + .../avro/producers/AvroUint8Producer.java | 42 + .../avro/producers/BaseAvroProducer.java | 73 + .../avro/producers/CompositeAvroProducer.java | 54 + .../adapter/avro/producers/Producer.java | 49 + .../logical/AvroDateDayProducer.java | 33 + .../logical/AvroDateMilliProducer.java | 49 + .../logical/AvroDecimal256Producer.java | 46 + .../logical/AvroDecimalProducer.java | 59 + .../logical/AvroTimeMicroProducer.java | 34 + .../logical/AvroTimeMilliProducer.java | 34 + .../logical/AvroTimeNanoProducer.java | 46 + .../logical/AvroTimeSecProducer.java | 52 + .../logical/AvroTimestampMicroProducer.java | 34 + .../logical/AvroTimestampMicroTzProducer.java | 35 + .../logical/AvroTimestampMilliProducer.java | 34 + .../logical/AvroTimestampMilliTzProducer.java | 35 + .../logical/AvroTimestampNanoProducer.java | 34 + .../logical/AvroTimestampNanoTzProducer.java | 35 + .../logical/AvroTimestampSecProducer.java | 52 + .../logical/AvroTimestampSecTzProducer.java | 55 + .../adapter/avro/ArrowToAvroDataTest.java | 2820 +++++++++++++++++ .../adapter/avro/ArrowToAvroSchemaTest.java | 1392 ++++++++ .../adapter/avro/TestWriteReadAvroRecord.java | 101 + 46 files changed, 6773 insertions(+) create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/ArrowToAvroUtils.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroBigIntProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroBooleanProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroBytesProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroEnumProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroFixedSizeBinaryProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroFixedSizeListProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroFloat2Producer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroFloat4Producer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroFloat8Producer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroIntProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroListProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroMapProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroNullProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroNullableProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroSmallIntProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroStringProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroStructProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroTinyIntProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroUint1Producer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroUint2Producer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroUint4Producer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroUint8Producer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/BaseAvroProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/CompositeAvroProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/Producer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroDateDayProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroDateMilliProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroDecimal256Producer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroDecimalProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimeMicroProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimeMilliProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimeNanoProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimeSecProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampMicroProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampMicroTzProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampMilliProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampMilliTzProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampNanoProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampNanoTzProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampSecProducer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampSecTzProducer.java create mode 100644 adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroDataTest.java create mode 100644 adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroSchemaTest.java diff --git a/adapter/avro/src/main/java/module-info.java b/adapter/avro/src/main/java/module-info.java index 5c6204be60..fee6c72199 100644 --- a/adapter/avro/src/main/java/module-info.java +++ b/adapter/avro/src/main/java/module-info.java @@ -18,6 +18,8 @@ module org.apache.arrow.adapter.avro { exports org.apache.arrow.adapter.avro.consumers; exports org.apache.arrow.adapter.avro.consumers.logical; + exports org.apache.arrow.adapter.avro.producers; + exports org.apache.arrow.adapter.avro.producers.logical; exports org.apache.arrow.adapter.avro; requires org.apache.arrow.memory.core; diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/ArrowToAvroUtils.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/ArrowToAvroUtils.java new file mode 100644 index 0000000000..6f0cb5cffc --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/ArrowToAvroUtils.java @@ -0,0 +1,524 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro; + +import java.util.ArrayList; +import java.util.List; +import org.apache.arrow.adapter.avro.producers.AvroBigIntProducer; +import org.apache.arrow.adapter.avro.producers.AvroBooleanProducer; +import org.apache.arrow.adapter.avro.producers.AvroBytesProducer; +import org.apache.arrow.adapter.avro.producers.AvroFixedSizeBinaryProducer; +import org.apache.arrow.adapter.avro.producers.AvroFixedSizeListProducer; +import org.apache.arrow.adapter.avro.producers.AvroFloat2Producer; +import org.apache.arrow.adapter.avro.producers.AvroFloat4Producer; +import org.apache.arrow.adapter.avro.producers.AvroFloat8Producer; +import org.apache.arrow.adapter.avro.producers.AvroIntProducer; +import org.apache.arrow.adapter.avro.producers.AvroListProducer; +import org.apache.arrow.adapter.avro.producers.AvroMapProducer; +import org.apache.arrow.adapter.avro.producers.AvroNullProducer; +import org.apache.arrow.adapter.avro.producers.AvroNullableProducer; +import org.apache.arrow.adapter.avro.producers.AvroSmallIntProducer; +import org.apache.arrow.adapter.avro.producers.AvroStringProducer; +import org.apache.arrow.adapter.avro.producers.AvroStructProducer; +import org.apache.arrow.adapter.avro.producers.AvroTinyIntProducer; +import org.apache.arrow.adapter.avro.producers.AvroUint1Producer; +import org.apache.arrow.adapter.avro.producers.AvroUint2Producer; +import org.apache.arrow.adapter.avro.producers.AvroUint4Producer; +import org.apache.arrow.adapter.avro.producers.AvroUint8Producer; +import org.apache.arrow.adapter.avro.producers.BaseAvroProducer; +import org.apache.arrow.adapter.avro.producers.CompositeAvroProducer; +import org.apache.arrow.adapter.avro.producers.Producer; +import org.apache.arrow.adapter.avro.producers.logical.AvroDateDayProducer; +import org.apache.arrow.adapter.avro.producers.logical.AvroDateMilliProducer; +import org.apache.arrow.adapter.avro.producers.logical.AvroDecimal256Producer; +import org.apache.arrow.adapter.avro.producers.logical.AvroDecimalProducer; +import org.apache.arrow.adapter.avro.producers.logical.AvroTimeMicroProducer; +import org.apache.arrow.adapter.avro.producers.logical.AvroTimeMilliProducer; +import org.apache.arrow.adapter.avro.producers.logical.AvroTimeNanoProducer; +import org.apache.arrow.adapter.avro.producers.logical.AvroTimeSecProducer; +import org.apache.arrow.adapter.avro.producers.logical.AvroTimestampMicroProducer; +import org.apache.arrow.adapter.avro.producers.logical.AvroTimestampMicroTzProducer; +import org.apache.arrow.adapter.avro.producers.logical.AvroTimestampMilliProducer; +import org.apache.arrow.adapter.avro.producers.logical.AvroTimestampMilliTzProducer; +import org.apache.arrow.adapter.avro.producers.logical.AvroTimestampNanoProducer; +import org.apache.arrow.adapter.avro.producers.logical.AvroTimestampNanoTzProducer; +import org.apache.arrow.adapter.avro.producers.logical.AvroTimestampSecProducer; +import org.apache.arrow.adapter.avro.producers.logical.AvroTimestampSecTzProducer; +import org.apache.arrow.util.Preconditions; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.DateDayVector; +import org.apache.arrow.vector.DateMilliVector; +import org.apache.arrow.vector.Decimal256Vector; +import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.FixedSizeBinaryVector; +import org.apache.arrow.vector.Float2Vector; +import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.NullVector; +import org.apache.arrow.vector.SmallIntVector; +import org.apache.arrow.vector.TimeMicroVector; +import org.apache.arrow.vector.TimeMilliVector; +import org.apache.arrow.vector.TimeNanoVector; +import org.apache.arrow.vector.TimeSecVector; +import org.apache.arrow.vector.TimeStampMicroTZVector; +import org.apache.arrow.vector.TimeStampMicroVector; +import org.apache.arrow.vector.TimeStampMilliTZVector; +import org.apache.arrow.vector.TimeStampMilliVector; +import org.apache.arrow.vector.TimeStampNanoTZVector; +import org.apache.arrow.vector.TimeStampNanoVector; +import org.apache.arrow.vector.TimeStampSecTZVector; +import org.apache.arrow.vector.TimeStampSecVector; +import org.apache.arrow.vector.TinyIntVector; +import org.apache.arrow.vector.UInt1Vector; +import org.apache.arrow.vector.UInt2Vector; +import org.apache.arrow.vector.UInt4Vector; +import org.apache.arrow.vector.UInt8Vector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.complex.FixedSizeListVector; +import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.StructVector; +import org.apache.arrow.vector.types.FloatingPointPrecision; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.avro.LogicalType; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; +import org.apache.avro.SchemaBuilder; + +public class ArrowToAvroUtils { + + public static final String GENERIC_RECORD_TYPE_NAME = "GenericRecord"; + + /** + * Create an Avro record schema for a given list of Arrow fields. + * + *

This method currently performs following type mapping for Avro data types to corresponding + * Arrow data types. + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Arrow typeAvro encoding
ArrowType.NullNULL
ArrowType.BoolBOOLEAN
ArrowType.Int(64 bit, unsigned 32 bit)LONG
ArrowType.Int(signed 32 bit, < 32 bit)INT
ArrowType.FloatingPoint(double)DOUBLE
ArrowType.FloatingPoint(single, half)FLOAT
ArrowType.Utf8STRING
ArrowType.LargeUtf8STRING
ArrowType.BinaryBYTES
ArrowType.LargeBinaryBYTES
ArrowType.FixedSizeBinaryFIXED
ArrowType.Decimaldecimal (FIXED)
ArrowType.Datedate (INT)
ArrowType.Time (SEC | MILLI)time-millis (INT)
ArrowType.Time (MICRO | NANO)time-micros (LONG)
ArrowType.Timestamp (NANOSECONDS, TZ != NULL)time-nanos (LONG)
ArrowType.Timestamp (MICROSECONDS, TZ != NULL)time-micros (LONG)
ArrowType.Timestamp (MILLISECONDS | SECONDS, TZ != NULL)time-millis (LONG)
ArrowType.Timestamp (NANOSECONDS, TZ == NULL)local-time-nanos (LONG)
ArrowType.Timestamp (MICROSECONDS, TZ == NULL)local-time-micros (LONG)
ArrowType.Timestamp (MILLISECONDS | SECONDS, TZ == NULL)local-time-millis (LONG)
ArrowType.Durationduration (FIXED)
ArrowType.Intervalduration (FIXED)
ArrowType.Structrecord
ArrowType.Listarray
ArrowType.LargeListarray
ArrowType.FixedSizeListarray
ArrowType.Mapmap
ArrowType.Unionunion
+ * + *

Nullable fields are represented as a union of [base-type | null]. Special treatment is given + * to nullability of unions - a union is considered nullable if any of its child fields are + * nullable. The schema for a nullable union will always contain a null type as its first member, + * with none of the child types being nullable. + * + *

List fields must contain precisely one child field, which may be nullable. Map fields are + * represented as a list of structs, where the struct fields are "key" and "value". The key field + * must always be of type STRING (Utf8) and cannot be nullable. The value can be of any type and + * may be nullable. Record types must contain at least one child field and cannot contain multiple + * fields with the same name + * + * @param arrowFields The arrow fields used to generate the Avro schema + * @param typeName Name of the top level Avro record type + * @param namespace Namespace of the top level Avro record type + * @return An Avro record schema for the given list of fields, with the specified name and + * namespace + */ + public static Schema createAvroSchema( + List arrowFields, String typeName, String namespace) { + SchemaBuilder.RecordBuilder assembler = + SchemaBuilder.record(typeName).namespace(namespace); + return buildRecordSchema(assembler, arrowFields, namespace); + } + + /** Overload provided for convenience, sets namespace = null. */ + public static Schema createAvroSchema(List arrowFields, String typeName) { + return createAvroSchema(arrowFields, typeName, null); + } + + /** Overload provided for convenience, sets name = GENERIC_RECORD_TYPE_NAME. */ + public static Schema createAvroSchema(List arrowFields) { + return createAvroSchema(arrowFields, GENERIC_RECORD_TYPE_NAME); + } + + private static T buildRecordSchema( + SchemaBuilder.RecordBuilder builder, List fields, String namespace) { + if (fields.isEmpty()) { + throw new IllegalArgumentException("Record field must have at least one child field"); + } + SchemaBuilder.FieldAssembler assembler = builder.namespace(namespace).fields(); + for (Field field : fields) { + assembler = buildFieldSchema(assembler, field, namespace); + } + return assembler.endRecord(); + } + + private static SchemaBuilder.FieldAssembler buildFieldSchema( + SchemaBuilder.FieldAssembler assembler, Field field, String namespace) { + + return assembler + .name(field.getName()) + .type(buildTypeSchema(SchemaBuilder.builder(), field, namespace)) + .noDefault(); + } + + private static T buildTypeSchema( + SchemaBuilder.TypeBuilder builder, Field field, String namespace) { + + // Nullable unions need special handling, since union types cannot be directly nested + if (field.getType().getTypeID() == ArrowType.ArrowTypeID.Union) { + boolean unionNullable = field.getChildren().stream().anyMatch(Field::isNullable); + if (unionNullable) { + SchemaBuilder.UnionAccumulator union = builder.unionOf().nullType(); + return addTypesToUnion(union, field.getChildren(), namespace); + } else { + Field headType = field.getChildren().get(0); + List tailTypes = field.getChildren().subList(1, field.getChildren().size()); + SchemaBuilder.UnionAccumulator union = + buildBaseTypeSchema(builder.unionOf(), headType, namespace); + return addTypesToUnion(union, tailTypes, namespace); + } + } else if (field.isNullable()) { + return buildBaseTypeSchema(builder.nullable(), field, namespace); + } else { + return buildBaseTypeSchema(builder, field, namespace); + } + } + + private static T buildArraySchema( + SchemaBuilder.ArrayBuilder builder, Field listField, String namespace) { + if (listField.getChildren().size() != 1) { + throw new IllegalArgumentException("List field must have exactly one child field"); + } + Field itemField = listField.getChildren().get(0); + return buildTypeSchema(builder.items(), itemField, namespace); + } + + private static T buildMapSchema( + SchemaBuilder.MapBuilder builder, Field mapField, String namespace) { + if (mapField.getChildren().size() != 1) { + throw new IllegalArgumentException("Map field must have exactly one child field"); + } + Field entriesField = mapField.getChildren().get(0); + if (mapField.getChildren().size() != 1) { + throw new IllegalArgumentException("Map entries must have exactly two child fields"); + } + Field keyField = entriesField.getChildren().get(0); + Field valueField = entriesField.getChildren().get(1); + if (keyField.getType().getTypeID() != ArrowType.ArrowTypeID.Utf8 || keyField.isNullable()) { + throw new IllegalArgumentException( + "Map keys must be of type string and cannot be nullable for conversion to Avro"); + } + return buildTypeSchema(builder.values(), valueField, namespace); + } + + private static T buildBaseTypeSchema( + SchemaBuilder.BaseTypeBuilder builder, Field field, String namespace) { + + ArrowType.ArrowTypeID typeID = field.getType().getTypeID(); + + switch (typeID) { + case Null: + return builder.nullType(); + + case Bool: + return builder.booleanType(); + + case Int: + ArrowType.Int intType = (ArrowType.Int) field.getType(); + if (intType.getBitWidth() > 32 || (intType.getBitWidth() == 32 && !intType.getIsSigned())) { + return builder.longType(); + } else { + return builder.intType(); + } + + case FloatingPoint: + ArrowType.FloatingPoint floatType = (ArrowType.FloatingPoint) field.getType(); + if (floatType.getPrecision() == FloatingPointPrecision.DOUBLE) { + return builder.doubleType(); + } else { + return builder.floatType(); + } + + case Utf8: + return builder.stringType(); + + case Binary: + return builder.bytesType(); + + case FixedSizeBinary: + ArrowType.FixedSizeBinary fixedType = (ArrowType.FixedSizeBinary) field.getType(); + String fixedTypeName = field.getName(); + int fixedTypeWidth = fixedType.getByteWidth(); + return builder.fixed(fixedTypeName).size(fixedTypeWidth); + + case Decimal: + ArrowType.Decimal decimalType = (ArrowType.Decimal) field.getType(); + return builder.type( + LogicalTypes.decimal(decimalType.getPrecision(), decimalType.getScale()) + .addToSchema( + Schema.createFixed( + field.getName(), namespace, "", decimalType.getBitWidth() / 8))); + + case Date: + return builder.type(LogicalTypes.date().addToSchema(Schema.create(Schema.Type.INT))); + + case Time: + ArrowType.Time timeType = (ArrowType.Time) field.getType(); + if ((timeType.getUnit() == TimeUnit.SECOND || timeType.getUnit() == TimeUnit.MILLISECOND)) { + // Second and millisecond time types are encoded as time-millis (INT) + return builder.type( + LogicalTypes.timeMillis().addToSchema(Schema.create(Schema.Type.INT))); + } else { + // All other time types (micro, nano) are encoded as time-micros (LONG) + return builder.type( + LogicalTypes.timeMicros().addToSchema(Schema.create(Schema.Type.LONG))); + } + + case Timestamp: + ArrowType.Timestamp timestampType = (ArrowType.Timestamp) field.getType(); + LogicalType timestampLogicalType = timestampLogicalType(timestampType); + return builder.type(timestampLogicalType.addToSchema(Schema.create(Schema.Type.LONG))); + + case Struct: + String childNamespace = + namespace == null ? field.getName() : namespace + "." + field.getName(); + return buildRecordSchema( + builder.record(field.getName()), field.getChildren(), childNamespace); + + case List: + case FixedSizeList: + return buildArraySchema(builder.array(), field, namespace); + + case Map: + return buildMapSchema(builder.map(), field, namespace); + + default: + throw new IllegalArgumentException( + "Element type not supported for Avro conversion: " + typeID.name()); + } + } + + private static T addTypesToUnion( + SchemaBuilder.UnionAccumulator accumulator, List unionFields, String namespace) { + for (var field : unionFields) { + accumulator = buildBaseTypeSchema(accumulator.and(), field, namespace); + } + return accumulator.endUnion(); + } + + private static LogicalType timestampLogicalType(ArrowType.Timestamp timestampType) { + boolean zoneAware = timestampType.getTimezone() != null; + if (timestampType.getUnit() == TimeUnit.NANOSECOND) { + return zoneAware ? LogicalTypes.timestampNanos() : LogicalTypes.localTimestampNanos(); + } else if (timestampType.getUnit() == TimeUnit.MICROSECOND) { + return zoneAware ? LogicalTypes.timestampMicros() : LogicalTypes.localTimestampMicros(); + } else { + // Timestamp in seconds will be cast to milliseconds, Avro does not support seconds + return zoneAware ? LogicalTypes.timestampMillis() : LogicalTypes.localTimestampMillis(); + } + } + + /** + * Create a composite Avro producer for a set of field vectors (typically the root set of a VSR). + * + * @param vectors The vectors that will be used to produce Avro data + * @return The resulting composite Avro producer + */ + public static CompositeAvroProducer createCompositeProducer(List vectors) { + + List> producers = new ArrayList<>(vectors.size()); + + for (FieldVector vector : vectors) { + BaseAvroProducer producer = createProducer(vector); + producers.add(producer); + } + + return new CompositeAvroProducer(producers); + } + + private static BaseAvroProducer createProducer(FieldVector vector) { + boolean nullable = vector.getField().isNullable(); + return createProducer(vector, nullable); + } + + private static BaseAvroProducer createProducer(FieldVector vector, boolean nullable) { + + Preconditions.checkNotNull(vector, "Arrow vector object can't be null"); + + final Types.MinorType minorType = vector.getMinorType(); + + // Avro understands nullable types as a union of type | null + // Most nullable fields in a VSR will not be unions, so provide a special wrapper + if (nullable && minorType != Types.MinorType.UNION) { + final BaseAvroProducer innerProducer = createProducer(vector, false); + return new AvroNullableProducer<>(innerProducer); + } + + switch (minorType) { + case NULL: + return new AvroNullProducer((NullVector) vector); + case BIT: + return new AvroBooleanProducer((BitVector) vector); + case TINYINT: + return new AvroTinyIntProducer((TinyIntVector) vector); + case SMALLINT: + return new AvroSmallIntProducer((SmallIntVector) vector); + case INT: + return new AvroIntProducer((IntVector) vector); + case BIGINT: + return new AvroBigIntProducer((BigIntVector) vector); + case UINT1: + return new AvroUint1Producer((UInt1Vector) vector); + case UINT2: + return new AvroUint2Producer((UInt2Vector) vector); + case UINT4: + return new AvroUint4Producer((UInt4Vector) vector); + case UINT8: + return new AvroUint8Producer((UInt8Vector) vector); + case FLOAT2: + return new AvroFloat2Producer((Float2Vector) vector); + case FLOAT4: + return new AvroFloat4Producer((Float4Vector) vector); + case FLOAT8: + return new AvroFloat8Producer((Float8Vector) vector); + case VARBINARY: + return new AvroBytesProducer((VarBinaryVector) vector); + case FIXEDSIZEBINARY: + return new AvroFixedSizeBinaryProducer((FixedSizeBinaryVector) vector); + case VARCHAR: + return new AvroStringProducer((VarCharVector) vector); + + // Logical types + + case DECIMAL: + return new AvroDecimalProducer((DecimalVector) vector); + case DECIMAL256: + return new AvroDecimal256Producer((Decimal256Vector) vector); + case DATEDAY: + return new AvroDateDayProducer((DateDayVector) vector); + case DATEMILLI: + return new AvroDateMilliProducer((DateMilliVector) vector); + case TIMESEC: + return new AvroTimeSecProducer((TimeSecVector) vector); + case TIMEMILLI: + return new AvroTimeMilliProducer((TimeMilliVector) vector); + case TIMEMICRO: + return new AvroTimeMicroProducer((TimeMicroVector) vector); + case TIMENANO: + return new AvroTimeNanoProducer((TimeNanoVector) vector); + case TIMESTAMPSEC: + return new AvroTimestampSecProducer((TimeStampSecVector) vector); + case TIMESTAMPMILLI: + return new AvroTimestampMilliProducer((TimeStampMilliVector) vector); + case TIMESTAMPMICRO: + return new AvroTimestampMicroProducer((TimeStampMicroVector) vector); + case TIMESTAMPNANO: + return new AvroTimestampNanoProducer((TimeStampNanoVector) vector); + case TIMESTAMPSECTZ: + return new AvroTimestampSecTzProducer((TimeStampSecTZVector) vector); + case TIMESTAMPMILLITZ: + return new AvroTimestampMilliTzProducer((TimeStampMilliTZVector) vector); + case TIMESTAMPMICROTZ: + return new AvroTimestampMicroTzProducer((TimeStampMicroTZVector) vector); + case TIMESTAMPNANOTZ: + return new AvroTimestampNanoTzProducer((TimeStampNanoTZVector) vector); + + // Complex types + + case STRUCT: + StructVector structVector = (StructVector) vector; + List childVectors = structVector.getChildrenFromFields(); + Producer[] childProducers = new Producer[childVectors.size()]; + for (int i = 0; i < childVectors.size(); i++) { + FieldVector childVector = childVectors.get(i); + childProducers[i] = createProducer(childVector, childVector.getField().isNullable()); + } + return new AvroStructProducer(structVector, childProducers); + + case LIST: + ListVector listVector = (ListVector) vector; + FieldVector itemVector = listVector.getDataVector(); + Producer itemProducer = createProducer(itemVector, itemVector.getField().isNullable()); + return new AvroListProducer(listVector, itemProducer); + + case FIXED_SIZE_LIST: + FixedSizeListVector fixedListVector = (FixedSizeListVector) vector; + FieldVector fixedItemVector = fixedListVector.getDataVector(); + Producer fixedItemProducer = + createProducer(fixedItemVector, fixedItemVector.getField().isNullable()); + return new AvroFixedSizeListProducer(fixedListVector, fixedItemProducer); + + case MAP: + MapVector mapVector = (MapVector) vector; + StructVector entryVector = (StructVector) mapVector.getDataVector(); + Types.MinorType keyType = entryVector.getChildrenFromFields().get(0).getMinorType(); + if (keyType != Types.MinorType.VARCHAR) { + throw new IllegalArgumentException("MAP key type must be VARCHAR for Avro encoding"); + } + VarCharVector keyVector = (VarCharVector) entryVector.getChildrenFromFields().get(0); + FieldVector valueVector = entryVector.getChildrenFromFields().get(1); + Producer keyProducer = new AvroStringProducer(keyVector); + Producer valueProducer = + createProducer(valueVector, valueVector.getField().isNullable()); + Producer entryProducer = + new AvroStructProducer(entryVector, new Producer[] {keyProducer, valueProducer}); + return new AvroMapProducer(mapVector, entryProducer); + + // Support for UNION and DENSEUNION is not currently available + // This is pending fixes in the implementation of the union vectors themselves + // https://github.com/apache/arrow-java/issues/108 + + default: + // Not all Arrow types are supported for encoding (yet)! + String error = + String.format( + "Encoding Arrow type %s to Avro is not currently supported", minorType.name()); + throw new UnsupportedOperationException(error); + } + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroBigIntProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroBigIntProducer.java new file mode 100644 index 0000000000..9712e157c6 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroBigIntProducer.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers; + +import java.io.IOException; +import org.apache.arrow.vector.BaseFixedWidthVector; +import org.apache.arrow.vector.BigIntVector; +import org.apache.avro.io.Encoder; + +/** + * Producer that produces long values from a {@link BigIntVector}, writes data to an Avro encoder. + * + *

Logical types are also supported, for vectors derived from {@link BaseFixedWidthVector} where + * the internal representation matches BigIntVector and requires no conversion. + */ +public class AvroBigIntProducer extends BaseAvroProducer { + + /** Instantiate an AvroBigIntProducer. */ + public AvroBigIntProducer(BigIntVector vector) { + super(vector); + } + + /** Protected constructor for logical types with a long representation. */ + protected AvroBigIntProducer(BaseFixedWidthVector vector) { + super(vector); + if (vector.getTypeWidth() != BigIntVector.TYPE_WIDTH) { + throw new IllegalArgumentException( + "AvroBigIntProducer requires type width = " + BigIntVector.TYPE_WIDTH); + } + } + + @Override + public void produce(Encoder encoder) throws IOException { + long value = vector.getDataBuffer().getLong(currentIndex * (long) BigIntVector.TYPE_WIDTH); + encoder.writeLong(value); + currentIndex++; + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroBooleanProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroBooleanProducer.java new file mode 100644 index 0000000000..523ddf110d --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroBooleanProducer.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers; + +import java.io.IOException; +import org.apache.arrow.vector.BitVector; +import org.apache.avro.io.Encoder; + +/** + * Producer that produces boolean values from a {@link BitVector}, writes data to an Avro encoder. + */ +public class AvroBooleanProducer extends BaseAvroProducer { + + /** Instantiate am AvroBooleanProducer. */ + public AvroBooleanProducer(BitVector vector) { + super(vector); + } + + @Override + public void produce(Encoder encoder) throws IOException { + int bitValue = vector.get(currentIndex++); + encoder.writeBoolean(bitValue != 0); + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroBytesProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroBytesProducer.java new file mode 100644 index 0000000000..e1fe6dddc6 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroBytesProducer.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers; + +import java.io.IOException; +import java.nio.ByteBuffer; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.avro.io.Encoder; + +/** + * Producer that produces byte array values from a {@link VarBinaryVector}, writes data to an Avro + * encoder. + */ +public class AvroBytesProducer extends BaseAvroProducer { + + /** Instantiate an AvroBytesProducer. */ + public AvroBytesProducer(VarBinaryVector vector) { + super(vector); + } + + @Override + public void produce(Encoder encoder) throws IOException { + // The nio ByteBuffer is created once per call, but underlying data is not copied + long offset = vector.getStartOffset(currentIndex); + long endOffset = vector.getEndOffset(currentIndex); + int length = (int) (endOffset - offset); + ByteBuffer nioBuffer = vector.getDataBuffer().nioBuffer(offset, length); + encoder.writeBytes(nioBuffer); + currentIndex++; + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroEnumProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroEnumProducer.java new file mode 100644 index 0000000000..068566493e --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroEnumProducer.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers; + +import java.io.IOException; +import org.apache.arrow.vector.IntVector; +import org.apache.avro.io.Encoder; + +/** + * Producer that produces enum values from a dictionary-encoded {@link IntVector}, writes data to an + * Avro encoder. + */ +public class AvroEnumProducer extends BaseAvroProducer { + + /** Instantiate an AvroEnumProducer. */ + public AvroEnumProducer(IntVector vector) { + super(vector); + } + + @Override + public void produce(Encoder encoder) throws IOException { + encoder.writeEnum(vector.get(currentIndex++)); + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroFixedSizeBinaryProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroFixedSizeBinaryProducer.java new file mode 100644 index 0000000000..9fb877cfa0 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroFixedSizeBinaryProducer.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers; + +import java.io.IOException; +import org.apache.arrow.vector.BaseFixedWidthVector; +import org.apache.arrow.vector.FixedSizeBinaryVector; +import org.apache.avro.io.Encoder; + +/** + * Producer that produces fixed-size binary values from a {@link FixedSizeBinaryVector}, writes data + * to an Avro encoder. + * + *

Logical types are also supported, for vectors derived from {@link BaseFixedWidthVector} where + * the internal representation is fixed width bytes and requires no conversion. + */ +public class AvroFixedSizeBinaryProducer extends BaseAvroProducer { + + private final byte[] reuseBytes; + + /** Instantiate an AvroFixedSizeBinaryProducer. */ + public AvroFixedSizeBinaryProducer(FixedSizeBinaryVector vector) { + super(vector); + reuseBytes = new byte[vector.getTypeWidth()]; + } + + /** Protected constructor for logical types with a fixed width representation. */ + protected AvroFixedSizeBinaryProducer(BaseFixedWidthVector vector) { + super(vector); + reuseBytes = new byte[vector.getTypeWidth()]; + } + + @Override + public void produce(Encoder encoder) throws IOException { + long offset = (long) currentIndex * vector.getTypeWidth(); + vector.getDataBuffer().getBytes(offset, reuseBytes); + encoder.writeFixed(reuseBytes); + currentIndex++; + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroFixedSizeListProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroFixedSizeListProducer.java new file mode 100644 index 0000000000..acb6fb8c00 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroFixedSizeListProducer.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers; + +import java.io.IOException; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.complex.FixedSizeListVector; +import org.apache.avro.io.Encoder; + +/** + * Producer that produces array values from a {@link FixedSizeListVector}, writes data to an avro + * encoder. + */ +public class AvroFixedSizeListProducer extends BaseAvroProducer { + + private final Producer delegate; + + /** Instantiate an AvroFixedSizeListProducer. */ + public AvroFixedSizeListProducer( + FixedSizeListVector vector, Producer delegate) { + super(vector); + this.delegate = delegate; + } + + @Override + public void produce(Encoder encoder) throws IOException { + + encoder.writeArrayStart(); + encoder.setItemCount(vector.getListSize()); + + for (int i = 0; i < vector.getListSize(); i++) { + encoder.startItem(); + delegate.produce(encoder); + } + + encoder.writeArrayEnd(); + currentIndex++; + } + + @Override + public void skipNull() { + super.skipNull(); + // Child vector contains a fixed number of elements for each entry + int childIndex = currentIndex * vector.getListSize(); + delegate.setPosition(childIndex); + } + + @Override + public void setPosition(int index) { + if (index < 0 || index > vector.getValueCount()) { + throw new IllegalArgumentException("Index out of bounds"); + } + super.setPosition(index); + // Child vector contains a fixed number of elements for each entry + int childIndex = currentIndex * vector.getListSize(); + delegate.setPosition(childIndex); + } + + @Override + @SuppressWarnings("unchecked") + public void resetValueVector(FixedSizeListVector vector) { + ((Producer) delegate).resetValueVector(vector.getDataVector()); + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroFloat2Producer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroFloat2Producer.java new file mode 100644 index 0000000000..07e5ea3591 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroFloat2Producer.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers; + +import java.io.IOException; +import org.apache.arrow.memory.util.Float16; +import org.apache.arrow.vector.Float2Vector; +import org.apache.avro.io.Encoder; + +/** + * Producer that produces float values from a {@link Float2Vector}, writes data to an Avro encoder. + */ +public class AvroFloat2Producer extends BaseAvroProducer { + + /** Instantiate an AvroFloat2Producer. */ + public AvroFloat2Producer(Float2Vector vector) { + super(vector); + } + + @Override + public void produce(Encoder encoder) throws IOException { + short rawValue = vector.getDataBuffer().getShort(currentIndex * (long) Float2Vector.TYPE_WIDTH); + encoder.writeFloat(Float16.toFloat(rawValue)); + currentIndex++; + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroFloat4Producer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroFloat4Producer.java new file mode 100644 index 0000000000..5121ba3a11 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroFloat4Producer.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers; + +import java.io.IOException; +import org.apache.arrow.vector.Float4Vector; +import org.apache.avro.io.Encoder; + +/** + * Producer that produces float values from a {@link Float4Vector}, writes data to an Avro encoder. + */ +public class AvroFloat4Producer extends BaseAvroProducer { + + /** Instantiate an AvroFloat4Producer. */ + public AvroFloat4Producer(Float4Vector vector) { + super(vector); + } + + @Override + public void produce(Encoder encoder) throws IOException { + float value = vector.getDataBuffer().getFloat(currentIndex * (long) Float4Vector.TYPE_WIDTH); + encoder.writeFloat(value); + currentIndex++; + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroFloat8Producer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroFloat8Producer.java new file mode 100644 index 0000000000..05fca750ba --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroFloat8Producer.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers; + +import java.io.IOException; +import org.apache.arrow.vector.Float8Vector; +import org.apache.avro.io.Encoder; + +/** + * Producer that produces double values from a {@link Float8Vector}, writes data to an Avro encoder. + */ +public class AvroFloat8Producer extends BaseAvroProducer { + + /** Instantiate an AvroFloat8Producer. */ + public AvroFloat8Producer(Float8Vector vector) { + super(vector); + } + + @Override + public void produce(Encoder encoder) throws IOException { + double value = vector.getDataBuffer().getDouble(currentIndex * (long) Float8Vector.TYPE_WIDTH); + encoder.writeDouble(value); + currentIndex++; + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroIntProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroIntProducer.java new file mode 100644 index 0000000000..4c9cc9b71a --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroIntProducer.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers; + +import java.io.IOException; +import org.apache.arrow.vector.BaseFixedWidthVector; +import org.apache.arrow.vector.IntVector; +import org.apache.avro.io.Encoder; + +/** + * Producer that produces int values from an {@link IntVector}, writes data to an avro encoder. + * + *

Logical types are also supported, for vectors derived from {@link BaseFixedWidthVector} where + * the internal representation matches IntVector and requires no conversion. + */ +public class AvroIntProducer extends BaseAvroProducer { + + /** Instantiate an AvroIntConsumer. */ + public AvroIntProducer(IntVector vector) { + super(vector); + } + + /** Protected constructor for a logical types with an integer representation. */ + protected AvroIntProducer(BaseFixedWidthVector vector) { + super(vector); + if (vector.getTypeWidth() != IntVector.TYPE_WIDTH) { + throw new IllegalArgumentException( + "AvroIntProducer requires type width = " + IntVector.TYPE_WIDTH); + } + } + + @Override + public void produce(Encoder encoder) throws IOException { + int value = vector.getDataBuffer().getInt(currentIndex * (long) IntVector.TYPE_WIDTH); + encoder.writeInt(value); + currentIndex++; + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroListProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroListProducer.java new file mode 100644 index 0000000000..10cfe9549a --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroListProducer.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers; + +import java.io.IOException; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.complex.ListVector; +import org.apache.avro.io.Encoder; + +/** + * Producer that produces array values from a {@link ListVector}, writes data to an avro encoder. + */ +public class AvroListProducer extends BaseAvroProducer { + + private final Producer delegate; + + /** Instantiate an AvroListProducer. */ + public AvroListProducer(ListVector vector, Producer delegate) { + super(vector); + this.delegate = delegate; + } + + @Override + public void produce(Encoder encoder) throws IOException { + + int startOffset = vector.getOffsetBuffer().getInt(currentIndex * (long) Integer.BYTES); + int endOffset = vector.getOffsetBuffer().getInt((currentIndex + 1) * (long) Integer.BYTES); + int nItems = endOffset - startOffset; + + encoder.writeArrayStart(); + encoder.setItemCount(nItems); + + for (int i = 0; i < nItems; i++) { + encoder.startItem(); + delegate.produce(encoder); + } + + encoder.writeArrayEnd(); + currentIndex++; + } + + // Do not override skipNull(), delegate will not have an entry if the list is null + + @Override + public void setPosition(int index) { + if (index < 0 || index > vector.getValueCount()) { + throw new IllegalArgumentException("Index out of bounds"); + } + int delegateOffset = vector.getOffsetBuffer().getInt(index * (long) Integer.BYTES); + delegate.setPosition(delegateOffset); + super.setPosition(index); + } + + @Override + @SuppressWarnings("unchecked") + public void resetValueVector(ListVector vector) { + ((Producer) delegate).resetValueVector(vector.getDataVector()); + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroMapProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroMapProducer.java new file mode 100644 index 0000000000..568d5b62e4 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroMapProducer.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers; + +import java.io.IOException; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.complex.MapVector; +import org.apache.avro.io.Encoder; + +/** Producer which produces map type values to avro encoder. Write the data to {@link MapVector}. */ +public class AvroMapProducer extends BaseAvroProducer { + + private final Producer delegate; + + /** Instantiate a AvroMapProducer. */ + public AvroMapProducer(MapVector vector, Producer delegate) { + super(vector); + this.delegate = delegate; + } + + @Override + public void produce(Encoder encoder) throws IOException { + + int startOffset = vector.getOffsetBuffer().getInt(currentIndex * (long) Integer.BYTES); + int endOffset = vector.getOffsetBuffer().getInt((currentIndex + 1) * (long) Integer.BYTES); + int nEntries = endOffset - startOffset; + + encoder.writeMapStart(); + encoder.setItemCount(nEntries); + + for (int i = 0; i < nEntries; i++) { + encoder.startItem(); + delegate.produce(encoder); + } + + encoder.writeMapEnd(); + currentIndex++; + } + + // Do not override skipNull(), delegate will not have an entry if the map is null + + @Override + public void setPosition(int index) { + if (index < 0 || index > vector.getValueCount()) { + throw new IllegalArgumentException("Index out of bounds"); + } + int delegateOffset = vector.getOffsetBuffer().getInt(index * (long) Integer.BYTES); + delegate.setPosition(delegateOffset); + super.setPosition(index); + } + + @Override + @SuppressWarnings("unchecked") + public void resetValueVector(MapVector vector) { + ((Producer) delegate).resetValueVector(vector.getDataVector()); + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroNullProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroNullProducer.java new file mode 100644 index 0000000000..1bd1e891f1 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroNullProducer.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers; + +import java.io.IOException; +import org.apache.arrow.vector.NullVector; +import org.apache.avro.io.Encoder; + +/** Producer that produces null values from a {@link NullVector}, writes data to an Avro encoder. */ +public class AvroNullProducer extends BaseAvroProducer { + + /** Instantiate an AvroNullProducer. */ + public AvroNullProducer(NullVector vector) { + super(vector); + } + + @Override + public void produce(Encoder encoder) throws IOException { + encoder.writeNull(); + currentIndex++; + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroNullableProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroNullableProducer.java new file mode 100644 index 0000000000..f4215dbf84 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroNullableProducer.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers; + +import java.io.IOException; +import org.apache.arrow.vector.FieldVector; +import org.apache.avro.io.Encoder; + +/** + * Producer wrapper which producers nullable types to an avro encoder. Write the data to the + * underlying {@link FieldVector}. + * + * @param The vector within producer or its delegate, used for partially produce purpose. + */ +public class AvroNullableProducer extends BaseAvroProducer { + + private final Producer delegate; + + /** Instantiate a AvroNullableProducer. */ + public AvroNullableProducer(Producer delegate) { + super(delegate.getVector()); + this.delegate = delegate; + } + + @Override + public void produce(Encoder encoder) throws IOException { + if (vector.isNull(currentIndex)) { + encoder.writeInt(1); + encoder.writeNull(); + delegate.skipNull(); + } else { + encoder.writeInt(0); + delegate.produce(encoder); + } + currentIndex++; + } + + @Override + public void skipNull() { + // Can be called by containers of nullable types + delegate.skipNull(); + currentIndex++; + } + + @Override + public void setPosition(int index) { + if (index < 0 || index > vector.getValueCount()) { + throw new IllegalArgumentException("Index out of bounds"); + } + delegate.setPosition(index); + super.setPosition(index); + } + + @Override + public void resetValueVector(T vector) { + delegate.resetValueVector(vector); + } + + @Override + public T getVector() { + return delegate.getVector(); + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroSmallIntProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroSmallIntProducer.java new file mode 100644 index 0000000000..9c37750d9f --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroSmallIntProducer.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers; + +import java.io.IOException; +import org.apache.arrow.vector.SmallIntVector; +import org.apache.avro.io.Encoder; + +/** + * Producer that produces int values from an {@link SmallIntVector}, writes data to an avro encoder. + */ +public class AvroSmallIntProducer extends BaseAvroProducer { + + /** Instantiate an AvroSmallIntProducer. */ + public AvroSmallIntProducer(SmallIntVector vector) { + super(vector); + } + + @Override + public void produce(Encoder encoder) throws IOException { + short value = vector.getDataBuffer().getShort(currentIndex * (long) SmallIntVector.TYPE_WIDTH); + encoder.writeInt(value); + currentIndex++; + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroStringProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroStringProducer.java new file mode 100644 index 0000000000..19e165cd13 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroStringProducer.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers; + +import java.io.IOException; +import java.nio.ByteBuffer; +import org.apache.arrow.vector.VarCharVector; +import org.apache.avro.io.Encoder; + +/** + * Producer that produces string values from a {@link VarCharVector}, writes data to an Avro + * encoder. + */ +public class AvroStringProducer extends BaseAvroProducer { + + /** Instantiate an AvroStringProducer. */ + public AvroStringProducer(VarCharVector vector) { + super(vector); + } + + @Override + public void produce(Encoder encoder) throws IOException { + + int start = vector.getStartOffset(currentIndex); + int end = vector.getEndOffset(currentIndex); + int length = end - start; + + // The nio ByteBuffer is created once per call, but underlying data is not copied + ByteBuffer nioBuffer = vector.getDataBuffer().nioBuffer(start, length); + encoder.writeBytes(nioBuffer); + + currentIndex++; + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroStructProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroStructProducer.java new file mode 100644 index 0000000000..86c1949bf6 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroStructProducer.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers; + +import java.io.IOException; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.complex.StructVector; +import org.apache.avro.io.Encoder; + +/** + * Producer which produces nested record type values to avro encoder. Read the data from {@link + * org.apache.arrow.vector.complex.StructVector}. + */ +public class AvroStructProducer extends BaseAvroProducer { + + private final Producer[] delegates; + + /** Instantiate a AvroStructProducer. */ + public AvroStructProducer(StructVector vector, Producer[] delegates) { + super(vector); + this.delegates = delegates; + } + + @Override + public void produce(Encoder encoder) throws IOException { + + for (Producer delegate : delegates) { + delegate.produce(encoder); + } + + currentIndex++; + } + + @Override + public void skipNull() { + for (Producer delegate : delegates) { + delegate.skipNull(); + } + super.skipNull(); + } + + @Override + public void setPosition(int index) { + if (index < 0 || index > vector.getValueCount()) { + throw new IllegalArgumentException("Index out of bounds: " + index); + } + for (Producer delegate : delegates) { + delegate.setPosition(index); + } + super.setPosition(index); + } + + @Override + @SuppressWarnings("unchecked") + public void resetValueVector(StructVector vector) { + for (int i = 0; i < delegates.length; i++) { + Producer delegate = (Producer) delegates[i]; + delegate.resetValueVector(vector.getChildrenFromFields().get(i)); + } + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroTinyIntProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroTinyIntProducer.java new file mode 100644 index 0000000000..30a80e5094 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroTinyIntProducer.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers; + +import java.io.IOException; +import org.apache.arrow.vector.TinyIntVector; +import org.apache.avro.io.Encoder; + +/** + * Producer that produces int values from an {@link TinyIntVector}, writes data to an avro encoder. + */ +public class AvroTinyIntProducer extends BaseAvroProducer { + + /** Instantiate an AvroTinyIntProducer. */ + public AvroTinyIntProducer(TinyIntVector vector) { + super(vector); + } + + @Override + public void produce(Encoder encoder) throws IOException { + byte value = vector.getDataBuffer().getByte(currentIndex * (long) TinyIntVector.TYPE_WIDTH); + encoder.writeInt(value); + currentIndex++; + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroUint1Producer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroUint1Producer.java new file mode 100644 index 0000000000..83cbc9ef8e --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroUint1Producer.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers; + +import java.io.IOException; +import org.apache.arrow.vector.UInt1Vector; +import org.apache.avro.io.Encoder; + +/** Producer that produces int values from a {@link UInt1Vector}, writes data to an avro encoder. */ +public class AvroUint1Producer extends BaseAvroProducer { + + /** Instantiate an AvroUint1Producer. */ + public AvroUint1Producer(UInt1Vector vector) { + super(vector); + } + + @Override + public void produce(Encoder encoder) throws IOException { + byte unsigned = vector.getDataBuffer().getByte(currentIndex * (long) UInt1Vector.TYPE_WIDTH); + int unsignedInt = Byte.toUnsignedInt(unsigned); + encoder.writeInt(unsignedInt); + currentIndex++; + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroUint2Producer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroUint2Producer.java new file mode 100644 index 0000000000..1e30c82cd2 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroUint2Producer.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers; + +import java.io.IOException; +import org.apache.arrow.vector.UInt2Vector; +import org.apache.avro.io.Encoder; + +/** Producer that produces int values from a {@link UInt2Vector}, writes data to an avro encoder. */ +public class AvroUint2Producer extends BaseAvroProducer { + + /** Instantiate an AvroUint2Producer. */ + public AvroUint2Producer(UInt2Vector vector) { + super(vector); + } + + @Override + public void produce(Encoder encoder) throws IOException { + short unsigned = vector.getDataBuffer().getShort(currentIndex * (long) UInt2Vector.TYPE_WIDTH); + int unsignedInt = Short.toUnsignedInt(unsigned); + encoder.writeInt(unsignedInt); + currentIndex++; + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroUint4Producer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroUint4Producer.java new file mode 100644 index 0000000000..63f78429dd --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroUint4Producer.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers; + +import java.io.IOException; +import org.apache.arrow.vector.UInt4Vector; +import org.apache.avro.io.Encoder; + +/** + * Producer that produces long values from a {@link UInt4Vector}, writes data to an avro encoder. + */ +public class AvroUint4Producer extends BaseAvroProducer { + + /** Instantiate an AvroUint4Producer. */ + public AvroUint4Producer(UInt4Vector vector) { + super(vector); + } + + @Override + public void produce(Encoder encoder) throws IOException { + int unsigned = vector.getDataBuffer().getInt(currentIndex * (long) UInt4Vector.TYPE_WIDTH); + long unsignedLong = Integer.toUnsignedLong(unsigned); + encoder.writeLong(unsignedLong); + currentIndex++; + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroUint8Producer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroUint8Producer.java new file mode 100644 index 0000000000..819b4d4140 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroUint8Producer.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers; + +import java.io.IOException; +import org.apache.arrow.vector.UInt8Vector; +import org.apache.avro.io.Encoder; + +/** + * Producer that produces long values from a {@link UInt8Vector}, writes data to an avro encoder. + */ +public class AvroUint8Producer extends BaseAvroProducer { + + /** Instantiate an AvroUint8Producer. */ + public AvroUint8Producer(UInt8Vector vector) { + super(vector); + } + + @Override + public void produce(Encoder encoder) throws IOException { + long unsigned = vector.getDataBuffer().getLong(currentIndex * (long) UInt8Vector.TYPE_WIDTH); + if (unsigned < 0) { + throw new ArithmeticException("Unsigned long value is too large for Avro encoding"); + } + encoder.writeLong(unsigned); + currentIndex++; + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/BaseAvroProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/BaseAvroProducer.java new file mode 100644 index 0000000000..30c004bdc6 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/BaseAvroProducer.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers; + +import org.apache.arrow.vector.FieldVector; + +/** + * Base class for avro producers. + * + * @param vector type. + */ +public abstract class BaseAvroProducer implements Producer { + + protected T vector; + protected int currentIndex; + + /** + * Constructs a base avro consumer. + * + * @param vector the vector to consume. + */ + protected BaseAvroProducer(T vector) { + this.vector = vector; + } + + @Override + public void skipNull() { + currentIndex++; + } + + /** + * Sets the current index for this producer against the underlying vector. + * + *

For a vector of length N, the valid range is [0, N] inclusive. Setting index = N signifies + * that no further data is available for production (this is the state the produce will be in when + * production for the current vector is complete). + * + * @param index New current index for the producer + */ + @Override + public void setPosition(int index) { + // currentIndex == value count is a valid state, no more values will be produced + if (index < 0 || index > vector.getValueCount()) { + throw new IllegalArgumentException("Index out of bounds"); + } + currentIndex = index; + } + + @Override + public void resetValueVector(T vector) { + this.vector = vector; + this.currentIndex = 0; + } + + @Override + public T getVector() { + return vector; + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/CompositeAvroProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/CompositeAvroProducer.java new file mode 100644 index 0000000000..d1ed506108 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/CompositeAvroProducer.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers; + +import java.io.IOException; +import java.util.List; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.avro.io.Encoder; + +/** Composite producer which holds all producers. It manages the produce and cleanup process. */ +public class CompositeAvroProducer { + + private final List> producers; + + public CompositeAvroProducer(List> producers) { + this.producers = producers; + } + + public List> getProducers() { + return producers; + } + + /** Produce encoder data. */ + public void produce(Encoder encoder) throws IOException { + for (Producer producer : producers) { + producer.produce(encoder); + } + } + + /** Reset vector of consumers with the given {@link VectorSchemaRoot}. */ + @SuppressWarnings({"unchecked", "rawtypes"}) + public void resetProducerVectors(VectorSchemaRoot root) { + // This method assumes that the VSR matches the constructed set of producers + int index = 0; + for (Producer producer : producers) { + producer.resetValueVector(root.getFieldVectors().get(index)); + } + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/Producer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/Producer.java new file mode 100644 index 0000000000..aed2543348 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/Producer.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers; + +import java.io.IOException; +import org.apache.arrow.vector.FieldVector; +import org.apache.avro.io.Encoder; + +/** + * Interface that is used to produce values to avro encoder. + * + * @param The vector within producer or its delegate, used for partially produce purpose. + */ +public interface Producer { + + /** + * Produce a specific type value from the vector and write it to avro encoder. + * + * @param encoder avro encoder to write data + * @throws IOException on error + */ + void produce(Encoder encoder) throws IOException; + + /** Skip null value in the vector by setting reader position + 1. */ + void skipNull(); + + /** Set the position to read value from vector. */ + void setPosition(int index); + + /** Reset the vector within producer. */ + void resetValueVector(T vector); + + /** Get the vector within the producer. */ + T getVector(); +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroDateDayProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroDateDayProducer.java new file mode 100644 index 0000000000..36680fb196 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroDateDayProducer.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers.logical; + +import org.apache.arrow.adapter.avro.producers.AvroIntProducer; +import org.apache.arrow.vector.DateDayVector; + +/** + * Producer that produces date values from a {@link DateDayVector}, writes data to an Avro encoder. + */ +public class AvroDateDayProducer extends AvroIntProducer { + + // Date stored as integer number of days, matches Avro date type + + /** Instantiate an AvroDateProducer. */ + public AvroDateDayProducer(DateDayVector vector) { + super(vector); + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroDateMilliProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroDateMilliProducer.java new file mode 100644 index 0000000000..a64bb3a021 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroDateMilliProducer.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers.logical; + +import java.io.IOException; +import org.apache.arrow.adapter.avro.producers.BaseAvroProducer; +import org.apache.arrow.vector.DateMilliVector; +import org.apache.avro.io.Encoder; + +/** + * Producer that converts days in milliseconds from a {@link DateMilliVector} and produces date + * (INT) values, writes data to an Avro encoder. + */ +public class AvroDateMilliProducer extends BaseAvroProducer { + + // Convert milliseconds to days for Avro date type + + private static final long MILLIS_PER_DAY = 86400000; + + /** Instantiate an AvroDateMilliProducer. */ + public AvroDateMilliProducer(DateMilliVector vector) { + super(vector); + } + + @Override + public void produce(Encoder encoder) throws IOException { + long millis = vector.getDataBuffer().getLong(currentIndex * (long) DateMilliVector.TYPE_WIDTH); + long days = millis / MILLIS_PER_DAY; + if (days > (long) Integer.MAX_VALUE || days < (long) Integer.MIN_VALUE) { + throw new ArithmeticException("Date value is too large for Avro encoding"); + } + encoder.writeInt((int) days); + currentIndex++; + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroDecimal256Producer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroDecimal256Producer.java new file mode 100644 index 0000000000..f72aa6d9e0 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroDecimal256Producer.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers.logical; + +import java.io.IOException; +import java.math.BigDecimal; +import org.apache.arrow.adapter.avro.producers.BaseAvroProducer; +import org.apache.arrow.vector.Decimal256Vector; +import org.apache.avro.io.Encoder; + +/** + * Producer that produces decimal values from a {@link Decimal256Vector}, writes data to an Avro + * encoder. + */ +public class AvroDecimal256Producer extends BaseAvroProducer { + + // Logic is the same as for DecimalVector (128 bit) + + byte[] encodedBytes = new byte[Decimal256Vector.TYPE_WIDTH]; + + /** Instantiate an AvroDecimalProducer. */ + public AvroDecimal256Producer(Decimal256Vector vector) { + super(vector); + } + + @Override + public void produce(Encoder encoder) throws IOException { + BigDecimal value = vector.getObject(currentIndex++); + AvroDecimalProducer.encodeDecimal(value, encodedBytes); + encoder.writeFixed(encodedBytes); + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroDecimalProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroDecimalProducer.java new file mode 100644 index 0000000000..51ad7c7200 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroDecimalProducer.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers.logical; + +import java.io.IOException; +import java.math.BigDecimal; +import org.apache.arrow.adapter.avro.producers.BaseAvroProducer; +import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.util.DecimalUtility; +import org.apache.avro.io.Encoder; + +/** + * Producer that produces decimal values from a {@link DecimalVector}, writes data to an Avro + * encoder. + */ +public class AvroDecimalProducer extends BaseAvroProducer { + + // Arrow stores decimals with native endianness, but Avro requires big endian + // Writing the Arrow representation as fixed bytes fails on little-end machines + // Instead, we replicate the big endian logic explicitly here + // See DecimalUtility.writeByteArrayToArrowBufHelper + + byte[] encodedBytes = new byte[DecimalVector.TYPE_WIDTH]; + + /** Instantiate an AvroDecimalProducer. */ + public AvroDecimalProducer(DecimalVector vector) { + super(vector); + } + + @Override + public void produce(Encoder encoder) throws IOException { + // Use getObject() to go back to a BigDecimal then re-encode + BigDecimal value = vector.getObject(currentIndex++); + encodeDecimal(value, encodedBytes); + encoder.writeFixed(encodedBytes); + } + + static void encodeDecimal(BigDecimal value, byte[] encodedBytes) { + byte[] valueBytes = value.unscaledValue().toByteArray(); + byte[] padding = valueBytes[0] < 0 ? DecimalUtility.minus_one : DecimalUtility.zeroes; + System.arraycopy(padding, 0, encodedBytes, 0, encodedBytes.length - valueBytes.length); + System.arraycopy( + valueBytes, 0, encodedBytes, encodedBytes.length - valueBytes.length, valueBytes.length); + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimeMicroProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimeMicroProducer.java new file mode 100644 index 0000000000..203d102034 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimeMicroProducer.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers.logical; + +import org.apache.arrow.adapter.avro.producers.AvroBigIntProducer; +import org.apache.arrow.vector.TimeMicroVector; + +/** + * Producer that produces time (microseconds) values from a {@link TimeMicroVector}, writes data to + * an Avro encoder. + */ +public class AvroTimeMicroProducer extends AvroBigIntProducer { + + // Time in microseconds stored as long, matches Avro time-micros type + + /** Instantiate an AvroTimeMicroProducer. */ + public AvroTimeMicroProducer(TimeMicroVector vector) { + super(vector); + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimeMilliProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimeMilliProducer.java new file mode 100644 index 0000000000..2a452e75a4 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimeMilliProducer.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers.logical; + +import org.apache.arrow.adapter.avro.producers.AvroIntProducer; +import org.apache.arrow.vector.TimeMilliVector; + +/** + * Producer that produces time (milliseconds) values from a {@link TimeMilliVector}, writes data to + * an Avro encoder. + */ +public class AvroTimeMilliProducer extends AvroIntProducer { + + // Time in milliseconds stored as integer, matches Avro time-millis type + + /** Instantiate an AvroTimeMilliProducer. */ + public AvroTimeMilliProducer(TimeMilliVector vector) { + super(vector); + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimeNanoProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimeNanoProducer.java new file mode 100644 index 0000000000..7034dbbb50 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimeNanoProducer.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers.logical; + +import java.io.IOException; +import org.apache.arrow.adapter.avro.producers.BaseAvroProducer; +import org.apache.arrow.vector.TimeNanoVector; +import org.apache.avro.io.Encoder; + +/** + * Producer that converts nanoseconds from a {@link TimeNanoVector} and produces time (microseconds) + * values, writes data to an Avro encoder. + */ +public class AvroTimeNanoProducer extends BaseAvroProducer { + + // Convert nanoseconds to microseconds for Avro time-micros (LONG) type + // Range is 1000 times less than for microseconds, so the type will fit (with loss of precision) + + private static final long NANOS_PER_MICRO = 1000; + + public AvroTimeNanoProducer(TimeNanoVector vector) { + super(vector); + } + + @Override + public void produce(Encoder encoder) throws IOException { + long nanos = vector.getDataBuffer().getLong(currentIndex * (long) TimeNanoVector.TYPE_WIDTH); + long micros = nanos / NANOS_PER_MICRO; + encoder.writeLong(micros); + currentIndex++; + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimeSecProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimeSecProducer.java new file mode 100644 index 0000000000..951605b6c3 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimeSecProducer.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers.logical; + +import java.io.IOException; +import org.apache.arrow.adapter.avro.producers.BaseAvroProducer; +import org.apache.arrow.vector.TimeSecVector; +import org.apache.avro.io.Encoder; + +/** + * Producer that converts seconds from a {@link TimeSecVector} and produces time (microseconds) + * values, writes data to an Avro encoder. + */ +public class AvroTimeSecProducer extends BaseAvroProducer { + + // Convert seconds to milliseconds for Avro time-millis (INT) type + // INT is enough to cover the number of milliseconds in a day + // So overflows should not happen if values are valid times of day + + private static final int MILLIS_PER_SECOND = 1000; + private static final long OVERFLOW_LIMIT = Integer.MAX_VALUE / 1000; + + /** Instantiate an AvroTimeSecProducer. */ + public AvroTimeSecProducer(TimeSecVector vector) { + super(vector); + } + + @Override + public void produce(Encoder encoder) throws IOException { + int seconds = vector.getDataBuffer().getInt(currentIndex * (long) TimeSecVector.TYPE_WIDTH); + if (Math.abs(seconds) > OVERFLOW_LIMIT) { + throw new ArithmeticException("Time value is too large for Avro encoding"); + } + int millis = seconds * MILLIS_PER_SECOND; + encoder.writeInt(millis); + currentIndex++; + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampMicroProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampMicroProducer.java new file mode 100644 index 0000000000..4e744b5e76 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampMicroProducer.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers.logical; + +import org.apache.arrow.adapter.avro.producers.AvroBigIntProducer; +import org.apache.arrow.vector.TimeStampMicroVector; + +/** + * Producer that produces local timestamp (microseconds) values from a {@link TimeStampMicroVector}, + * writes data to an Avro encoder. + */ +public class AvroTimestampMicroProducer extends AvroBigIntProducer { + + // Local timestamp in epoch microseconds stored as long, matches Avro local-timestamp-micros type + + /** Instantiate an AvroTimestampMicroProducer. */ + public AvroTimestampMicroProducer(TimeStampMicroVector vector) { + super(vector); + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampMicroTzProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampMicroTzProducer.java new file mode 100644 index 0000000000..ece7482303 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampMicroTzProducer.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers.logical; + +import org.apache.arrow.adapter.avro.producers.AvroBigIntProducer; +import org.apache.arrow.vector.TimeStampMicroTZVector; + +/** + * Producer that produces UTC timestamp (microseconds) values from a {@link TimeStampMicroTZVector}, + * writes data to an Avro encoder. + */ +public class AvroTimestampMicroTzProducer extends AvroBigIntProducer { + + // UTC timestamp in epoch microseconds stored as long, matches Avro timestamp-micros type + // Both Arrow and Avro store zone-aware times in UTC so zone conversion is not needed + + /** Instantiate an AvroTimestampMicroTzProducer. */ + public AvroTimestampMicroTzProducer(TimeStampMicroTZVector vector) { + super(vector); + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampMilliProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampMilliProducer.java new file mode 100644 index 0000000000..e71acff220 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampMilliProducer.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers.logical; + +import org.apache.arrow.adapter.avro.producers.AvroBigIntProducer; +import org.apache.arrow.vector.TimeStampMilliVector; + +/** + * Producer that produces local timestamp (milliseconds) values from a {@link TimeStampMilliVector}, + * writes data to an Avro encoder. + */ +public class AvroTimestampMilliProducer extends AvroBigIntProducer { + + // Local timestamp in epoch milliseconds stored as long, matches Avro local-timestamp-millis type + + /** Instantiate an AvroTimestampMilliProducer. */ + public AvroTimestampMilliProducer(TimeStampMilliVector vector) { + super(vector); + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampMilliTzProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampMilliTzProducer.java new file mode 100644 index 0000000000..b1b55fca78 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampMilliTzProducer.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers.logical; + +import org.apache.arrow.adapter.avro.producers.AvroBigIntProducer; +import org.apache.arrow.vector.TimeStampMilliTZVector; + +/** + * Producer that produces UTC timestamp (milliseconds) values from a {@link TimeStampMilliTZVector}, + * writes data to an Avro encoder. + */ +public class AvroTimestampMilliTzProducer extends AvroBigIntProducer { + + // UTC timestamp in epoch milliseconds stored as long, matches Avro timestamp-millis type + // Both Arrow and Avro store zone-aware times in UTC so zone conversion is not needed + + /** Instantiate an AvroTimestampMilliTzProducer. */ + public AvroTimestampMilliTzProducer(TimeStampMilliTZVector vector) { + super(vector); + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampNanoProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampNanoProducer.java new file mode 100644 index 0000000000..9e172ea91e --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampNanoProducer.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers.logical; + +import org.apache.arrow.adapter.avro.producers.AvroBigIntProducer; +import org.apache.arrow.vector.TimeStampNanoVector; + +/** + * Producer that produces local timestamp (nanoseconds) values from a {@link TimeStampNanoVector}, + * writes data to an Avro encoder. + */ +public class AvroTimestampNanoProducer extends AvroBigIntProducer { + + // Local timestamp in epoch nanoseconds stored as long, matches Avro local-timestamp-nanos type + + /** Instantiate an AvroTimestampNanoProducer. */ + public AvroTimestampNanoProducer(TimeStampNanoVector vector) { + super(vector); + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampNanoTzProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampNanoTzProducer.java new file mode 100644 index 0000000000..ae261d8396 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampNanoTzProducer.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers.logical; + +import org.apache.arrow.adapter.avro.producers.AvroBigIntProducer; +import org.apache.arrow.vector.TimeStampNanoTZVector; + +/** + * Producer that produces local timestamp (nanoseconds) values from a {@link TimeStampNanoTZVector}, + * writes data to an Avro encoder. + */ +public class AvroTimestampNanoTzProducer extends AvroBigIntProducer { + + // UTC timestamp in epoch nanoseconds stored as long, matches Avro timestamp-nanos type + // Both Arrow and Avro store zone-aware times in UTC so zone conversion is not needed + + /** Instantiate an AvroTimestampNanoTzProducer. */ + public AvroTimestampNanoTzProducer(TimeStampNanoTZVector vector) { + super(vector); + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampSecProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampSecProducer.java new file mode 100644 index 0000000000..a6ade2d19b --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampSecProducer.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers.logical; + +import java.io.IOException; +import org.apache.arrow.adapter.avro.producers.BaseAvroProducer; +import org.apache.arrow.vector.TimeStampSecVector; +import org.apache.avro.io.Encoder; + +/** + * Producer that converts epoch seconds from a {@link TimeStampSecVector} and produces local + * timestamp (milliseconds) values, writes data to an Avro encoder. + */ +public class AvroTimestampSecProducer extends BaseAvroProducer { + + // Avro does not support timestamps in seconds, so convert to local-timestamp-millis type + // Check for overflow and raise an exception + + private static final long MILLIS_PER_SECOND = 1000; + private static final long OVERFLOW_LIMIT = Long.MAX_VALUE / MILLIS_PER_SECOND; + + /** Instantiate an AvroTimestampSecProducer. */ + public AvroTimestampSecProducer(TimeStampSecVector vector) { + super(vector); + } + + @Override + public void produce(Encoder encoder) throws IOException { + long seconds = + vector.getDataBuffer().getLong(currentIndex * (long) TimeStampSecVector.TYPE_WIDTH); + if (Math.abs(seconds) > OVERFLOW_LIMIT) { + throw new ArithmeticException("Timestamp value is too large for Avro encoding"); + } + long millis = seconds * MILLIS_PER_SECOND; + encoder.writeLong(millis); + currentIndex++; + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampSecTzProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampSecTzProducer.java new file mode 100644 index 0000000000..bd6cc14dad --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/logical/AvroTimestampSecTzProducer.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers.logical; + +import java.io.IOException; +import org.apache.arrow.adapter.avro.producers.BaseAvroProducer; +import org.apache.arrow.vector.TimeStampSecTZVector; +import org.apache.arrow.vector.TimeStampVector; +import org.apache.avro.io.Encoder; + +/** + * Producer that converts epoch seconds from a {@link TimeStampSecTZVector} and produces UTC + * timestamp (milliseconds) values, writes data to an Avro encoder. + */ +public class AvroTimestampSecTzProducer extends BaseAvroProducer { + + // Avro does not support timestamps in seconds, so convert to timestamp-millis type + // Check for overflow and raise an exception + + // Both Arrow and Avro store zone-aware times in UTC so zone conversion is not needed + + private static final long MILLIS_PER_SECOND = 1000; + private static final long OVERFLOW_LIMIT = Long.MAX_VALUE / MILLIS_PER_SECOND; + + /** Instantiate an AvroTimestampSecTzProducer. */ + public AvroTimestampSecTzProducer(TimeStampSecTZVector vector) { + super(vector); + } + + @Override + public void produce(Encoder encoder) throws IOException { + long utcSeconds = + vector.getDataBuffer().getLong(currentIndex * (long) TimeStampVector.TYPE_WIDTH); + if (Math.abs(utcSeconds) > OVERFLOW_LIMIT) { + throw new ArithmeticException("Timestamp value is too large for Avro encoding"); + } + long utcMillis = utcSeconds * MILLIS_PER_SECOND; + encoder.writeLong(utcMillis); + currentIndex++; + } +} diff --git a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroDataTest.java b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroDataTest.java new file mode 100644 index 0000000000..2d70b45021 --- /dev/null +++ b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroDataTest.java @@ -0,0 +1,2820 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro; + +import static org.junit.jupiter.api.Assertions.*; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.nio.ByteBuffer; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZonedDateTime; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.arrow.adapter.avro.producers.CompositeAvroProducer; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.memory.util.Float16; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.DateDayVector; +import org.apache.arrow.vector.DateMilliVector; +import org.apache.arrow.vector.Decimal256Vector; +import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.FixedSizeBinaryVector; +import org.apache.arrow.vector.Float2Vector; +import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.NullVector; +import org.apache.arrow.vector.SmallIntVector; +import org.apache.arrow.vector.TimeMicroVector; +import org.apache.arrow.vector.TimeMilliVector; +import org.apache.arrow.vector.TimeNanoVector; +import org.apache.arrow.vector.TimeSecVector; +import org.apache.arrow.vector.TimeStampMicroTZVector; +import org.apache.arrow.vector.TimeStampMicroVector; +import org.apache.arrow.vector.TimeStampMilliTZVector; +import org.apache.arrow.vector.TimeStampMilliVector; +import org.apache.arrow.vector.TimeStampNanoTZVector; +import org.apache.arrow.vector.TimeStampNanoVector; +import org.apache.arrow.vector.TimeStampSecTZVector; +import org.apache.arrow.vector.TimeStampSecVector; +import org.apache.arrow.vector.TinyIntVector; +import org.apache.arrow.vector.UInt1Vector; +import org.apache.arrow.vector.UInt2Vector; +import org.apache.arrow.vector.UInt4Vector; +import org.apache.arrow.vector.UInt8Vector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.complex.FixedSizeListVector; +import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.StructVector; +import org.apache.arrow.vector.complex.writer.BaseWriter; +import org.apache.arrow.vector.complex.writer.FieldWriter; +import org.apache.arrow.vector.types.DateUnit; +import org.apache.arrow.vector.types.FloatingPointPrecision; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.JsonStringArrayList; +import org.apache.arrow.vector.util.JsonStringHashMap; +import org.apache.avro.Conversions; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.EncoderFactory; +import org.apache.avro.util.Utf8; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +public class ArrowToAvroDataTest { + + @TempDir public static File TMP; + + // Data production for primitive types, nullable and non-nullable + + @Test + public void testWriteNullColumn() throws Exception { + + // Field definition + FieldType nullField = new FieldType(false, new ArrowType.Null(), null); + + // Create empty vector + NullVector nullVector = new NullVector(new Field("nullColumn", nullField, null)); + + int rowCount = 10; + + // Set up VSR + List vectors = Arrays.asList(nullVector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set all values to null + for (int row = 0; row < rowCount; row++) { + nullVector.setNull(row); + } + + File dataFile = new File(TMP, "testWriteNullColumn.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + GenericRecord record = null; + + // Read and check values + for (int row = 0; row < rowCount; row++) { + record = datumReader.read(record, decoder); + assertNull(record.get("nullColumn")); + } + } + } + } + + @Test + public void testWriteBooleans() throws Exception { + + // Field definition + FieldType booleanField = new FieldType(false, new ArrowType.Bool(), null); + + // Create empty vector + BufferAllocator allocator = new RootAllocator(); + BitVector booleanVector = new BitVector(new Field("boolean", booleanField, null), allocator); + + // Set up VSR + List vectors = Arrays.asList(booleanVector); + int rowCount = 10; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + for (int row = 0; row < rowCount; row++) { + booleanVector.set(row, row % 2 == 0 ? 1 : 0); + } + + File dataFile = new File(TMP, "testWriteBooleans.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + GenericRecord record = null; + + // Read and check values + for (int row = 0; row < rowCount; row++) { + record = datumReader.read(record, decoder); + assertEquals(booleanVector.get(row) == 1, record.get("boolean")); + } + } + } + } + + @Test + public void testWriteNullableBooleans() throws Exception { + + // Field definition + FieldType booleanField = new FieldType(true, new ArrowType.Bool(), null); + + // Create empty vector + BufferAllocator allocator = new RootAllocator(); + BitVector booleanVector = new BitVector(new Field("boolean", booleanField, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = Arrays.asList(booleanVector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Null value + booleanVector.setNull(0); + + // False value + booleanVector.set(1, 0); + + // True value + booleanVector.set(2, 1); + + File dataFile = new File(TMP, "testWriteNullableBooleans.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + + // Read and check values + GenericRecord record = datumReader.read(null, decoder); + assertNull(record.get("boolean")); + + for (int row = 1; row < rowCount; row++) { + record = datumReader.read(record, decoder); + assertEquals(booleanVector.get(row) == 1, record.get("boolean")); + } + } + } + } + + @Test + public void testWriteIntegers() throws Exception { + + // Field definitions + FieldType int8Field = new FieldType(false, new ArrowType.Int(8, true), null); + FieldType int16Field = new FieldType(false, new ArrowType.Int(16, true), null); + FieldType int32Field = new FieldType(false, new ArrowType.Int(32, true), null); + FieldType int64Field = new FieldType(false, new ArrowType.Int(64, true), null); + FieldType uint8Field = new FieldType(false, new ArrowType.Int(8, false), null); + FieldType uint16Field = new FieldType(false, new ArrowType.Int(16, false), null); + FieldType uint32Field = new FieldType(false, new ArrowType.Int(32, false), null); + FieldType uint64Field = new FieldType(false, new ArrowType.Int(64, false), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + TinyIntVector int8Vector = new TinyIntVector(new Field("int8", int8Field, null), allocator); + SmallIntVector int16Vector = + new SmallIntVector(new Field("int16", int16Field, null), allocator); + IntVector int32Vector = new IntVector(new Field("int32", int32Field, null), allocator); + BigIntVector int64Vector = new BigIntVector(new Field("int64", int64Field, null), allocator); + UInt1Vector uint8Vector = new UInt1Vector(new Field("uint8", uint8Field, null), allocator); + UInt2Vector uint16Vector = new UInt2Vector(new Field("uint16", uint16Field, null), allocator); + UInt4Vector uint32Vector = new UInt4Vector(new Field("uint32", uint32Field, null), allocator); + UInt8Vector uint64Vector = new UInt8Vector(new Field("uint64", uint64Field, null), allocator); + + // Set up VSR + List vectors = + Arrays.asList( + int8Vector, + int16Vector, + int32Vector, + int64Vector, + uint8Vector, + uint16Vector, + uint32Vector, + uint64Vector); + + int rowCount = 12; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + for (int row = 0; row < 10; row++) { + int8Vector.set(row, 11 * row * (row % 2 == 0 ? 1 : -1)); + int16Vector.set(row, 63 * row * (row % 2 == 0 ? 1 : -1)); + int32Vector.set(row, 513 * row * (row % 2 == 0 ? 1 : -1)); + int64Vector.set(row, 3791L * row * (row % 2 == 0 ? 1 : -1)); + uint8Vector.set(row, 11 * row); + uint16Vector.set(row, 63 * row); + uint32Vector.set(row, 513 * row); + uint64Vector.set(row, 3791L * row); + } + + // Min values + int8Vector.set(10, Byte.MIN_VALUE); + int16Vector.set(10, Short.MIN_VALUE); + int32Vector.set(10, Integer.MIN_VALUE); + int64Vector.set(10, Long.MIN_VALUE); + uint8Vector.set(10, 0); + uint16Vector.set(10, 0); + uint32Vector.set(10, 0); + uint64Vector.set(10, 0); + + // Max values + int8Vector.set(11, Byte.MAX_VALUE); + int16Vector.set(11, Short.MAX_VALUE); + int32Vector.set(11, Integer.MAX_VALUE); + int64Vector.set(11, Long.MAX_VALUE); + uint8Vector.set(11, 0xff); + uint16Vector.set(11, 0xffff); + uint32Vector.set(11, 0xffffffff); + uint64Vector.set(11, Long.MAX_VALUE); // Max that can be encoded + + File dataFile = new File(TMP, "testWriteIntegers.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + GenericRecord record = null; + + // Read and check values + for (int row = 0; row < rowCount; row++) { + record = datumReader.read(record, decoder); + assertEquals((int) int8Vector.get(row), record.get("int8")); + assertEquals((int) int16Vector.get(row), record.get("int16")); + assertEquals(int32Vector.get(row), record.get("int32")); + assertEquals(int64Vector.get(row), record.get("int64")); + assertEquals(Byte.toUnsignedInt(uint8Vector.get(row)), record.get("uint8")); + assertEquals(Short.toUnsignedInt((short) uint16Vector.get(row)), record.get("uint16")); + assertEquals(Integer.toUnsignedLong(uint32Vector.get(row)), record.get("uint32")); + assertEquals(uint64Vector.get(row), record.get("uint64")); + } + } + } + } + + @Test + public void testWriteNullableIntegers() throws Exception { + + // Field definitions + FieldType int8Field = new FieldType(true, new ArrowType.Int(8, true), null); + FieldType int16Field = new FieldType(true, new ArrowType.Int(16, true), null); + FieldType int32Field = new FieldType(true, new ArrowType.Int(32, true), null); + FieldType int64Field = new FieldType(true, new ArrowType.Int(64, true), null); + FieldType uint8Field = new FieldType(true, new ArrowType.Int(8, false), null); + FieldType uint16Field = new FieldType(true, new ArrowType.Int(16, false), null); + FieldType uint32Field = new FieldType(true, new ArrowType.Int(32, false), null); + FieldType uint64Field = new FieldType(true, new ArrowType.Int(64, false), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + TinyIntVector int8Vector = new TinyIntVector(new Field("int8", int8Field, null), allocator); + SmallIntVector int16Vector = + new SmallIntVector(new Field("int16", int16Field, null), allocator); + IntVector int32Vector = new IntVector(new Field("int32", int32Field, null), allocator); + BigIntVector int64Vector = new BigIntVector(new Field("int64", int64Field, null), allocator); + UInt1Vector uint8Vector = new UInt1Vector(new Field("uint8", uint8Field, null), allocator); + UInt2Vector uint16Vector = new UInt2Vector(new Field("uint16", uint16Field, null), allocator); + UInt4Vector uint32Vector = new UInt4Vector(new Field("uint32", uint32Field, null), allocator); + UInt8Vector uint64Vector = new UInt8Vector(new Field("uint64", uint64Field, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = + Arrays.asList( + int8Vector, + int16Vector, + int32Vector, + int64Vector, + uint8Vector, + uint16Vector, + uint32Vector, + uint64Vector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Null values + int8Vector.setNull(0); + int16Vector.setNull(0); + int32Vector.setNull(0); + int64Vector.setNull(0); + uint8Vector.setNull(0); + uint16Vector.setNull(0); + uint32Vector.setNull(0); + uint64Vector.setNull(0); + + // Zero values + int8Vector.set(1, 0); + int16Vector.set(1, 0); + int32Vector.set(1, 0); + int64Vector.set(1, 0); + uint8Vector.set(1, 0); + uint16Vector.set(1, 0); + uint32Vector.set(1, 0); + uint64Vector.set(1, 0); + + // Non-zero values + int8Vector.set(2, Byte.MAX_VALUE); + int16Vector.set(2, Short.MAX_VALUE); + int32Vector.set(2, Integer.MAX_VALUE); + int64Vector.set(2, Long.MAX_VALUE); + uint8Vector.set(2, Byte.MAX_VALUE); + uint16Vector.set(2, Short.MAX_VALUE); + uint32Vector.set(2, Integer.MAX_VALUE); + uint64Vector.set(2, Long.MAX_VALUE); + + File dataFile = new File(TMP, "testWriteNullableIntegers.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + + // Read and check values + GenericRecord record = datumReader.read(null, decoder); + assertNull(record.get("int8")); + assertNull(record.get("int16")); + assertNull(record.get("int32")); + assertNull(record.get("int64")); + assertNull(record.get("uint8")); + assertNull(record.get("uint16")); + assertNull(record.get("uint32")); + assertNull(record.get("uint64")); + + for (int row = 1; row < rowCount; row++) { + record = datumReader.read(record, decoder); + assertEquals((int) int8Vector.get(row), record.get("int8")); + assertEquals((int) int16Vector.get(row), record.get("int16")); + assertEquals(int32Vector.get(row), record.get("int32")); + assertEquals(int64Vector.get(row), record.get("int64")); + assertEquals(Byte.toUnsignedInt(uint8Vector.get(row)), record.get("uint8")); + assertEquals(Short.toUnsignedInt((short) uint16Vector.get(row)), record.get("uint16")); + assertEquals(Integer.toUnsignedLong(uint32Vector.get(row)), record.get("uint32")); + assertEquals(uint64Vector.get(row), record.get("uint64")); + } + } + } + } + + @Test + public void testWriteFloatingPoints() throws Exception { + + // Field definitions + FieldType float16Field = + new FieldType(false, new ArrowType.FloatingPoint(FloatingPointPrecision.HALF), null); + FieldType float32Field = + new FieldType(false, new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE), null); + FieldType float64Field = + new FieldType(false, new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + Float2Vector float16Vector = + new Float2Vector(new Field("float16", float16Field, null), allocator); + Float4Vector float32Vector = + new Float4Vector(new Field("float32", float32Field, null), allocator); + Float8Vector float64Vector = + new Float8Vector(new Field("float64", float64Field, null), allocator); + + // Set up VSR + List vectors = Arrays.asList(float16Vector, float32Vector, float64Vector); + int rowCount = 15; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + for (int row = 0; row < 10; row++) { + float16Vector.set(row, Float16.toFloat16(3.6f * row * (row % 2 == 0 ? 1.0f : -1.0f))); + float32Vector.set(row, 37.6f * row * (row % 2 == 0 ? 1 : -1)); + float64Vector.set(row, 37.6d * row * (row % 2 == 0 ? 1 : -1)); + } + + float16Vector.set(10, Float16.toFloat16(Float.MIN_VALUE)); + float32Vector.set(10, Float.MIN_VALUE); + float64Vector.set(10, Double.MIN_VALUE); + + float16Vector.set(11, Float16.toFloat16(Float.MAX_VALUE)); + float32Vector.set(11, Float.MAX_VALUE); + float64Vector.set(11, Double.MAX_VALUE); + + float16Vector.set(12, Float16.toFloat16(Float.NaN)); + float32Vector.set(12, Float.NaN); + float64Vector.set(12, Double.NaN); + + float16Vector.set(13, Float16.toFloat16(Float.POSITIVE_INFINITY)); + float32Vector.set(13, Float.POSITIVE_INFINITY); + float64Vector.set(13, Double.POSITIVE_INFINITY); + + float16Vector.set(14, Float16.toFloat16(Float.NEGATIVE_INFINITY)); + float32Vector.set(14, Float.NEGATIVE_INFINITY); + float64Vector.set(14, Double.NEGATIVE_INFINITY); + + File dataFile = new File(TMP, "testWriteFloatingPoints.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + GenericRecord record = null; + + // Read and check values + for (int row = 0; row < rowCount; row++) { + record = datumReader.read(record, decoder); + assertEquals(float16Vector.getValueAsFloat(row), record.get("float16")); + assertEquals(float32Vector.get(row), record.get("float32")); + assertEquals(float64Vector.get(row), record.get("float64")); + } + } + } + } + + @Test + public void testWriteNullableFloatingPoints() throws Exception { + + // Field definitions + FieldType float16Field = + new FieldType(true, new ArrowType.FloatingPoint(FloatingPointPrecision.HALF), null); + FieldType float32Field = + new FieldType(true, new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE), null); + FieldType float64Field = + new FieldType(true, new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + Float2Vector float16Vector = + new Float2Vector(new Field("float16", float16Field, null), allocator); + Float4Vector float32Vector = + new Float4Vector(new Field("float32", float32Field, null), allocator); + Float8Vector float64Vector = + new Float8Vector(new Field("float64", float64Field, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = Arrays.asList(float16Vector, float32Vector, float64Vector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Null values + float16Vector.setNull(0); + float32Vector.setNull(0); + float64Vector.setNull(0); + + // Zero values + float16Vector.setSafeWithPossibleTruncate(1, 0.0f); + float32Vector.set(1, 0.0f); + float64Vector.set(1, 0.0); + + // Non-zero values + float16Vector.setSafeWithPossibleTruncate(2, 1.0f); + float32Vector.set(2, 1.0f); + float64Vector.set(2, 1.0); + + File dataFile = new File(TMP, "testWriteNullableFloatingPoints.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + + // Read and check values + GenericRecord record = datumReader.read(null, decoder); + assertNull(record.get("float16")); + assertNull(record.get("float32")); + assertNull(record.get("float64")); + + for (int row = 1; row < rowCount; row++) { + record = datumReader.read(record, decoder); + assertEquals(float16Vector.getValueAsFloat(row), record.get("float16")); + assertEquals(float32Vector.get(row), record.get("float32")); + assertEquals(float64Vector.get(row), record.get("float64")); + } + } + } + } + + @Test + public void testWriteStrings() throws Exception { + + // Field definition + FieldType stringField = new FieldType(false, new ArrowType.Utf8(), null); + + // Create empty vector + BufferAllocator allocator = new RootAllocator(); + VarCharVector stringVector = + new VarCharVector(new Field("string", stringField, null), allocator); + + // Set up VSR + List vectors = Arrays.asList(stringVector); + int rowCount = 5; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + stringVector.setSafe(0, "Hello world!".getBytes()); + stringVector.setSafe(1, "<%**\r\n\t\\abc\0$$>".getBytes()); + stringVector.setSafe(2, "你好世界".getBytes()); + stringVector.setSafe(3, "مرحبا بالعالم".getBytes()); + stringVector.setSafe(4, "(P ∧ P ⇒ Q) ⇒ Q".getBytes()); + + File dataFile = new File(TMP, "testWriteStrings.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + GenericRecord record = null; + + // Read and check values + for (int row = 0; row < rowCount; row++) { + record = datumReader.read(record, decoder); + assertEquals(stringVector.getObject(row).toString(), record.get("string").toString()); + } + } + } + } + + @Test + public void testWriteNullableStrings() throws Exception { + + // Field definition + FieldType stringField = new FieldType(true, new ArrowType.Utf8(), null); + + // Create empty vector + BufferAllocator allocator = new RootAllocator(); + VarCharVector stringVector = + new VarCharVector(new Field("string", stringField, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = Arrays.asList(stringVector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + stringVector.setNull(0); + stringVector.setSafe(1, "".getBytes()); + stringVector.setSafe(2, "not empty".getBytes()); + + File dataFile = new File(TMP, "testWriteNullableStrings.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + + // Read and check values + GenericRecord record = datumReader.read(null, decoder); + assertNull(record.get("string")); + + for (int row = 1; row < rowCount; row++) { + record = datumReader.read(record, decoder); + assertEquals(stringVector.getObject(row).toString(), record.get("string").toString()); + } + } + } + } + + @Test + public void testWriteBinary() throws Exception { + + // Field definition + FieldType binaryField = new FieldType(false, new ArrowType.Binary(), null); + FieldType fixedField = new FieldType(false, new ArrowType.FixedSizeBinary(5), null); + + // Create empty vector + BufferAllocator allocator = new RootAllocator(); + VarBinaryVector binaryVector = + new VarBinaryVector(new Field("binary", binaryField, null), allocator); + FixedSizeBinaryVector fixedVector = + new FixedSizeBinaryVector(new Field("fixed", fixedField, null), allocator); + + // Set up VSR + List vectors = Arrays.asList(binaryVector, fixedVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + binaryVector.setSafe(0, new byte[] {1, 2, 3}); + binaryVector.setSafe(1, new byte[] {4, 5, 6, 7}); + binaryVector.setSafe(2, new byte[] {8, 9}); + + fixedVector.setSafe(0, new byte[] {1, 2, 3, 4, 5}); + fixedVector.setSafe(1, new byte[] {4, 5, 6, 7, 8, 9}); + fixedVector.setSafe(2, new byte[] {8, 9, 10, 11, 12}); + + File dataFile = new File(TMP, "testWriteBinary.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + GenericRecord record = null; + + // Read and check values + for (int row = 0; row < rowCount; row++) { + record = datumReader.read(record, decoder); + ByteBuffer buf = ((ByteBuffer) record.get("binary")); + byte[] bytes = new byte[buf.remaining()]; + buf.get(bytes); + byte[] fixedBytes = ((GenericData.Fixed) record.get("fixed")).bytes(); + assertArrayEquals(binaryVector.getObject(row), bytes); + assertArrayEquals(fixedVector.getObject(row), fixedBytes); + } + } + } + } + + @Test + public void testWriteNullableBinary() throws Exception { + + // Field definition + FieldType binaryField = new FieldType(true, new ArrowType.Binary(), null); + FieldType fixedField = new FieldType(true, new ArrowType.FixedSizeBinary(5), null); + + // Create empty vector + BufferAllocator allocator = new RootAllocator(); + VarBinaryVector binaryVector = + new VarBinaryVector(new Field("binary", binaryField, null), allocator); + FixedSizeBinaryVector fixedVector = + new FixedSizeBinaryVector(new Field("fixed", fixedField, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = Arrays.asList(binaryVector, fixedVector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + binaryVector.setNull(0); + binaryVector.setSafe(1, new byte[] {}); + binaryVector.setSafe(2, new byte[] {10, 11, 12}); + + fixedVector.setNull(0); + fixedVector.setSafe(1, new byte[] {0, 0, 0, 0, 0}); + fixedVector.setSafe(2, new byte[] {10, 11, 12, 13, 14}); + + File dataFile = new File(TMP, "testWriteNullableBinary.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + + // Read and check values + GenericRecord record = datumReader.read(null, decoder); + assertNull(record.get("binary")); + assertNull(record.get("fixed")); + + for (int row = 1; row < rowCount; row++) { + record = datumReader.read(record, decoder); + ByteBuffer buf = ((ByteBuffer) record.get("binary")); + byte[] bytes = new byte[buf.remaining()]; + buf.get(bytes); + byte[] fixedBytes = ((GenericData.Fixed) record.get("fixed")).bytes(); + assertArrayEquals(binaryVector.getObject(row), bytes); + assertArrayEquals(fixedVector.getObject(row), fixedBytes); + } + } + } + } + + // Data production for logical types, nullable and non-nullable + + @Test + public void testWriteDecimals() throws Exception { + + // Field definitions + FieldType decimal128Field1 = new FieldType(false, new ArrowType.Decimal(38, 10, 128), null); + FieldType decimal128Field2 = new FieldType(false, new ArrowType.Decimal(38, 5, 128), null); + FieldType decimal256Field1 = new FieldType(false, new ArrowType.Decimal(76, 20, 256), null); + FieldType decimal256Field2 = new FieldType(false, new ArrowType.Decimal(76, 10, 256), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + DecimalVector decimal128Vector1 = + new DecimalVector(new Field("decimal128_1", decimal128Field1, null), allocator); + DecimalVector decimal128Vector2 = + new DecimalVector(new Field("decimal128_2", decimal128Field2, null), allocator); + Decimal256Vector decimal256Vector1 = + new Decimal256Vector(new Field("decimal256_1", decimal256Field1, null), allocator); + Decimal256Vector decimal256Vector2 = + new Decimal256Vector(new Field("decimal256_2", decimal256Field2, null), allocator); + + // Set up VSR + List vectors = + Arrays.asList(decimal128Vector1, decimal128Vector2, decimal256Vector1, decimal256Vector2); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + decimal128Vector1.setSafe( + 0, new BigDecimal("12345.67890").setScale(10, RoundingMode.UNNECESSARY)); + decimal128Vector1.setSafe( + 1, new BigDecimal("-98765.43210").setScale(10, RoundingMode.UNNECESSARY)); + decimal128Vector1.setSafe( + 2, new BigDecimal("54321.09876").setScale(10, RoundingMode.UNNECESSARY)); + + decimal128Vector2.setSafe( + 0, new BigDecimal("12345.67890").setScale(5, RoundingMode.UNNECESSARY)); + decimal128Vector2.setSafe( + 1, new BigDecimal("-98765.43210").setScale(5, RoundingMode.UNNECESSARY)); + decimal128Vector2.setSafe( + 2, new BigDecimal("54321.09876").setScale(5, RoundingMode.UNNECESSARY)); + + decimal256Vector1.setSafe( + 0, + new BigDecimal("12345678901234567890.12345678901234567890") + .setScale(20, RoundingMode.UNNECESSARY)); + decimal256Vector1.setSafe( + 1, + new BigDecimal("-98765432109876543210.98765432109876543210") + .setScale(20, RoundingMode.UNNECESSARY)); + decimal256Vector1.setSafe( + 2, + new BigDecimal("54321098765432109876.54321098765432109876") + .setScale(20, RoundingMode.UNNECESSARY)); + + decimal256Vector2.setSafe( + 0, + new BigDecimal("12345678901234567890.1234567890").setScale(10, RoundingMode.UNNECESSARY)); + decimal256Vector2.setSafe( + 1, + new BigDecimal("-98765432109876543210.9876543210") + .setScale(10, RoundingMode.UNNECESSARY)); + decimal256Vector2.setSafe( + 2, + new BigDecimal("54321098765432109876.5432109876").setScale(10, RoundingMode.UNNECESSARY)); + + File dataFile = new File(TMP, "testWriteDecimals.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + GenericRecord record = null; + + // Read and check values + for (int row = 0; row < rowCount; row++) { + record = datumReader.read(record, decoder); + assertEquals( + decimal128Vector1.getObject(row), decodeFixedDecimal(record, "decimal128_1")); + assertEquals( + decimal128Vector2.getObject(row), decodeFixedDecimal(record, "decimal128_2")); + assertEquals( + decimal256Vector1.getObject(row), decodeFixedDecimal(record, "decimal256_1")); + assertEquals( + decimal256Vector2.getObject(row), decodeFixedDecimal(record, "decimal256_2")); + } + } + } + } + + @Test + public void testWriteNullableDecimals() throws Exception { + + // Field definitions + FieldType decimal128Field1 = new FieldType(true, new ArrowType.Decimal(38, 10, 128), null); + FieldType decimal128Field2 = new FieldType(true, new ArrowType.Decimal(38, 5, 128), null); + FieldType decimal256Field1 = new FieldType(true, new ArrowType.Decimal(76, 20, 256), null); + FieldType decimal256Field2 = new FieldType(true, new ArrowType.Decimal(76, 10, 256), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + DecimalVector decimal128Vector1 = + new DecimalVector(new Field("decimal128_1", decimal128Field1, null), allocator); + DecimalVector decimal128Vector2 = + new DecimalVector(new Field("decimal128_2", decimal128Field2, null), allocator); + Decimal256Vector decimal256Vector1 = + new Decimal256Vector(new Field("decimal256_1", decimal256Field1, null), allocator); + Decimal256Vector decimal256Vector2 = + new Decimal256Vector(new Field("decimal256_2", decimal256Field2, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = + Arrays.asList(decimal128Vector1, decimal128Vector2, decimal256Vector1, decimal256Vector2); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + decimal128Vector1.setNull(0); + decimal128Vector1.setSafe(1, BigDecimal.ZERO.setScale(10, RoundingMode.UNNECESSARY)); + decimal128Vector1.setSafe( + 2, new BigDecimal("12345.67890").setScale(10, RoundingMode.UNNECESSARY)); + + decimal128Vector2.setNull(0); + decimal128Vector2.setSafe(1, BigDecimal.ZERO.setScale(5, RoundingMode.UNNECESSARY)); + decimal128Vector2.setSafe( + 2, new BigDecimal("98765.43210").setScale(5, RoundingMode.UNNECESSARY)); + + decimal256Vector1.setNull(0); + decimal256Vector1.setSafe(1, BigDecimal.ZERO.setScale(20, RoundingMode.UNNECESSARY)); + decimal256Vector1.setSafe( + 2, + new BigDecimal("12345678901234567890.12345678901234567890") + .setScale(20, RoundingMode.UNNECESSARY)); + + decimal256Vector2.setNull(0); + decimal256Vector2.setSafe(1, BigDecimal.ZERO.setScale(10, RoundingMode.UNNECESSARY)); + decimal256Vector2.setSafe( + 2, + new BigDecimal("98765432109876543210.9876543210").setScale(10, RoundingMode.UNNECESSARY)); + + File dataFile = new File(TMP, "testWriteNullableDecimals.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + + // Read and check values + GenericRecord record = datumReader.read(null, decoder); + assertNull(record.get("decimal128_1")); + assertNull(record.get("decimal128_2")); + assertNull(record.get("decimal256_1")); + assertNull(record.get("decimal256_2")); + + for (int row = 1; row < rowCount; row++) { + record = datumReader.read(record, decoder); + assertEquals( + decimal128Vector1.getObject(row), decodeFixedDecimal(record, "decimal128_1")); + assertEquals( + decimal128Vector2.getObject(row), decodeFixedDecimal(record, "decimal128_2")); + assertEquals( + decimal256Vector1.getObject(row), decodeFixedDecimal(record, "decimal256_1")); + assertEquals( + decimal256Vector2.getObject(row), decodeFixedDecimal(record, "decimal256_2")); + } + } + } + } + + private static BigDecimal decodeFixedDecimal(GenericRecord record, String fieldName) { + GenericData.Fixed fixed = (GenericData.Fixed) record.get(fieldName); + var logicalType = LogicalTypes.fromSchema(fixed.getSchema()); + return new Conversions.DecimalConversion().fromFixed(fixed, fixed.getSchema(), logicalType); + } + + @Test + public void testWriteDates() throws Exception { + + // Field definitions + FieldType dateDayField = new FieldType(false, new ArrowType.Date(DateUnit.DAY), null); + FieldType dateMillisField = + new FieldType(false, new ArrowType.Date(DateUnit.MILLISECOND), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + DateDayVector dateDayVector = + new DateDayVector(new Field("dateDay", dateDayField, null), allocator); + DateMilliVector dateMillisVector = + new DateMilliVector(new Field("dateMillis", dateMillisField, null), allocator); + + // Set up VSR + List vectors = Arrays.asList(dateDayVector, dateMillisVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + dateDayVector.setSafe(0, (int) LocalDate.now().toEpochDay()); + dateDayVector.setSafe(1, (int) LocalDate.now().toEpochDay() + 1); + dateDayVector.setSafe(2, (int) LocalDate.now().toEpochDay() + 2); + + dateMillisVector.setSafe(0, LocalDate.now().toEpochDay() * 86400000L); + dateMillisVector.setSafe(1, (LocalDate.now().toEpochDay() + 1) * 86400000L); + dateMillisVector.setSafe(2, (LocalDate.now().toEpochDay() + 2) * 86400000L); + + File dataFile = new File(TMP, "testWriteDates.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + GenericRecord record = null; + + // Read and check values + for (int row = 0; row < rowCount; row++) { + record = datumReader.read(record, decoder); + assertEquals(dateDayVector.get(row), record.get("dateDay")); + assertEquals( + dateMillisVector.get(row), ((long) (Integer) record.get("dateMillis")) * 86400000L); + } + } + } + } + + @Test + public void testWriteNullableDates() throws Exception { + + // Field definitions + FieldType dateDayField = new FieldType(true, new ArrowType.Date(DateUnit.DAY), null); + FieldType dateMillisField = new FieldType(true, new ArrowType.Date(DateUnit.MILLISECOND), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + DateDayVector dateDayVector = + new DateDayVector(new Field("dateDay", dateDayField, null), allocator); + DateMilliVector dateMillisVector = + new DateMilliVector(new Field("dateMillis", dateMillisField, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = Arrays.asList(dateDayVector, dateMillisVector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + dateDayVector.setNull(0); + dateDayVector.setSafe(1, 0); + dateDayVector.setSafe(2, (int) LocalDate.now().toEpochDay()); + + dateMillisVector.setNull(0); + dateMillisVector.setSafe(1, 0); + dateMillisVector.setSafe(2, LocalDate.now().toEpochDay() * 86400000L); + + File dataFile = new File(TMP, "testWriteNullableDates.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + + // Read and check values + GenericRecord record = datumReader.read(null, decoder); + assertNull(record.get("dateDay")); + assertNull(record.get("dateMillis")); + + for (int row = 1; row < rowCount; row++) { + record = datumReader.read(record, decoder); + assertEquals(dateDayVector.get(row), record.get("dateDay")); + assertEquals( + dateMillisVector.get(row), ((long) (Integer) record.get("dateMillis")) * 86400000L); + } + } + } + } + + @Test + public void testWriteTimes() throws Exception { + + // Field definitions + FieldType timeSecField = new FieldType(false, new ArrowType.Time(TimeUnit.SECOND, 32), null); + FieldType timeMillisField = + new FieldType(false, new ArrowType.Time(TimeUnit.MILLISECOND, 32), null); + FieldType timeMicrosField = + new FieldType(false, new ArrowType.Time(TimeUnit.MICROSECOND, 64), null); + FieldType timeNanosField = + new FieldType(false, new ArrowType.Time(TimeUnit.NANOSECOND, 64), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + TimeSecVector timeSecVector = + new TimeSecVector(new Field("timeSec", timeSecField, null), allocator); + TimeMilliVector timeMillisVector = + new TimeMilliVector(new Field("timeMillis", timeMillisField, null), allocator); + TimeMicroVector timeMicrosVector = + new TimeMicroVector(new Field("timeMicros", timeMicrosField, null), allocator); + TimeNanoVector timeNanosVector = + new TimeNanoVector(new Field("timeNanos", timeNanosField, null), allocator); + + // Set up VSR + List vectors = + Arrays.asList(timeSecVector, timeMillisVector, timeMicrosVector, timeNanosVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + timeSecVector.setSafe(0, ZonedDateTime.now().toLocalTime().toSecondOfDay()); + timeSecVector.setSafe(1, ZonedDateTime.now().toLocalTime().toSecondOfDay() - 1); + timeSecVector.setSafe(2, ZonedDateTime.now().toLocalTime().toSecondOfDay() - 2); + + timeMillisVector.setSafe( + 0, (int) (ZonedDateTime.now().toLocalTime().toNanoOfDay() / 1000000)); + timeMillisVector.setSafe( + 1, (int) (ZonedDateTime.now().toLocalTime().toNanoOfDay() / 1000000) - 1000); + timeMillisVector.setSafe( + 2, (int) (ZonedDateTime.now().toLocalTime().toNanoOfDay() / 1000000) - 2000); + + timeMicrosVector.setSafe(0, ZonedDateTime.now().toLocalTime().toNanoOfDay() / 1000); + timeMicrosVector.setSafe(1, ZonedDateTime.now().toLocalTime().toNanoOfDay() / 1000 - 1000000); + timeMicrosVector.setSafe(2, ZonedDateTime.now().toLocalTime().toNanoOfDay() / 1000 - 2000000); + + timeNanosVector.setSafe(0, ZonedDateTime.now().toLocalTime().toNanoOfDay()); + timeNanosVector.setSafe(1, ZonedDateTime.now().toLocalTime().toNanoOfDay() - 1000000000); + timeNanosVector.setSafe(2, ZonedDateTime.now().toLocalTime().toNanoOfDay() - 2000000000); + + File dataFile = new File(TMP, "testWriteTimes.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + GenericRecord record = null; + + // Read and check values + for (int row = 0; row < rowCount; row++) { + record = datumReader.read(record, decoder); + assertEquals(timeSecVector.get(row), (int) (record.get("timeSec")) / 1000); + assertEquals(timeMillisVector.get(row), record.get("timeMillis")); + assertEquals(timeMicrosVector.get(row), record.get("timeMicros")); + // Avro doesn't have time-nanos (mar 2025), so expect column to be saved as micros + long nanosAsMicros = (timeNanosVector.get(row) / 1000); + assertEquals(nanosAsMicros, (long) record.get("timeNanos")); + } + } + } + } + + @Test + public void testWriteNullableTimes() throws Exception { + + // Field definitions + FieldType timeSecField = new FieldType(true, new ArrowType.Time(TimeUnit.SECOND, 32), null); + FieldType timeMillisField = + new FieldType(true, new ArrowType.Time(TimeUnit.MILLISECOND, 32), null); + FieldType timeMicrosField = + new FieldType(true, new ArrowType.Time(TimeUnit.MICROSECOND, 64), null); + FieldType timeNanosField = + new FieldType(true, new ArrowType.Time(TimeUnit.NANOSECOND, 64), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + TimeSecVector timeSecVector = + new TimeSecVector(new Field("timeSec", timeSecField, null), allocator); + TimeMilliVector timeMillisVector = + new TimeMilliVector(new Field("timeMillis", timeMillisField, null), allocator); + TimeMicroVector timeMicrosVector = + new TimeMicroVector(new Field("timeMicros", timeMicrosField, null), allocator); + TimeNanoVector timeNanosVector = + new TimeNanoVector(new Field("timeNanos", timeNanosField, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = + Arrays.asList(timeSecVector, timeMillisVector, timeMicrosVector, timeNanosVector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + timeSecVector.setNull(0); + timeSecVector.setSafe(1, 0); + timeSecVector.setSafe(2, ZonedDateTime.now().toLocalTime().toSecondOfDay()); + + timeMillisVector.setNull(0); + timeMillisVector.setSafe(1, 0); + timeMillisVector.setSafe( + 2, (int) (ZonedDateTime.now().toLocalTime().toNanoOfDay() / 1000000)); + + timeMicrosVector.setNull(0); + timeMicrosVector.setSafe(1, 0); + timeMicrosVector.setSafe(2, ZonedDateTime.now().toLocalTime().toNanoOfDay() / 1000); + + timeNanosVector.setNull(0); + timeNanosVector.setSafe(1, 0); + timeNanosVector.setSafe(2, ZonedDateTime.now().toLocalTime().toNanoOfDay()); + + File dataFile = new File(TMP, "testWriteNullableTimes.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + + // Read and check values + GenericRecord record = datumReader.read(null, decoder); + assertNull(record.get("timeSec")); + assertNull(record.get("timeMillis")); + assertNull(record.get("timeMicros")); + assertNull(record.get("timeNanos")); + + for (int row = 1; row < rowCount; row++) { + record = datumReader.read(record, decoder); + assertEquals(timeSecVector.get(row), ((int) record.get("timeSec") / 1000)); + assertEquals(timeMillisVector.get(row), record.get("timeMillis")); + assertEquals(timeMicrosVector.get(row), record.get("timeMicros")); + // Avro doesn't have time-nanos (mar 2025), so expect column to be saved as micros + long nanosAsMicros = (timeNanosVector.get(row) / 1000); + assertEquals(nanosAsMicros, (long) record.get("timeNanos")); + } + } + } + } + + @Test + public void testWriteZoneAwareTimestamps() throws Exception { + + // Field definitions + FieldType timestampSecField = + new FieldType(false, new ArrowType.Timestamp(TimeUnit.SECOND, "UTC"), null); + FieldType timestampMillisField = + new FieldType(false, new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC"), null); + FieldType timestampMicrosField = + new FieldType(false, new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC"), null); + FieldType timestampNanosField = + new FieldType(false, new ArrowType.Timestamp(TimeUnit.NANOSECOND, "UTC"), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + TimeStampSecTZVector timestampSecVector = + new TimeStampSecTZVector(new Field("timestampSec", timestampSecField, null), allocator); + TimeStampMilliTZVector timestampMillisVector = + new TimeStampMilliTZVector( + new Field("timestampMillis", timestampMillisField, null), allocator); + TimeStampMicroTZVector timestampMicrosVector = + new TimeStampMicroTZVector( + new Field("timestampMicros", timestampMicrosField, null), allocator); + TimeStampNanoTZVector timestampNanosVector = + new TimeStampNanoTZVector( + new Field("timestampNanos", timestampNanosField, null), allocator); + + // Set up VSR + List vectors = + Arrays.asList( + timestampSecVector, timestampMillisVector, timestampMicrosVector, timestampNanosVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + timestampSecVector.setSafe(0, (int) Instant.now().getEpochSecond()); + timestampSecVector.setSafe(1, (int) Instant.now().getEpochSecond() - 1); + timestampSecVector.setSafe(2, (int) Instant.now().getEpochSecond() - 2); + + timestampMillisVector.setSafe(0, (int) Instant.now().toEpochMilli()); + timestampMillisVector.setSafe(1, (int) Instant.now().toEpochMilli() - 1000); + timestampMillisVector.setSafe(2, (int) Instant.now().toEpochMilli() - 2000); + + timestampMicrosVector.setSafe(0, Instant.now().toEpochMilli() * 1000); + timestampMicrosVector.setSafe(1, (Instant.now().toEpochMilli() - 1000) * 1000); + timestampMicrosVector.setSafe(2, (Instant.now().toEpochMilli() - 2000) * 1000); + + timestampNanosVector.setSafe(0, Instant.now().toEpochMilli() * 1000000); + timestampNanosVector.setSafe(1, (Instant.now().toEpochMilli() - 1000) * 1000000); + timestampNanosVector.setSafe(2, (Instant.now().toEpochMilli() - 2000) * 1000000); + + File dataFile = new File(TMP, "testWriteZoneAwareTimestamps.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + GenericRecord record = null; + + // Read and check values + for (int row = 0; row < rowCount; row++) { + record = datumReader.read(record, decoder); + assertEquals( + timestampSecVector.get(row), (int) ((long) record.get("timestampSec") / 1000)); + assertEquals(timestampMillisVector.get(row), (int) (long) record.get("timestampMillis")); + assertEquals(timestampMicrosVector.get(row), record.get("timestampMicros")); + assertEquals(timestampNanosVector.get(row), record.get("timestampNanos")); + } + } + } + } + + @Test + public void testWriteNullableZoneAwareTimestamps() throws Exception { + + // Field definitions + FieldType timestampSecField = + new FieldType(true, new ArrowType.Timestamp(TimeUnit.SECOND, "UTC"), null); + FieldType timestampMillisField = + new FieldType(true, new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC"), null); + FieldType timestampMicrosField = + new FieldType(true, new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC"), null); + FieldType timestampNanosField = + new FieldType(true, new ArrowType.Timestamp(TimeUnit.NANOSECOND, "UTC"), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + TimeStampSecTZVector timestampSecVector = + new TimeStampSecTZVector(new Field("timestampSec", timestampSecField, null), allocator); + TimeStampMilliTZVector timestampMillisVector = + new TimeStampMilliTZVector( + new Field("timestampMillis", timestampMillisField, null), allocator); + TimeStampMicroTZVector timestampMicrosVector = + new TimeStampMicroTZVector( + new Field("timestampMicros", timestampMicrosField, null), allocator); + TimeStampNanoTZVector timestampNanosVector = + new TimeStampNanoTZVector( + new Field("timestampNanos", timestampNanosField, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = + Arrays.asList( + timestampSecVector, timestampMillisVector, timestampMicrosVector, timestampNanosVector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + timestampSecVector.setNull(0); + timestampSecVector.setSafe(1, 0); + timestampSecVector.setSafe(2, (int) Instant.now().getEpochSecond()); + + timestampMillisVector.setNull(0); + timestampMillisVector.setSafe(1, 0); + timestampMillisVector.setSafe(2, (int) Instant.now().toEpochMilli()); + + timestampMicrosVector.setNull(0); + timestampMicrosVector.setSafe(1, 0); + timestampMicrosVector.setSafe(2, Instant.now().toEpochMilli() * 1000); + + timestampNanosVector.setNull(0); + timestampNanosVector.setSafe(1, 0); + timestampNanosVector.setSafe(2, Instant.now().toEpochMilli() * 1000000); + + File dataFile = new File(TMP, "testWriteNullableZoneAwareTimestamps.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + + // Read and check values + GenericRecord record = datumReader.read(null, decoder); + assertNull(record.get("timestampSec")); + assertNull(record.get("timestampMillis")); + assertNull(record.get("timestampMicros")); + assertNull(record.get("timestampNanos")); + + for (int row = 1; row < rowCount; row++) { + record = datumReader.read(record, decoder); + assertEquals( + timestampSecVector.get(row), (int) ((long) record.get("timestampSec") / 1000)); + assertEquals(timestampMillisVector.get(row), (int) (long) record.get("timestampMillis")); + assertEquals(timestampMicrosVector.get(row), record.get("timestampMicros")); + assertEquals(timestampNanosVector.get(row), record.get("timestampNanos")); + } + } + } + } + + @Test + public void testWriteLocalTimestamps() throws Exception { + + // Field definitions + FieldType timestampSecField = + new FieldType(false, new ArrowType.Timestamp(TimeUnit.SECOND, null), null); + FieldType timestampMillisField = + new FieldType(false, new ArrowType.Timestamp(TimeUnit.MILLISECOND, null), null); + FieldType timestampMicrosField = + new FieldType(false, new ArrowType.Timestamp(TimeUnit.MICROSECOND, null), null); + FieldType timestampNanosField = + new FieldType(false, new ArrowType.Timestamp(TimeUnit.NANOSECOND, null), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + TimeStampSecVector timestampSecVector = + new TimeStampSecVector(new Field("timestampSec", timestampSecField, null), allocator); + TimeStampMilliVector timestampMillisVector = + new TimeStampMilliVector( + new Field("timestampMillis", timestampMillisField, null), allocator); + TimeStampMicroVector timestampMicrosVector = + new TimeStampMicroVector( + new Field("timestampMicros", timestampMicrosField, null), allocator); + TimeStampNanoVector timestampNanosVector = + new TimeStampNanoVector(new Field("timestampNanos", timestampNanosField, null), allocator); + + // Set up VSR + List vectors = + Arrays.asList( + timestampSecVector, timestampMillisVector, timestampMicrosVector, timestampNanosVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + timestampSecVector.setSafe(0, (int) Instant.now().getEpochSecond()); + timestampSecVector.setSafe(1, (int) Instant.now().getEpochSecond() - 1); + timestampSecVector.setSafe(2, (int) Instant.now().getEpochSecond() - 2); + + timestampMillisVector.setSafe(0, (int) Instant.now().toEpochMilli()); + timestampMillisVector.setSafe(1, (int) Instant.now().toEpochMilli() - 1000); + timestampMillisVector.setSafe(2, (int) Instant.now().toEpochMilli() - 2000); + + timestampMicrosVector.setSafe(0, Instant.now().toEpochMilli() * 1000); + timestampMicrosVector.setSafe(1, (Instant.now().toEpochMilli() - 1000) * 1000); + timestampMicrosVector.setSafe(2, (Instant.now().toEpochMilli() - 2000) * 1000); + + timestampNanosVector.setSafe(0, Instant.now().toEpochMilli() * 1000000); + timestampNanosVector.setSafe(1, (Instant.now().toEpochMilli() - 1000) * 1000000); + timestampNanosVector.setSafe(2, (Instant.now().toEpochMilli() - 2000) * 1000000); + + File dataFile = new File(TMP, "testWriteZoneAwareTimestamps.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + GenericRecord record = null; + + // Read and check values + for (int row = 0; row < rowCount; row++) { + record = datumReader.read(record, decoder); + assertEquals( + timestampSecVector.get(row), (int) ((long) record.get("timestampSec") / 1000)); + assertEquals(timestampMillisVector.get(row), (int) (long) record.get("timestampMillis")); + assertEquals(timestampMicrosVector.get(row), record.get("timestampMicros")); + assertEquals(timestampNanosVector.get(row), record.get("timestampNanos")); + } + } + } + } + + @Test + public void testWriteNullableLocalTimestamps() throws Exception { + + // Field definitions + FieldType timestampSecField = + new FieldType(true, new ArrowType.Timestamp(TimeUnit.SECOND, null), null); + FieldType timestampMillisField = + new FieldType(true, new ArrowType.Timestamp(TimeUnit.MILLISECOND, null), null); + FieldType timestampMicrosField = + new FieldType(true, new ArrowType.Timestamp(TimeUnit.MICROSECOND, null), null); + FieldType timestampNanosField = + new FieldType(true, new ArrowType.Timestamp(TimeUnit.NANOSECOND, null), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + TimeStampSecVector timestampSecVector = + new TimeStampSecVector(new Field("timestampSec", timestampSecField, null), allocator); + TimeStampMilliVector timestampMillisVector = + new TimeStampMilliVector( + new Field("timestampMillis", timestampMillisField, null), allocator); + TimeStampMicroVector timestampMicrosVector = + new TimeStampMicroVector( + new Field("timestampMicros", timestampMicrosField, null), allocator); + TimeStampNanoVector timestampNanosVector = + new TimeStampNanoVector(new Field("timestampNanos", timestampNanosField, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = + Arrays.asList( + timestampSecVector, timestampMillisVector, timestampMicrosVector, timestampNanosVector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + timestampSecVector.setNull(0); + timestampSecVector.setSafe(1, 0); + timestampSecVector.setSafe(2, (int) Instant.now().getEpochSecond()); + + timestampMillisVector.setNull(0); + timestampMillisVector.setSafe(1, 0); + timestampMillisVector.setSafe(2, (int) Instant.now().toEpochMilli()); + + timestampMicrosVector.setNull(0); + timestampMicrosVector.setSafe(1, 0); + timestampMicrosVector.setSafe(2, Instant.now().toEpochMilli() * 1000); + + timestampNanosVector.setNull(0); + timestampNanosVector.setSafe(1, 0); + timestampNanosVector.setSafe(2, Instant.now().toEpochMilli() * 1000000); + + File dataFile = new File(TMP, "testWriteNullableZoneAwareTimestamps.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + + // Read and check values + GenericRecord record = datumReader.read(null, decoder); + assertNull(record.get("timestampSec")); + assertNull(record.get("timestampMillis")); + assertNull(record.get("timestampMicros")); + assertNull(record.get("timestampNanos")); + + for (int row = 1; row < rowCount; row++) { + record = datumReader.read(record, decoder); + assertEquals( + timestampSecVector.get(row), (int) ((long) record.get("timestampSec") / 1000)); + assertEquals(timestampMillisVector.get(row), (int) (long) record.get("timestampMillis")); + assertEquals(timestampMicrosVector.get(row), record.get("timestampMicros")); + assertEquals(timestampNanosVector.get(row), record.get("timestampNanos")); + } + } + } + } + + // Data production for containers of primitive and logical types, nullable and non-nullable + + @Test + public void testWriteLists() throws Exception { + + // Field definitions + FieldType intListField = new FieldType(false, new ArrowType.List(), null); + FieldType stringListField = new FieldType(false, new ArrowType.List(), null); + FieldType dateListField = new FieldType(false, new ArrowType.List(), null); + + Field intField = new Field("item", FieldType.notNullable(new ArrowType.Int(32, true)), null); + Field stringField = new Field("item", FieldType.notNullable(new ArrowType.Utf8()), null); + Field dateField = + new Field("item", FieldType.notNullable(new ArrowType.Date(DateUnit.DAY)), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + ListVector intListVector = new ListVector("intList", allocator, intListField, null); + ListVector stringListVector = new ListVector("stringList", allocator, stringListField, null); + ListVector dateListVector = new ListVector("dateList", allocator, dateListField, null); + + intListVector.initializeChildrenFromFields(Arrays.asList(intField)); + stringListVector.initializeChildrenFromFields(Arrays.asList(stringField)); + dateListVector.initializeChildrenFromFields(Arrays.asList(dateField)); + + // Set up VSR + List vectors = Arrays.asList(intListVector, stringListVector, dateListVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + FieldWriter intListWriter = intListVector.getWriter(); + FieldWriter stringListWriter = stringListVector.getWriter(); + FieldWriter dateListWriter = dateListVector.getWriter(); + + // Set test data for intList + for (int i = 0; i < rowCount; i++) { + intListWriter.startList(); + for (int j = 0; j < 5 - i; j++) { + intListWriter.writeInt(j); + } + intListWriter.endList(); + } + + // Set test data for stringList + for (int i = 0; i < rowCount; i++) { + stringListWriter.startList(); + for (int j = 0; j < 5 - i; j++) { + stringListWriter.writeVarChar("string" + j); + } + stringListWriter.endList(); + } + + // Set test data for dateList + for (int i = 0; i < rowCount; i++) { + dateListWriter.startList(); + for (int j = 0; j < 5 - i; j++) { + dateListWriter.writeDateDay((int) LocalDate.now().plusDays(j).toEpochDay()); + } + dateListWriter.endList(); + } + + // Update count for the vectors + intListVector.setValueCount(rowCount); + stringListVector.setValueCount(rowCount); + dateListVector.setValueCount(rowCount); + + File dataFile = new File(TMP, "testWriteLists.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + GenericRecord record = null; + + // Read and check values + for (int row = 0; row < rowCount; row++) { + record = datumReader.read(record, decoder); + assertEquals(intListVector.getObject(row), record.get("intList")); + assertEquals(dateListVector.getObject(row), record.get("dateList")); + // Handle conversion from Arrow Text type + List vectorList = stringListVector.getObject(row); + List recordList = (List) record.get("stringList"); + assertEquals(vectorList.size(), recordList.size()); + for (int i = 0; i < vectorList.size(); i++) { + assertEquals(vectorList.get(i).toString(), recordList.get(i).toString()); + } + } + } + } + } + + @Test + public void testWriteNullableLists() throws Exception { + + // Field definitions + FieldType nullListType = new FieldType(true, new ArrowType.List(), null); + FieldType nonNullListType = new FieldType(false, new ArrowType.List(), null); + + Field nullFieldType = new Field("item", FieldType.nullable(new ArrowType.Int(32, true)), null); + Field nonNullFieldType = + new Field("item", FieldType.notNullable(new ArrowType.Int(32, true)), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + ListVector nullEntriesVector = + new ListVector("nullEntriesVector", allocator, nonNullListType, null); + ListVector nullListVector = new ListVector("nullListVector", allocator, nullListType, null); + ListVector nullBothVector = new ListVector("nullBothVector", allocator, nullListType, null); + + nullEntriesVector.initializeChildrenFromFields(Arrays.asList(nullFieldType)); + nullListVector.initializeChildrenFromFields(Arrays.asList(nonNullFieldType)); + nullBothVector.initializeChildrenFromFields(Arrays.asList(nullFieldType)); + + // Set up VSR + List vectors = Arrays.asList(nullEntriesVector, nullListVector, nullBothVector); + int rowCount = 4; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data for nullEntriesVector + FieldWriter nullEntriesWriter = nullEntriesVector.getWriter(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeNull(); + nullEntriesWriter.integer().writeNull(); + nullEntriesWriter.endList(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeInt(0); + nullEntriesWriter.integer().writeInt(0); + nullEntriesWriter.endList(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeInt(123); + nullEntriesWriter.integer().writeInt(456); + nullEntriesWriter.endList(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeInt(789); + nullEntriesWriter.integer().writeInt(789); + nullEntriesWriter.endList(); + + // Set test data for nullListVector + FieldWriter nullListWriter = nullListVector.getWriter(); + nullListWriter.writeNull(); + nullListWriter.setPosition(1); // writeNull() does not inc. idx() on list vector + nullListWriter.startList(); + nullListWriter.integer().writeInt(0); + nullListWriter.integer().writeInt(0); + nullListWriter.endList(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeInt(123); + nullEntriesWriter.integer().writeInt(456); + nullEntriesWriter.endList(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeInt(789); + nullEntriesWriter.integer().writeInt(789); + nullEntriesWriter.endList(); + + // Set test data for nullBothVector + FieldWriter nullBothWriter = nullBothVector.getWriter(); + nullBothWriter.writeNull(); + nullBothWriter.setPosition(1); + nullBothWriter.startList(); + nullBothWriter.integer().writeNull(); + nullBothWriter.integer().writeNull(); + nullBothWriter.endList(); + nullListWriter.startList(); + nullListWriter.integer().writeInt(0); + nullListWriter.integer().writeInt(0); + nullListWriter.endList(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeInt(123); + nullEntriesWriter.integer().writeInt(456); + nullEntriesWriter.endList(); + + // Update count for the vectors + nullListVector.setValueCount(4); + nullEntriesVector.setValueCount(4); + nullBothVector.setValueCount(4); + + File dataFile = new File(TMP, "testWriteNullableLists.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + GenericRecord record = null; + + // Read and check values + for (int row = 0; row < rowCount; row++) { + record = datumReader.read(record, decoder); + for (String list : + Arrays.asList("nullEntriesVector", "nullListVector", "nullBothVector")) { + ListVector vector = (ListVector) root.getVector(list); + Object recordField = record.get(list); + if (vector.isNull(row)) { + assertNull(recordField); + } else { + assertEquals(vector.getObject(row), recordField); + } + } + } + } + } + } + + @Test + public void testWriteFixedLists() throws Exception { + + // Field definitions + FieldType intListField = new FieldType(false, new ArrowType.FixedSizeList(5), null); + FieldType stringListField = new FieldType(false, new ArrowType.FixedSizeList(5), null); + FieldType dateListField = new FieldType(false, new ArrowType.FixedSizeList(5), null); + + Field intField = new Field("item", FieldType.notNullable(new ArrowType.Int(32, true)), null); + Field stringField = new Field("item", FieldType.notNullable(new ArrowType.Utf8()), null); + Field dateField = + new Field("item", FieldType.notNullable(new ArrowType.Date(DateUnit.DAY)), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + FixedSizeListVector intListVector = + new FixedSizeListVector("intList", allocator, intListField, null); + FixedSizeListVector stringListVector = + new FixedSizeListVector("stringList", allocator, stringListField, null); + FixedSizeListVector dateListVector = + new FixedSizeListVector("dateList", allocator, dateListField, null); + + intListVector.initializeChildrenFromFields(Arrays.asList(intField)); + stringListVector.initializeChildrenFromFields(Arrays.asList(stringField)); + dateListVector.initializeChildrenFromFields(Arrays.asList(dateField)); + + // Set up VSR + List vectors = Arrays.asList(intListVector, stringListVector, dateListVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + FieldWriter intListWriter = intListVector.getWriter(); + FieldWriter stringListWriter = stringListVector.getWriter(); + FieldWriter dateListWriter = dateListVector.getWriter(); + + // Set test data for intList + for (int i = 0; i < rowCount; i++) { + intListWriter.startList(); + for (int j = 0; j < 5; j++) { + intListWriter.writeInt(j); + } + intListWriter.endList(); + } + + // Set test data for stringList + for (int i = 0; i < rowCount; i++) { + stringListWriter.startList(); + for (int j = 0; j < 5; j++) { + stringListWriter.writeVarChar("string" + j); + } + stringListWriter.endList(); + } + + // Set test data for dateList + for (int i = 0; i < rowCount; i++) { + dateListWriter.startList(); + for (int j = 0; j < 5; j++) { + dateListWriter.writeDateDay((int) LocalDate.now().plusDays(j).toEpochDay()); + } + dateListWriter.endList(); + } + File dataFile = new File(TMP, "testWriteFixedLists.avro"); + + // Update count for the vectors + intListVector.setValueCount(rowCount); + stringListVector.setValueCount(rowCount); + dateListVector.setValueCount(rowCount); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + GenericRecord record = null; + + // Read and check values + for (int row = 0; row < rowCount; row++) { + record = datumReader.read(record, decoder); + assertEquals(intListVector.getObject(row), record.get("intList")); + assertEquals(dateListVector.getObject(row), record.get("dateList")); + // Handle conversion from Arrow Text type + List vectorList = stringListVector.getObject(row); + List recordList = (List) record.get("stringList"); + assertEquals(vectorList.size(), recordList.size()); + for (int i = 0; i < vectorList.size(); i++) { + assertEquals(vectorList.get(i).toString(), recordList.get(i).toString()); + } + } + } + } + } + + @Test + public void testWriteNullableFixedLists() throws Exception { + + // Field definitions + FieldType nullListType = new FieldType(true, new ArrowType.FixedSizeList(2), null); + FieldType nonNullListType = new FieldType(false, new ArrowType.FixedSizeList(2), null); + + Field nullFieldType = new Field("item", FieldType.nullable(new ArrowType.Int(32, true)), null); + Field nonNullFieldType = + new Field("item", FieldType.notNullable(new ArrowType.Int(32, true)), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + FixedSizeListVector nullEntriesVector = + new FixedSizeListVector("nullEntriesVector", allocator, nonNullListType, null); + FixedSizeListVector nullListVector = + new FixedSizeListVector("nullListVector", allocator, nullListType, null); + FixedSizeListVector nullBothVector = + new FixedSizeListVector("nullBothVector", allocator, nullListType, null); + + nullEntriesVector.initializeChildrenFromFields(Arrays.asList(nullFieldType)); + nullListVector.initializeChildrenFromFields(Arrays.asList(nonNullFieldType)); + nullBothVector.initializeChildrenFromFields(Arrays.asList(nullFieldType)); + + // Set up VSR + List vectors = Arrays.asList(nullEntriesVector, nullListVector, nullBothVector); + int rowCount = 4; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data for nullEntriesVector + FieldWriter nullEntriesWriter = nullEntriesVector.getWriter(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeNull(); + nullEntriesWriter.integer().writeNull(); + nullEntriesWriter.endList(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeInt(0); + nullEntriesWriter.integer().writeInt(0); + nullEntriesWriter.endList(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeInt(123); + nullEntriesWriter.integer().writeInt(456); + nullEntriesWriter.endList(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeInt(789); + nullEntriesWriter.integer().writeInt(789); + nullEntriesWriter.endList(); + + // Set test data for nullListVector + FieldWriter nullListWriter = nullListVector.getWriter(); + nullListWriter.writeNull(); + nullListWriter.setPosition(1); // writeNull() does not inc. idx() on list vector + nullListWriter.startList(); + nullListWriter.integer().writeInt(123); + nullListWriter.integer().writeInt(456); + nullListWriter.endList(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeInt(789); + nullEntriesWriter.integer().writeInt(456); + nullEntriesWriter.endList(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeInt(12345); + nullEntriesWriter.integer().writeInt(67891); + nullEntriesWriter.endList(); + + // Set test data for nullBothVector + FieldWriter nullBothWriter = nullBothVector.getWriter(); + nullBothWriter.writeNull(); + nullBothWriter.setPosition(1); + nullBothWriter.startList(); + nullListWriter.integer().writeNull(); + nullListWriter.integer().writeNull(); + nullBothWriter.endList(); + nullListWriter.startList(); + nullListWriter.integer().writeInt(123); + nullListWriter.integer().writeInt(456); + nullListWriter.endList(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeInt(789); + nullEntriesWriter.integer().writeInt(456); + nullEntriesWriter.endList(); + + // Update count for the vectors + nullListVector.setValueCount(4); + nullEntriesVector.setValueCount(4); + nullBothVector.setValueCount(4); + + File dataFile = new File(TMP, "testWriteNullableFixedLists.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + GenericRecord record = null; + + // Read and check values + for (int row = 0; row < rowCount; row++) { + record = datumReader.read(record, decoder); + for (String list : + Arrays.asList("nullEntriesVector", "nullListVector", "nullBothVector")) { + FixedSizeListVector vector = (FixedSizeListVector) root.getVector(list); + Object recordField = record.get(list); + if (vector.isNull(row)) { + assertNull(recordField); + } else { + assertEquals(vector.getObject(row), recordField); + } + } + } + } + } + } + + @Test + public void testWriteMap() throws Exception { + + // Field definitions + FieldType intMapField = new FieldType(false, new ArrowType.Map(false), null); + FieldType stringMapField = new FieldType(false, new ArrowType.Map(false), null); + FieldType dateMapField = new FieldType(false, new ArrowType.Map(false), null); + + Field keyField = new Field("key", FieldType.notNullable(new ArrowType.Utf8()), null); + Field intField = new Field("value", FieldType.notNullable(new ArrowType.Int(32, true)), null); + Field stringField = new Field("value", FieldType.notNullable(new ArrowType.Utf8()), null); + Field dateField = + new Field("value", FieldType.notNullable(new ArrowType.Date(DateUnit.DAY)), null); + + Field intEntryField = + new Field( + "entries", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList(keyField, intField)); + Field stringEntryField = + new Field( + "entries", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList(keyField, stringField)); + Field dateEntryField = + new Field( + "entries", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList(keyField, dateField)); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + MapVector intMapVector = new MapVector("intMap", allocator, intMapField, null); + MapVector stringMapVector = new MapVector("stringMap", allocator, stringMapField, null); + MapVector dateMapVector = new MapVector("dateMap", allocator, dateMapField, null); + + intMapVector.initializeChildrenFromFields(Arrays.asList(intEntryField)); + stringMapVector.initializeChildrenFromFields(Arrays.asList(stringEntryField)); + dateMapVector.initializeChildrenFromFields(Arrays.asList(dateEntryField)); + + // Set up VSR + List vectors = Arrays.asList(intMapVector, stringMapVector, dateMapVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Total number of entries that will be writen to each vector + int entryCount = 5 + 4 + 3; + + // Set test data for intList + BaseWriter.MapWriter writer = intMapVector.getWriter(); + for (int i = 0; i < rowCount; i++) { + writer.startMap(); + for (int j = 0; j < 5 - i; j++) { + writer.startEntry(); + writer.key().varChar().writeVarChar("key" + j); + writer.value().integer().writeInt(j); + writer.endEntry(); + } + writer.endMap(); + } + + // Update count for data vector (map writer does not do this) + intMapVector.getDataVector().setValueCount(entryCount); + + // Set test data for stringList + BaseWriter.MapWriter stringWriter = stringMapVector.getWriter(); + for (int i = 0; i < rowCount; i++) { + stringWriter.startMap(); + for (int j = 0; j < 5 - i; j++) { + stringWriter.startEntry(); + stringWriter.key().varChar().writeVarChar("key" + j); + stringWriter.value().varChar().writeVarChar("string" + j); + stringWriter.endEntry(); + } + stringWriter.endMap(); + } + + // Update count for the vectors + intMapVector.setValueCount(rowCount); + stringMapVector.setValueCount(rowCount); + dateMapVector.setValueCount(rowCount); + + // Update count for data vector (map writer does not do this) + stringMapVector.getDataVector().setValueCount(entryCount); + + // Set test data for dateList + BaseWriter.MapWriter dateWriter = dateMapVector.getWriter(); + for (int i = 0; i < rowCount; i++) { + dateWriter.startMap(); + for (int j = 0; j < 5 - i; j++) { + dateWriter.startEntry(); + dateWriter.key().varChar().writeVarChar("key" + j); + dateWriter.value().dateDay().writeDateDay((int) LocalDate.now().plusDays(j).toEpochDay()); + dateWriter.endEntry(); + } + dateWriter.endMap(); + } + + // Update count for data vector (map writer does not do this) + dateMapVector.getDataVector().setValueCount(entryCount); + + File dataFile = new File(TMP, "testWriteMap.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + GenericRecord record = null; + + // Read and check values + for (int row = 0; row < rowCount; row++) { + record = datumReader.read(record, decoder); + Map intMap = convertMap(intMapVector.getObject(row)); + Map stringMap = convertMap(stringMapVector.getObject(row)); + Map dateMap = convertMap(dateMapVector.getObject(row)); + compareMaps(intMap, (Map) record.get("intMap")); + compareMaps(stringMap, (Map) record.get("stringMap")); + compareMaps(dateMap, (Map) record.get("dateMap")); + } + } + } + } + + @Test + public void testWriteNullableMap() throws Exception { + + // Field definitions + FieldType nullMapType = new FieldType(true, new ArrowType.Map(false), null); + FieldType nonNullMapType = new FieldType(false, new ArrowType.Map(false), null); + + Field keyField = new Field("key", FieldType.notNullable(new ArrowType.Utf8()), null); + Field nullFieldType = new Field("value", FieldType.nullable(new ArrowType.Int(32, true)), null); + Field nonNullFieldType = + new Field("value", FieldType.notNullable(new ArrowType.Int(32, true)), null); + Field nullEntryField = + new Field( + "entries", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList(keyField, nullFieldType)); + Field nonNullEntryField = + new Field( + "entries", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList(keyField, nonNullFieldType)); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + MapVector nullEntriesVector = + new MapVector("nullEntriesVector", allocator, nonNullMapType, null); + MapVector nullMapVector = new MapVector("nullMapVector", allocator, nullMapType, null); + MapVector nullBothVector = new MapVector("nullBothVector", allocator, nullMapType, null); + + nullEntriesVector.initializeChildrenFromFields(Arrays.asList(nullEntryField)); + nullMapVector.initializeChildrenFromFields(Arrays.asList(nonNullEntryField)); + nullBothVector.initializeChildrenFromFields(Arrays.asList(nullEntryField)); + + // Set up VSR + List vectors = Arrays.asList(nullEntriesVector, nullMapVector, nullBothVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data for intList + BaseWriter.MapWriter writer = nullEntriesVector.getWriter(); + writer.startMap(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key0"); + writer.value().integer().writeNull(); + writer.endEntry(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key1"); + writer.value().integer().writeNull(); + writer.endEntry(); + writer.endMap(); + writer.startMap(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key2"); + writer.value().integer().writeInt(0); + writer.endEntry(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key3"); + writer.value().integer().writeInt(0); + writer.endEntry(); + writer.endMap(); + writer.startMap(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key4"); + writer.value().integer().writeInt(123); + writer.endEntry(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key5"); + writer.value().integer().writeInt(456); + writer.endEntry(); + writer.endMap(); + + // Set test data for stringList + BaseWriter.MapWriter nullMapWriter = nullMapVector.getWriter(); + nullMapWriter.writeNull(); + nullMapWriter.setPosition(1); // writeNull() does not inc. idx() on map (list) vector + nullMapWriter.startMap(); + nullMapWriter.startEntry(); + nullMapWriter.key().varChar().writeVarChar("key2"); + nullMapWriter.value().integer().writeInt(0); + nullMapWriter.endEntry(); + writer.startMap(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key3"); + writer.value().integer().writeInt(0); + writer.endEntry(); + nullMapWriter.endMap(); + nullMapWriter.startMap(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key4"); + writer.value().integer().writeInt(123); + writer.endEntry(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key5"); + writer.value().integer().writeInt(456); + writer.endEntry(); + nullMapWriter.endMap(); + + // Set test data for dateList + BaseWriter.MapWriter nullBothWriter = nullBothVector.getWriter(); + nullBothWriter.writeNull(); + nullBothWriter.setPosition(1); + nullBothWriter.startMap(); + nullBothWriter.startEntry(); + nullBothWriter.key().varChar().writeVarChar("key2"); + nullBothWriter.value().integer().writeNull(); + nullBothWriter.endEntry(); + nullBothWriter.startEntry(); + nullBothWriter.key().varChar().writeVarChar("key3"); + nullBothWriter.value().integer().writeNull(); + nullBothWriter.endEntry(); + nullBothWriter.endMap(); + nullBothWriter.startMap(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key4"); + writer.value().integer().writeInt(123); + writer.endEntry(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key5"); + writer.value().integer().writeInt(456); + writer.endEntry(); + nullBothWriter.endMap(); + + // Update count for the vectors + nullEntriesVector.setValueCount(3); + nullMapVector.setValueCount(3); + nullBothVector.setValueCount(3); + + File dataFile = new File(TMP, "testWriteNullableMap.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + GenericRecord record = null; + + // Read and check values + for (int row = 0; row < rowCount; row++) { + record = datumReader.read(record, decoder); + Map intMap = convertMap(nullEntriesVector.getObject(row)); + Map stringMap = convertMap(nullMapVector.getObject(row)); + Map dateMap = convertMap(nullBothVector.getObject(row)); + compareMaps(intMap, (Map) record.get("nullEntriesVector")); + compareMaps(stringMap, (Map) record.get("nullMapVector")); + compareMaps(dateMap, (Map) record.get("nullBothVector")); + } + } + } + } + + private Map convertMap(List entryList) { + + if (entryList == null) { + return null; + } + + Map map = new HashMap<>(); + JsonStringArrayList structList = (JsonStringArrayList) entryList; + for (Object entry : structList) { + JsonStringHashMap structEntry = (JsonStringHashMap) entry; + String key = structEntry.get(MapVector.KEY_NAME).toString(); + Object value = structEntry.get(MapVector.VALUE_NAME); + map.put(key, value); + } + return map; + } + + private void compareMaps(Map expected, Map actual) { + if (expected == null) { + assertNull(actual); + } else { + assertEquals(expected.size(), actual.size()); + for (Object key : actual.keySet()) { + assertTrue(expected.containsKey(key.toString())); + Object actualValue = actual.get(key); + if (actualValue instanceof Utf8) { + assertEquals(expected.get(key.toString()).toString(), actualValue.toString()); + } else { + assertEquals(expected.get(key.toString()), actual.get(key)); + } + } + } + } + + @Test + public void testWriteStruct() throws Exception { + + // Field definitions + FieldType structFieldType = new FieldType(false, new ArrowType.Struct(), null); + Field intField = + new Field("intField", FieldType.notNullable(new ArrowType.Int(32, true)), null); + Field stringField = new Field("stringField", FieldType.notNullable(new ArrowType.Utf8()), null); + Field dateField = + new Field("dateField", FieldType.notNullable(new ArrowType.Date(DateUnit.DAY)), null); + + // Create empty vector + BufferAllocator allocator = new RootAllocator(); + StructVector structVector = new StructVector("struct", allocator, structFieldType, null); + structVector.initializeChildrenFromFields(Arrays.asList(intField, stringField, dateField)); + + // Set up VSR + List vectors = Arrays.asList(structVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + BaseWriter.StructWriter structWriter = structVector.getWriter(); + + for (int i = 0; i < rowCount; i++) { + structWriter.start(); + structWriter.integer("intField").writeInt(i); + structWriter.varChar("stringField").writeVarChar("string" + i); + structWriter.dateDay("dateField").writeDateDay((int) LocalDate.now().toEpochDay() + i); + structWriter.end(); + } + + File dataFile = new File(TMP, "testWriteStruct.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Update count for the vector + structVector.setValueCount(rowCount); + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + GenericRecord record = null; + + // Read and check values + for (int row = 0; row < rowCount; row++) { + record = datumReader.read(record, decoder); + assertNotNull(record.get("struct")); + GenericRecord structRecord = (GenericRecord) record.get("struct"); + assertEquals(row, structRecord.get("intField")); + assertEquals("string" + row, structRecord.get("stringField").toString()); + assertEquals((int) LocalDate.now().toEpochDay() + row, structRecord.get("dateField")); + } + } + } + } + + @Test + public void testWriteNullableStructs() throws Exception { + + // Field definitions + FieldType structFieldType = new FieldType(false, new ArrowType.Struct(), null); + FieldType nullableStructFieldType = new FieldType(true, new ArrowType.Struct(), null); + Field intField = + new Field("intField", FieldType.notNullable(new ArrowType.Int(32, true)), null); + Field nullableIntField = + new Field("nullableIntField", FieldType.nullable(new ArrowType.Int(32, true)), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + StructVector structVector = new StructVector("struct", allocator, structFieldType, null); + StructVector nullableStructVector = + new StructVector("nullableStruct", allocator, nullableStructFieldType, null); + structVector.initializeChildrenFromFields(Arrays.asList(intField, nullableIntField)); + nullableStructVector.initializeChildrenFromFields(Arrays.asList(intField, nullableIntField)); + + // Set up VSR + List vectors = Arrays.asList(structVector, nullableStructVector); + int rowCount = 4; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data for structVector + BaseWriter.StructWriter structWriter = structVector.getWriter(); + for (int i = 0; i < rowCount; i++) { + structWriter.setPosition(i); + structWriter.start(); + structWriter.integer("intField").writeInt(i); + if (i % 2 == 0) { + structWriter.integer("nullableIntField").writeInt(i * 10); + } else { + structWriter.integer("nullableIntField").writeNull(); + } + structWriter.end(); + } + + // Set test data for nullableStructVector + BaseWriter.StructWriter nullableStructWriter = nullableStructVector.getWriter(); + for (int i = 0; i < rowCount; i++) { + nullableStructWriter.setPosition(i); + if (i >= 2) { + nullableStructWriter.start(); + nullableStructWriter.integer("intField").writeInt(i); + if (i % 2 == 0) { + nullableStructWriter.integer("nullableIntField").writeInt(i * 10); + } else { + nullableStructWriter.integer("nullableIntField").writeNull(); + } + nullableStructWriter.end(); + } else { + nullableStructWriter.writeNull(); + } + } + + // Update count for the vector + structVector.setValueCount(rowCount); + nullableStructVector.setValueCount(rowCount); + + File dataFile = new File(TMP, "testWriteNullableStructs.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + GenericRecord record = null; + + // Read and check values + for (int row = 0; row < rowCount; row++) { + record = datumReader.read(record, decoder); + if (row % 2 == 0) { + assertNotNull(record.get("struct")); + GenericRecord structRecord = (GenericRecord) record.get("struct"); + assertEquals(row, structRecord.get("intField")); + assertEquals(row * 10, structRecord.get("nullableIntField")); + } else { + assertNotNull(record.get("struct")); + GenericRecord structRecord = (GenericRecord) record.get("struct"); + assertEquals(row, structRecord.get("intField")); + assertNull(structRecord.get("nullableIntField")); + } + if (row >= 2) { + assertNotNull(record.get("nullableStruct")); + GenericRecord nullableStructRecord = (GenericRecord) record.get("nullableStruct"); + assertEquals(row, nullableStructRecord.get("intField")); + if (row % 2 == 0) { + assertEquals(row * 10, nullableStructRecord.get("nullableIntField")); + } else { + assertNull(nullableStructRecord.get("nullableIntField")); + } + } else { + assertNull(record.get("nullableStruct")); + } + } + } + } + } +} diff --git a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroSchemaTest.java b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroSchemaTest.java new file mode 100644 index 0000000000..a05bbc1653 --- /dev/null +++ b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroSchemaTest.java @@ -0,0 +1,1392 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.Arrays; +import java.util.List; +import org.apache.arrow.vector.types.DateUnit; +import org.apache.arrow.vector.types.FloatingPointPrecision; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.UnionMode; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; +import org.junit.jupiter.api.Test; + +public class ArrowToAvroSchemaTest { + + // Schema conversion for primitive types, nullable and non-nullable + + @Test + public void testConvertNullType() { + List fields = + Arrays.asList(new Field("nullType", FieldType.notNullable(new ArrowType.Null()), null)); + + Schema schema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord"); + + assertEquals(Schema.Type.RECORD, schema.getType()); + assertEquals(1, schema.getFields().size()); + + assertEquals(Schema.Type.NULL, schema.getField("nullType").schema().getType()); + } + + @Test + public void testConvertBooleanTypes() { + List fields = + Arrays.asList( + new Field("nullableBool", FieldType.nullable(new ArrowType.Bool()), null), + new Field("nonNullableBool", FieldType.notNullable(new ArrowType.Bool()), null)); + + Schema schema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord"); + + assertEquals(Schema.Type.RECORD, schema.getType()); + assertEquals(2, schema.getFields().size()); + + assertEquals(Schema.Type.UNION, schema.getField("nullableBool").schema().getType()); + assertEquals(2, schema.getField("nullableBool").schema().getTypes().size()); + assertEquals( + Schema.Type.BOOLEAN, schema.getField("nullableBool").schema().getTypes().get(0).getType()); + assertEquals( + Schema.Type.NULL, schema.getField("nullableBool").schema().getTypes().get(1).getType()); + assertEquals(Schema.Type.BOOLEAN, schema.getField("nonNullableBool").schema().getType()); + } + + @Test + public void testConvertIntegralTypes() { + List fields = + Arrays.asList( + new Field("nullableInt8", FieldType.nullable(new ArrowType.Int(8, true)), null), + new Field("nonNullableInt8", FieldType.notNullable(new ArrowType.Int(8, true)), null), + new Field("nullableUInt8", FieldType.nullable(new ArrowType.Int(8, false)), null), + new Field("nonNullableUInt8", FieldType.notNullable(new ArrowType.Int(8, false)), null), + new Field("nullableInt16", FieldType.nullable(new ArrowType.Int(16, true)), null), + new Field("nonNullableInt16", FieldType.notNullable(new ArrowType.Int(16, true)), null), + new Field("nullableUInt16", FieldType.nullable(new ArrowType.Int(16, false)), null), + new Field( + "nonNullableUInt16", FieldType.notNullable(new ArrowType.Int(16, false)), null), + new Field("nullableInt32", FieldType.nullable(new ArrowType.Int(32, true)), null), + new Field("nonNullableInt32", FieldType.notNullable(new ArrowType.Int(32, true)), null), + new Field("nullableUInt32", FieldType.nullable(new ArrowType.Int(32, false)), null), + new Field( + "nonNullableUInt32", FieldType.notNullable(new ArrowType.Int(32, false)), null), + new Field("nullableInt64", FieldType.nullable(new ArrowType.Int(64, true)), null), + new Field("nonNullableInt64", FieldType.notNullable(new ArrowType.Int(64, true)), null), + new Field("nullableUInt64", FieldType.nullable(new ArrowType.Int(64, false)), null), + new Field( + "nonNullableUInt64", FieldType.notNullable(new ArrowType.Int(64, false)), null)); + + Schema schema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord"); + + assertEquals(Schema.Type.RECORD, schema.getType()); + assertEquals(16, schema.getFields().size()); + + assertEquals(Schema.Type.UNION, schema.getField("nullableInt8").schema().getType()); + assertEquals(2, schema.getField("nullableInt8").schema().getTypes().size()); + assertEquals( + Schema.Type.INT, schema.getField("nullableInt8").schema().getTypes().get(0).getType()); + assertEquals( + Schema.Type.NULL, schema.getField("nullableInt8").schema().getTypes().get(1).getType()); + assertEquals(Schema.Type.INT, schema.getField("nonNullableInt8").schema().getType()); + + assertEquals(Schema.Type.UNION, schema.getField("nullableUInt8").schema().getType()); + assertEquals(2, schema.getField("nullableUInt8").schema().getTypes().size()); + assertEquals( + Schema.Type.INT, schema.getField("nullableUInt8").schema().getTypes().get(0).getType()); + assertEquals( + Schema.Type.NULL, schema.getField("nullableUInt8").schema().getTypes().get(1).getType()); + assertEquals(Schema.Type.INT, schema.getField("nonNullableUInt8").schema().getType()); + + assertEquals(Schema.Type.UNION, schema.getField("nullableInt16").schema().getType()); + assertEquals(2, schema.getField("nullableInt16").schema().getTypes().size()); + assertEquals( + Schema.Type.INT, schema.getField("nullableInt16").schema().getTypes().get(0).getType()); + assertEquals( + Schema.Type.NULL, schema.getField("nullableInt16").schema().getTypes().get(1).getType()); + assertEquals(Schema.Type.INT, schema.getField("nonNullableInt16").schema().getType()); + + assertEquals(Schema.Type.UNION, schema.getField("nullableUInt16").schema().getType()); + assertEquals(2, schema.getField("nullableUInt16").schema().getTypes().size()); + assertEquals( + Schema.Type.INT, schema.getField("nullableUInt16").schema().getTypes().get(0).getType()); + assertEquals( + Schema.Type.NULL, schema.getField("nullableUInt16").schema().getTypes().get(1).getType()); + assertEquals(Schema.Type.INT, schema.getField("nonNullableUInt16").schema().getType()); + + assertEquals(Schema.Type.UNION, schema.getField("nullableInt32").schema().getType()); + assertEquals(2, schema.getField("nullableInt32").schema().getTypes().size()); + assertEquals( + Schema.Type.INT, schema.getField("nullableInt32").schema().getTypes().get(0).getType()); + assertEquals( + Schema.Type.NULL, schema.getField("nullableInt32").schema().getTypes().get(1).getType()); + assertEquals(Schema.Type.INT, schema.getField("nonNullableInt32").schema().getType()); + + assertEquals(Schema.Type.UNION, schema.getField("nullableUInt32").schema().getType()); + assertEquals(2, schema.getField("nullableUInt32").schema().getTypes().size()); + assertEquals( + Schema.Type.LONG, schema.getField("nullableUInt32").schema().getTypes().get(0).getType()); + assertEquals( + Schema.Type.NULL, schema.getField("nullableUInt32").schema().getTypes().get(1).getType()); + assertEquals(Schema.Type.LONG, schema.getField("nonNullableUInt32").schema().getType()); + + assertEquals(Schema.Type.UNION, schema.getField("nullableInt64").schema().getType()); + assertEquals(2, schema.getField("nullableInt64").schema().getTypes().size()); + assertEquals( + Schema.Type.LONG, schema.getField("nullableInt64").schema().getTypes().get(0).getType()); + assertEquals( + Schema.Type.NULL, schema.getField("nullableInt64").schema().getTypes().get(1).getType()); + assertEquals(Schema.Type.LONG, schema.getField("nonNullableInt64").schema().getType()); + + assertEquals(Schema.Type.UNION, schema.getField("nullableUInt64").schema().getType()); + assertEquals(2, schema.getField("nullableUInt64").schema().getTypes().size()); + assertEquals( + Schema.Type.LONG, schema.getField("nullableUInt64").schema().getTypes().get(0).getType()); + assertEquals( + Schema.Type.NULL, schema.getField("nullableUInt64").schema().getTypes().get(1).getType()); + assertEquals(Schema.Type.LONG, schema.getField("nonNullableUInt64").schema().getType()); + } + + @Test + public void testConvertFloatingPointTypes() { + List fields = + Arrays.asList( + new Field( + "nullableFloat16", + FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.HALF)), + null), + new Field( + "nonNullableFloat16", + FieldType.notNullable(new ArrowType.FloatingPoint(FloatingPointPrecision.HALF)), + null), + new Field( + "nullableFloat32", + FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)), + null), + new Field( + "nonNullableFloat32", + FieldType.notNullable(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)), + null), + new Field( + "nullableFloat64", + FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + null), + new Field( + "nonNullableFloat64", + FieldType.notNullable(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + null)); + + Schema schema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord"); + + assertEquals(Schema.Type.RECORD, schema.getType()); + assertEquals(6, schema.getFields().size()); + + assertEquals(Schema.Type.UNION, schema.getField("nullableFloat16").schema().getType()); + assertEquals(2, schema.getField("nullableFloat16").schema().getTypes().size()); + assertEquals( + Schema.Type.FLOAT, schema.getField("nullableFloat16").schema().getTypes().get(0).getType()); + assertEquals( + Schema.Type.NULL, schema.getField("nullableFloat16").schema().getTypes().get(1).getType()); + assertEquals(Schema.Type.FLOAT, schema.getField("nonNullableFloat16").schema().getType()); + + assertEquals(Schema.Type.UNION, schema.getField("nullableFloat32").schema().getType()); + assertEquals(2, schema.getField("nullableFloat32").schema().getTypes().size()); + assertEquals( + Schema.Type.FLOAT, schema.getField("nullableFloat32").schema().getTypes().get(0).getType()); + assertEquals( + Schema.Type.NULL, schema.getField("nullableFloat32").schema().getTypes().get(1).getType()); + assertEquals(Schema.Type.FLOAT, schema.getField("nonNullableFloat32").schema().getType()); + + assertEquals(Schema.Type.UNION, schema.getField("nullableFloat64").schema().getType()); + assertEquals(2, schema.getField("nullableFloat64").schema().getTypes().size()); + assertEquals( + Schema.Type.DOUBLE, + schema.getField("nullableFloat64").schema().getTypes().get(0).getType()); + assertEquals( + Schema.Type.NULL, schema.getField("nullableFloat64").schema().getTypes().get(1).getType()); + assertEquals(Schema.Type.DOUBLE, schema.getField("nonNullableFloat64").schema().getType()); + } + + @Test + public void testConvertStringTypes() { + List fields = + Arrays.asList( + new Field("nullableUtf8", FieldType.nullable(new ArrowType.Utf8()), null), + new Field("nonNullableUtf8", FieldType.notNullable(new ArrowType.Utf8()), null)); + + Schema schema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord"); + + assertEquals(Schema.Type.RECORD, schema.getType()); + assertEquals(2, schema.getFields().size()); + + assertEquals(Schema.Type.UNION, schema.getField("nullableUtf8").schema().getType()); + assertEquals(2, schema.getField("nullableUtf8").schema().getTypes().size()); + assertEquals( + Schema.Type.STRING, schema.getField("nullableUtf8").schema().getTypes().get(0).getType()); + assertEquals( + Schema.Type.NULL, schema.getField("nullableUtf8").schema().getTypes().get(1).getType()); + assertEquals(Schema.Type.STRING, schema.getField("nonNullableUtf8").schema().getType()); + } + + @Test + public void testConvertBinaryTypes() { + List fields = + Arrays.asList( + new Field("nullableBinary", FieldType.nullable(new ArrowType.Binary()), null), + new Field("nonNullableBinary", FieldType.notNullable(new ArrowType.Binary()), null)); + + Schema schema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord"); + + assertEquals(Schema.Type.RECORD, schema.getType()); + assertEquals(2, schema.getFields().size()); + + assertEquals(Schema.Type.UNION, schema.getField("nullableBinary").schema().getType()); + assertEquals(2, schema.getField("nullableBinary").schema().getTypes().size()); + assertEquals( + Schema.Type.BYTES, schema.getField("nullableBinary").schema().getTypes().get(0).getType()); + assertEquals( + Schema.Type.NULL, schema.getField("nullableBinary").schema().getTypes().get(1).getType()); + assertEquals(Schema.Type.BYTES, schema.getField("nonNullableBinary").schema().getType()); + } + + @Test + public void testConvertFixedSizeBinaryTypes() { + List fields = + Arrays.asList( + new Field( + "nullableFixedSizeBinary", + FieldType.nullable(new ArrowType.FixedSizeBinary(10)), + null), + new Field( + "nonNullableFixedSizeBinary", + FieldType.notNullable(new ArrowType.FixedSizeBinary(10)), + null)); + + Schema schema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord"); + + assertEquals(Schema.Type.RECORD, schema.getType()); + assertEquals(2, schema.getFields().size()); + + assertEquals(Schema.Type.UNION, schema.getField("nullableFixedSizeBinary").schema().getType()); + assertEquals(2, schema.getField("nullableFixedSizeBinary").schema().getTypes().size()); + Schema nullableFixedSizeBinarySchema = + schema.getField("nullableFixedSizeBinary").schema().getTypes().get(0); + assertEquals(Schema.Type.FIXED, nullableFixedSizeBinarySchema.getType()); + assertEquals(10, nullableFixedSizeBinarySchema.getFixedSize()); + assertEquals( + Schema.Type.NULL, + schema.getField("nullableFixedSizeBinary").schema().getTypes().get(1).getType()); + Schema nonNullableFixedSizeBinarySchema = + schema.getField("nullableFixedSizeBinary").schema().getTypes().get(0); + assertEquals(Schema.Type.FIXED, nonNullableFixedSizeBinarySchema.getType()); + assertEquals(10, nonNullableFixedSizeBinarySchema.getFixedSize()); + } + + // Schema conversion for logical types, nullable and non-nullable + + @Test + public void testConvertDecimalTypes() { + List fields = + Arrays.asList( + new Field( + "nullableDecimal128", FieldType.nullable(new ArrowType.Decimal(10, 2, 128)), null), + new Field( + "nonNullableDecimal1281", + FieldType.notNullable(new ArrowType.Decimal(10, 2, 128)), + null), + new Field( + "nonNullableDecimal1282", + FieldType.notNullable(new ArrowType.Decimal(15, 5, 128)), + null), + new Field( + "nonNullableDecimal1283", + FieldType.notNullable(new ArrowType.Decimal(20, 10, 128)), + null), + new Field( + "nullableDecimal256", FieldType.nullable(new ArrowType.Decimal(20, 4, 256)), null), + new Field( + "nonNullableDecimal2561", + FieldType.notNullable(new ArrowType.Decimal(20, 4, 256)), + null), + new Field( + "nonNullableDecimal2562", + FieldType.notNullable(new ArrowType.Decimal(25, 8, 256)), + null), + new Field( + "nonNullableDecimal2563", + FieldType.notNullable(new ArrowType.Decimal(30, 15, 256)), + null)); + + Schema schema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord"); + + assertEquals(Schema.Type.RECORD, schema.getType()); + assertEquals(8, schema.getFields().size()); + + // Assertions for nullableDecimal128 + assertEquals(Schema.Type.UNION, schema.getField("nullableDecimal128").schema().getType()); + assertEquals(2, schema.getField("nullableDecimal128").schema().getTypes().size()); + Schema nullableDecimal128Schema = + schema.getField("nullableDecimal128").schema().getTypes().get(0); + assertEquals(Schema.Type.FIXED, nullableDecimal128Schema.getType()); + assertEquals(16, nullableDecimal128Schema.getFixedSize()); + assertEquals(LogicalTypes.decimal(10, 2), nullableDecimal128Schema.getLogicalType()); + assertEquals(10, nullableDecimal128Schema.getObjectProp("precision")); + assertEquals(2, nullableDecimal128Schema.getObjectProp("scale")); + assertEquals( + Schema.Type.NULL, + schema.getField("nullableDecimal128").schema().getTypes().get(1).getType()); + + // Assertions for nonNullableDecimal1281 + Schema nonNullableDecimal1281Schema = schema.getField("nonNullableDecimal1281").schema(); + assertEquals(Schema.Type.FIXED, nonNullableDecimal1281Schema.getType()); + assertEquals(16, nonNullableDecimal1281Schema.getFixedSize()); + assertEquals(LogicalTypes.decimal(10, 2), nonNullableDecimal1281Schema.getLogicalType()); + assertEquals(10, nonNullableDecimal1281Schema.getObjectProp("precision")); + assertEquals(2, nonNullableDecimal1281Schema.getObjectProp("scale")); + + // Assertions for nonNullableDecimal1282 + Schema nonNullableDecimal1282Schema = schema.getField("nonNullableDecimal1282").schema(); + assertEquals(Schema.Type.FIXED, nonNullableDecimal1282Schema.getType()); + assertEquals(16, nonNullableDecimal1282Schema.getFixedSize()); + assertEquals(LogicalTypes.decimal(15, 5), nonNullableDecimal1282Schema.getLogicalType()); + assertEquals(15, nonNullableDecimal1282Schema.getObjectProp("precision")); + assertEquals(5, nonNullableDecimal1282Schema.getObjectProp("scale")); + + // Assertions for nonNullableDecimal1283 + Schema nonNullableDecimal1283Schema = schema.getField("nonNullableDecimal1283").schema(); + assertEquals(Schema.Type.FIXED, nonNullableDecimal1283Schema.getType()); + assertEquals(16, nonNullableDecimal1283Schema.getFixedSize()); + assertEquals(LogicalTypes.decimal(20, 10), nonNullableDecimal1283Schema.getLogicalType()); + assertEquals(20, nonNullableDecimal1283Schema.getObjectProp("precision")); + assertEquals(10, nonNullableDecimal1283Schema.getObjectProp("scale")); + + // Assertions for nullableDecimal256 + assertEquals(Schema.Type.UNION, schema.getField("nullableDecimal256").schema().getType()); + assertEquals(2, schema.getField("nullableDecimal256").schema().getTypes().size()); + Schema nullableDecimal256Schema = + schema.getField("nullableDecimal256").schema().getTypes().get(0); + assertEquals(Schema.Type.FIXED, nullableDecimal256Schema.getType()); + assertEquals(32, nullableDecimal256Schema.getFixedSize()); + assertEquals(LogicalTypes.decimal(20, 4), nullableDecimal256Schema.getLogicalType()); + assertEquals(20, nullableDecimal256Schema.getObjectProp("precision")); + assertEquals(4, nullableDecimal256Schema.getObjectProp("scale")); + assertEquals( + Schema.Type.NULL, + schema.getField("nullableDecimal256").schema().getTypes().get(1).getType()); + + // Assertions for nonNullableDecimal2561 + Schema nonNullableDecimal2561Schema = schema.getField("nonNullableDecimal2561").schema(); + assertEquals(Schema.Type.FIXED, nonNullableDecimal2561Schema.getType()); + assertEquals(32, nonNullableDecimal2561Schema.getFixedSize()); + assertEquals(LogicalTypes.decimal(20, 4), nonNullableDecimal2561Schema.getLogicalType()); + assertEquals(20, nonNullableDecimal2561Schema.getObjectProp("precision")); + assertEquals(4, nonNullableDecimal2561Schema.getObjectProp("scale")); + + // Assertions for nonNullableDecimal2562 + Schema nonNullableDecimal2562Schema = schema.getField("nonNullableDecimal2562").schema(); + assertEquals(Schema.Type.FIXED, nonNullableDecimal2562Schema.getType()); + assertEquals(32, nonNullableDecimal2562Schema.getFixedSize()); + assertEquals(LogicalTypes.decimal(25, 8), nonNullableDecimal2562Schema.getLogicalType()); + assertEquals(25, nonNullableDecimal2562Schema.getObjectProp("precision")); + assertEquals(8, nonNullableDecimal2562Schema.getObjectProp("scale")); + + // Assertions for nonNullableDecimal2563 + Schema nonNullableDecimal2563Schema = schema.getField("nonNullableDecimal2563").schema(); + assertEquals(Schema.Type.FIXED, nonNullableDecimal2563Schema.getType()); + assertEquals(32, nonNullableDecimal2563Schema.getFixedSize()); + assertEquals(LogicalTypes.decimal(30, 15), nonNullableDecimal2563Schema.getLogicalType()); + assertEquals(30, nonNullableDecimal2563Schema.getObjectProp("precision")); + assertEquals(15, nonNullableDecimal2563Schema.getObjectProp("scale")); + } + + @Test + public void testConvertDateTypes() { + List fields = + Arrays.asList( + new Field( + "nullableDateDay", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null), + new Field( + "nonNullableDateDay", + FieldType.notNullable(new ArrowType.Date(DateUnit.DAY)), + null), + new Field( + "nullableDateMilli", + FieldType.nullable(new ArrowType.Date(DateUnit.MILLISECOND)), + null), + new Field( + "nonNullableDateMilli", + FieldType.notNullable(new ArrowType.Date(DateUnit.MILLISECOND)), + null)); + + Schema schema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord"); + + assertEquals(Schema.Type.RECORD, schema.getType()); + assertEquals(4, schema.getFields().size()); + + // Assertions for nullableDateDay + assertEquals(Schema.Type.UNION, schema.getField("nullableDateDay").schema().getType()); + assertEquals(2, schema.getField("nullableDateDay").schema().getTypes().size()); + Schema nullableDateDaySchema = schema.getField("nullableDateDay").schema().getTypes().get(0); + assertEquals(Schema.Type.INT, nullableDateDaySchema.getType()); + assertEquals(LogicalTypes.date(), nullableDateDaySchema.getLogicalType()); + assertEquals( + Schema.Type.NULL, schema.getField("nullableDateDay").schema().getTypes().get(1).getType()); + + // Assertions for nonNullableDateDay + Schema nonNullableDateDaySchema = schema.getField("nonNullableDateDay").schema(); + assertEquals(Schema.Type.INT, nonNullableDateDaySchema.getType()); + assertEquals(LogicalTypes.date(), nonNullableDateDaySchema.getLogicalType()); + + // Assertions for nullableDateMilli + assertEquals(Schema.Type.UNION, schema.getField("nullableDateMilli").schema().getType()); + assertEquals(2, schema.getField("nullableDateMilli").schema().getTypes().size()); + Schema nullableDateMilliSchema = + schema.getField("nullableDateMilli").schema().getTypes().get(0); + assertEquals(Schema.Type.INT, nullableDateMilliSchema.getType()); + assertEquals(LogicalTypes.date(), nullableDateMilliSchema.getLogicalType()); + assertEquals( + Schema.Type.NULL, + schema.getField("nullableDateMilli").schema().getTypes().get(1).getType()); + + // Assertions for nonNullableDateMilli + Schema nonNullableDateMilliSchema = schema.getField("nonNullableDateMilli").schema(); + assertEquals(Schema.Type.INT, nonNullableDateMilliSchema.getType()); + assertEquals(LogicalTypes.date(), nonNullableDateMilliSchema.getLogicalType()); + } + + @Test + public void testConvertTimeTypes() { + List fields = + Arrays.asList( + new Field( + "nullableTimeSec", + FieldType.nullable(new ArrowType.Time(TimeUnit.SECOND, 32)), + null), + new Field( + "nonNullableTimeSec", + FieldType.notNullable(new ArrowType.Time(TimeUnit.SECOND, 32)), + null), + new Field( + "nullableTimeMillis", + FieldType.nullable(new ArrowType.Time(TimeUnit.MILLISECOND, 32)), + null), + new Field( + "nonNullableTimeMillis", + FieldType.notNullable(new ArrowType.Time(TimeUnit.MILLISECOND, 32)), + null), + new Field( + "nullableTimeMicros", + FieldType.nullable(new ArrowType.Time(TimeUnit.MICROSECOND, 64)), + null), + new Field( + "nonNullableTimeMicros", + FieldType.notNullable(new ArrowType.Time(TimeUnit.MICROSECOND, 64)), + null), + new Field( + "nullableTimeNanos", + FieldType.nullable(new ArrowType.Time(TimeUnit.NANOSECOND, 64)), + null), + new Field( + "nonNullableTimeNanos", + FieldType.notNullable(new ArrowType.Time(TimeUnit.NANOSECOND, 64)), + null)); + + Schema schema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord"); + + assertEquals(Schema.Type.RECORD, schema.getType()); + assertEquals(8, schema.getFields().size()); + + // Assertions for nullableTimeSec + assertEquals(Schema.Type.UNION, schema.getField("nullableTimeSec").schema().getType()); + assertEquals(2, schema.getField("nullableTimeSec").schema().getTypes().size()); + Schema nullableTimeSecSchema = schema.getField("nullableTimeSec").schema().getTypes().get(0); + assertEquals(Schema.Type.INT, nullableTimeSecSchema.getType()); + assertEquals(LogicalTypes.timeMillis(), nullableTimeSecSchema.getLogicalType()); + assertEquals( + Schema.Type.NULL, schema.getField("nullableTimeSec").schema().getTypes().get(1).getType()); + + // Assertions for nonNullableTimeSec + Schema nonNullableTimeSecSchema = schema.getField("nonNullableTimeSec").schema(); + assertEquals(Schema.Type.INT, nonNullableTimeSecSchema.getType()); + assertEquals(LogicalTypes.timeMillis(), nonNullableTimeSecSchema.getLogicalType()); + + // Assertions for nullableTimeMillis + assertEquals(Schema.Type.UNION, schema.getField("nullableTimeMillis").schema().getType()); + assertEquals(2, schema.getField("nullableTimeMillis").schema().getTypes().size()); + Schema nullableTimeMillisSchema = + schema.getField("nullableTimeMillis").schema().getTypes().get(0); + assertEquals(Schema.Type.INT, nullableTimeMillisSchema.getType()); + assertEquals(LogicalTypes.timeMillis(), nullableTimeMillisSchema.getLogicalType()); + assertEquals( + Schema.Type.NULL, + schema.getField("nullableTimeMillis").schema().getTypes().get(1).getType()); + + // Assertions for nonNullableTimeMillis + Schema nonNullableTimeMillisSchema = schema.getField("nonNullableTimeMillis").schema(); + assertEquals(Schema.Type.INT, nonNullableTimeMillisSchema.getType()); + assertEquals(LogicalTypes.timeMillis(), nonNullableTimeMillisSchema.getLogicalType()); + + // Assertions for nullableTimeMicros + assertEquals(Schema.Type.UNION, schema.getField("nullableTimeMicros").schema().getType()); + assertEquals(2, schema.getField("nullableTimeMicros").schema().getTypes().size()); + Schema nullableTimeMicrosSchema = + schema.getField("nullableTimeMicros").schema().getTypes().get(0); + assertEquals(Schema.Type.LONG, nullableTimeMicrosSchema.getType()); + assertEquals(LogicalTypes.timeMicros(), nullableTimeMicrosSchema.getLogicalType()); + assertEquals( + Schema.Type.NULL, + schema.getField("nullableTimeMicros").schema().getTypes().get(1).getType()); + + // Assertions for nonNullableTimeMicros + Schema nonNullableTimeMicrosSchema = schema.getField("nonNullableTimeMicros").schema(); + assertEquals(Schema.Type.LONG, nonNullableTimeMicrosSchema.getType()); + assertEquals(LogicalTypes.timeMicros(), nonNullableTimeMicrosSchema.getLogicalType()); + + // Assertions for nullableTimeNanos + assertEquals(Schema.Type.UNION, schema.getField("nullableTimeNanos").schema().getType()); + assertEquals(2, schema.getField("nullableTimeNanos").schema().getTypes().size()); + Schema nullableTimeNanosSchema = + schema.getField("nullableTimeNanos").schema().getTypes().get(0); + assertEquals(Schema.Type.LONG, nullableTimeNanosSchema.getType()); + assertEquals(LogicalTypes.timeMicros(), nullableTimeNanosSchema.getLogicalType()); + assertEquals( + Schema.Type.NULL, + schema.getField("nullableTimeNanos").schema().getTypes().get(1).getType()); + + // Assertions for nonNullableTimeNanos + Schema nonNullableTimeNanosSchema = schema.getField("nonNullableTimeNanos").schema(); + assertEquals(Schema.Type.LONG, nonNullableTimeNanosSchema.getType()); + assertEquals(LogicalTypes.timeMicros(), nonNullableTimeNanosSchema.getLogicalType()); + } + + @Test + public void testConvertZoneAwareTimestampTypes() { + List fields = + Arrays.asList( + new Field( + "nullableTimestampSecTz", + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.SECOND, "UTC")), + null), + new Field( + "nonNullableTimestampSecTz", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.SECOND, "UTC")), + null), + new Field( + "nullableTimestampMillisTz", + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC")), + null), + new Field( + "nonNullableTimestampMillisTz", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC")), + null), + new Field( + "nullableTimestampMicrosTz", + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC")), + null), + new Field( + "nonNullableTimestampMicrosTz", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC")), + null), + new Field( + "nullableTimestampNanosTz", + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.NANOSECOND, "UTC")), + null), + new Field( + "nonNullableTimestampNanosTz", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.NANOSECOND, "UTC")), + null)); + + Schema schema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord"); + + assertEquals(Schema.Type.RECORD, schema.getType()); + assertEquals(8, schema.getFields().size()); + + // Assertions for nullableTimestampSecTz + assertEquals(Schema.Type.UNION, schema.getField("nullableTimestampSecTz").schema().getType()); + assertEquals(2, schema.getField("nullableTimestampSecTz").schema().getTypes().size()); + Schema nullableTimestampSecTzSchema = + schema.getField("nullableTimestampSecTz").schema().getTypes().get(0); + assertEquals(Schema.Type.LONG, nullableTimestampSecTzSchema.getType()); + assertEquals(LogicalTypes.timestampMillis(), nullableTimestampSecTzSchema.getLogicalType()); + assertEquals( + Schema.Type.NULL, + schema.getField("nullableTimestampSecTz").schema().getTypes().get(1).getType()); + + // Assertions for nonNullableTimestampSecTz + Schema nonNullableTimestampSecTzSchema = schema.getField("nonNullableTimestampSecTz").schema(); + assertEquals(Schema.Type.LONG, nonNullableTimestampSecTzSchema.getType()); + assertEquals(LogicalTypes.timestampMillis(), nonNullableTimestampSecTzSchema.getLogicalType()); + + // Assertions for nullableTimestampMillisTz + assertEquals( + Schema.Type.UNION, schema.getField("nullableTimestampMillisTz").schema().getType()); + assertEquals(2, schema.getField("nullableTimestampMillisTz").schema().getTypes().size()); + Schema nullableTimestampMillisTzSchema = + schema.getField("nullableTimestampMillisTz").schema().getTypes().get(0); + assertEquals(Schema.Type.LONG, nullableTimestampMillisTzSchema.getType()); + assertEquals(LogicalTypes.timestampMillis(), nullableTimestampMillisTzSchema.getLogicalType()); + assertEquals( + Schema.Type.NULL, + schema.getField("nullableTimestampMillisTz").schema().getTypes().get(1).getType()); + + // Assertions for nonNullableTimestampMillisTz + Schema nonNullableTimestampMillisTzSchema = + schema.getField("nonNullableTimestampMillisTz").schema(); + assertEquals(Schema.Type.LONG, nonNullableTimestampMillisTzSchema.getType()); + assertEquals( + LogicalTypes.timestampMillis(), nonNullableTimestampMillisTzSchema.getLogicalType()); + + // Assertions for nullableTimestampMicrosTz + assertEquals( + Schema.Type.UNION, schema.getField("nullableTimestampMicrosTz").schema().getType()); + assertEquals(2, schema.getField("nullableTimestampMicrosTz").schema().getTypes().size()); + Schema nullableTimestampMicrosTzSchema = + schema.getField("nullableTimestampMicrosTz").schema().getTypes().get(0); + assertEquals(Schema.Type.LONG, nullableTimestampMicrosTzSchema.getType()); + assertEquals(LogicalTypes.timestampMicros(), nullableTimestampMicrosTzSchema.getLogicalType()); + assertEquals( + Schema.Type.NULL, + schema.getField("nullableTimestampMicrosTz").schema().getTypes().get(1).getType()); + + // Assertions for nonNullableTimestampMicrosTz + Schema nonNullableTimestampMicrosTzSchema = + schema.getField("nonNullableTimestampMicrosTz").schema(); + assertEquals(Schema.Type.LONG, nonNullableTimestampMicrosTzSchema.getType()); + assertEquals( + LogicalTypes.timestampMicros(), nonNullableTimestampMicrosTzSchema.getLogicalType()); + + // Assertions for nullableTimestampNanosTz + assertEquals(Schema.Type.UNION, schema.getField("nullableTimestampNanosTz").schema().getType()); + assertEquals(2, schema.getField("nullableTimestampNanosTz").schema().getTypes().size()); + Schema nullableTimestampNanosTzSchema = + schema.getField("nullableTimestampNanosTz").schema().getTypes().get(0); + assertEquals(Schema.Type.LONG, nullableTimestampNanosTzSchema.getType()); + assertEquals(LogicalTypes.timestampNanos(), nullableTimestampNanosTzSchema.getLogicalType()); + assertEquals( + Schema.Type.NULL, + schema.getField("nullableTimestampNanosTz").schema().getTypes().get(1).getType()); + + // Assertions for nonNullableTimestampNanosTz + Schema nonNullableTimestampNanosTzSchema = + schema.getField("nonNullableTimestampNanosTz").schema(); + assertEquals(Schema.Type.LONG, nonNullableTimestampNanosTzSchema.getType()); + assertEquals(LogicalTypes.timestampNanos(), nonNullableTimestampNanosTzSchema.getLogicalType()); + } + + @Test + public void testConvertLocalTimestampTypes() { + List fields = + Arrays.asList( + new Field( + "nullableTimestampSec", + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.SECOND, null)), + null), + new Field( + "nonNullableTimestampSec", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.SECOND, null)), + null), + new Field( + "nullableTimestampMillis", + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, null)), + null), + new Field( + "nonNullableTimestampMillis", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, null)), + null), + new Field( + "nullableTimestampMicros", + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MICROSECOND, null)), + null), + new Field( + "nonNullableTimestampMicros", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.MICROSECOND, null)), + null), + new Field( + "nullableTimestampNanos", + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.NANOSECOND, null)), + null), + new Field( + "nonNullableTimestampNanos", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.NANOSECOND, null)), + null)); + + Schema schema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord"); + + assertEquals(Schema.Type.RECORD, schema.getType()); + assertEquals(8, schema.getFields().size()); + + // Assertions for nullableTimestampSec + assertEquals(Schema.Type.UNION, schema.getField("nullableTimestampSec").schema().getType()); + assertEquals(2, schema.getField("nullableTimestampSec").schema().getTypes().size()); + Schema nullableTimestampSecSchema = + schema.getField("nullableTimestampSec").schema().getTypes().get(0); + assertEquals(Schema.Type.LONG, nullableTimestampSecSchema.getType()); + assertEquals(LogicalTypes.localTimestampMillis(), nullableTimestampSecSchema.getLogicalType()); + assertEquals( + Schema.Type.NULL, + schema.getField("nullableTimestampSec").schema().getTypes().get(1).getType()); + + // Assertions for nonNullableTimestampSec + Schema nonNullableTimestampSecSchema = schema.getField("nonNullableTimestampSec").schema(); + assertEquals(Schema.Type.LONG, nonNullableTimestampSecSchema.getType()); + assertEquals( + LogicalTypes.localTimestampMillis(), nonNullableTimestampSecSchema.getLogicalType()); + + // Assertions for nullableTimestampMillis + assertEquals(Schema.Type.UNION, schema.getField("nullableTimestampMillis").schema().getType()); + assertEquals(2, schema.getField("nullableTimestampMillis").schema().getTypes().size()); + Schema nullableTimestampMillisSchema = + schema.getField("nullableTimestampMillis").schema().getTypes().get(0); + assertEquals(Schema.Type.LONG, nullableTimestampMillisSchema.getType()); + assertEquals( + LogicalTypes.localTimestampMillis(), nullableTimestampMillisSchema.getLogicalType()); + assertEquals( + Schema.Type.NULL, + schema.getField("nullableTimestampMillis").schema().getTypes().get(1).getType()); + + // Assertions for nonNullableTimestampMillis + Schema nonNullableTimestampMillisSchema = + schema.getField("nonNullableTimestampMillis").schema(); + assertEquals(Schema.Type.LONG, nonNullableTimestampMillisSchema.getType()); + assertEquals( + LogicalTypes.localTimestampMillis(), nonNullableTimestampMillisSchema.getLogicalType()); + + // Assertions for nullableTimestampMicros + assertEquals(Schema.Type.UNION, schema.getField("nullableTimestampMicros").schema().getType()); + assertEquals(2, schema.getField("nullableTimestampMicros").schema().getTypes().size()); + Schema nullableTimestampMicrosSchema = + schema.getField("nullableTimestampMicros").schema().getTypes().get(0); + assertEquals(Schema.Type.LONG, nullableTimestampMicrosSchema.getType()); + assertEquals( + LogicalTypes.localTimestampMicros(), nullableTimestampMicrosSchema.getLogicalType()); + assertEquals( + Schema.Type.NULL, + schema.getField("nullableTimestampMicros").schema().getTypes().get(1).getType()); + + // Assertions for nonNullableTimestampMicros + Schema nonNullableTimestampMicrosSchema = + schema.getField("nonNullableTimestampMicros").schema(); + assertEquals(Schema.Type.LONG, nonNullableTimestampMicrosSchema.getType()); + assertEquals( + LogicalTypes.localTimestampMicros(), nonNullableTimestampMicrosSchema.getLogicalType()); + + // Assertions for nullableTimestampNanos + assertEquals(Schema.Type.UNION, schema.getField("nullableTimestampNanos").schema().getType()); + assertEquals(2, schema.getField("nullableTimestampNanos").schema().getTypes().size()); + Schema nullableTimestampNanosSchema = + schema.getField("nullableTimestampNanos").schema().getTypes().get(0); + assertEquals(Schema.Type.LONG, nullableTimestampNanosSchema.getType()); + assertEquals(LogicalTypes.localTimestampNanos(), nullableTimestampNanosSchema.getLogicalType()); + assertEquals( + Schema.Type.NULL, + schema.getField("nullableTimestampNanos").schema().getTypes().get(1).getType()); + + // Assertions for nonNullableTimestampNanos + Schema nonNullableTimestampNanosSchema = schema.getField("nonNullableTimestampNanos").schema(); + assertEquals(Schema.Type.LONG, nonNullableTimestampNanosSchema.getType()); + assertEquals( + LogicalTypes.localTimestampNanos(), nonNullableTimestampNanosSchema.getLogicalType()); + } + + // Schema conversion for complex types, where the contents are primitive and logical types + + @Test + public void testConvertListTypes() { + List fields = + Arrays.asList( + new Field( + "nullableIntList", + FieldType.nullable(new ArrowType.List()), + Arrays.asList( + new Field("item", FieldType.nullable(new ArrowType.Int(32, true)), null))), + new Field( + "nullableDoubleList", + FieldType.nullable(new ArrowType.List()), + Arrays.asList( + new Field( + "item", + FieldType.notNullable( + new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + null))), + new Field( + "nonNullableDecimalList", + FieldType.notNullable(new ArrowType.List()), + Arrays.asList( + new Field( + "item", FieldType.nullable(new ArrowType.Decimal(10, 2, 128)), null))), + new Field( + "nonNullableTimestampList", + FieldType.notNullable(new ArrowType.List()), + Arrays.asList( + new Field( + "item", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC")), + null)))); + + Schema schema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord"); + + assertEquals(Schema.Type.RECORD, schema.getType()); + assertEquals(4, schema.getFields().size()); + + // Assertions for nullableIntList + assertEquals(Schema.Type.UNION, schema.getField("nullableIntList").schema().getType()); + assertEquals(2, schema.getField("nullableIntList").schema().getTypes().size()); + assertEquals( + Schema.Type.ARRAY, schema.getField("nullableIntList").schema().getTypes().get(0).getType()); + assertEquals( + Schema.Type.NULL, schema.getField("nullableIntList").schema().getTypes().get(1).getType()); + Schema nullableIntListItemSchema = + schema.getField("nullableIntList").schema().getTypes().get(0).getElementType(); + assertEquals(Schema.Type.UNION, nullableIntListItemSchema.getType()); + assertEquals(2, nullableIntListItemSchema.getTypes().size()); + assertEquals(Schema.Type.INT, nullableIntListItemSchema.getTypes().get(0).getType()); + assertEquals(Schema.Type.NULL, nullableIntListItemSchema.getTypes().get(1).getType()); + + // Assertions for nullableDoubleList + assertEquals(Schema.Type.UNION, schema.getField("nullableDoubleList").schema().getType()); + assertEquals(2, schema.getField("nullableDoubleList").schema().getTypes().size()); + assertEquals( + Schema.Type.ARRAY, + schema.getField("nullableDoubleList").schema().getTypes().get(0).getType()); + assertEquals( + Schema.Type.NULL, + schema.getField("nullableDoubleList").schema().getTypes().get(1).getType()); + Schema nullableDoubleListItemSchema = + schema.getField("nullableDoubleList").schema().getTypes().get(0).getElementType(); + assertEquals(Schema.Type.DOUBLE, nullableDoubleListItemSchema.getType()); + + // Assertions for nonNullableDecimalList + assertEquals(Schema.Type.ARRAY, schema.getField("nonNullableDecimalList").schema().getType()); + Schema nonNullableDecimalListItemSchema = + schema.getField("nonNullableDecimalList").schema().getElementType(); + assertEquals(Schema.Type.UNION, nonNullableDecimalListItemSchema.getType()); + assertEquals(2, nonNullableDecimalListItemSchema.getTypes().size()); + Schema nullableDecimalSchema = nonNullableDecimalListItemSchema.getTypes().get(0); + assertEquals(Schema.Type.FIXED, nullableDecimalSchema.getType()); + assertEquals(16, nullableDecimalSchema.getFixedSize()); + assertEquals(LogicalTypes.decimal(10, 2), nullableDecimalSchema.getLogicalType()); + assertEquals(10, nullableDecimalSchema.getObjectProp("precision")); + assertEquals(2, nullableDecimalSchema.getObjectProp("scale")); + assertEquals(Schema.Type.NULL, nonNullableDecimalListItemSchema.getTypes().get(1).getType()); + + // Assertions for nonNullableTimestampList + assertEquals(Schema.Type.ARRAY, schema.getField("nonNullableTimestampList").schema().getType()); + Schema nonNullableTimestampListItemSchema = + schema.getField("nonNullableTimestampList").schema().getElementType(); + assertEquals(Schema.Type.LONG, nonNullableTimestampListItemSchema.getType()); + assertEquals( + LogicalTypes.timestampMillis(), nonNullableTimestampListItemSchema.getLogicalType()); + } + + @Test + public void testConvertFixedSizeListTypes() { + List fields = + Arrays.asList( + new Field( + "nullableFixedSizeIntList", + FieldType.nullable(new ArrowType.FixedSizeList(3)), + Arrays.asList( + new Field("item", FieldType.nullable(new ArrowType.Int(32, true)), null))), + new Field( + "nullableFixedSizeDoubleList", + FieldType.nullable(new ArrowType.FixedSizeList(3)), + Arrays.asList( + new Field( + "item", + FieldType.notNullable( + new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + null))), + new Field( + "nonNullableFixedSizeDecimalList", + FieldType.notNullable(new ArrowType.FixedSizeList(3)), + Arrays.asList( + new Field( + "item", FieldType.nullable(new ArrowType.Decimal(10, 2, 128)), null))), + new Field( + "nonNullableFixedSizeTimestampList", + FieldType.notNullable(new ArrowType.FixedSizeList(3)), + Arrays.asList( + new Field( + "item", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC")), + null)))); + + Schema schema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord"); + + assertEquals(Schema.Type.RECORD, schema.getType()); + assertEquals(4, schema.getFields().size()); + + // Assertions for nullableFixedSizeIntList + assertEquals(Schema.Type.UNION, schema.getField("nullableFixedSizeIntList").schema().getType()); + assertEquals(2, schema.getField("nullableFixedSizeIntList").schema().getTypes().size()); + assertEquals( + Schema.Type.ARRAY, + schema.getField("nullableFixedSizeIntList").schema().getTypes().get(0).getType()); + assertEquals( + Schema.Type.NULL, + schema.getField("nullableFixedSizeIntList").schema().getTypes().get(1).getType()); + Schema nullableFixedSizeIntListItemSchema = + schema.getField("nullableFixedSizeIntList").schema().getTypes().get(0).getElementType(); + assertEquals(Schema.Type.UNION, nullableFixedSizeIntListItemSchema.getType()); + assertEquals(2, nullableFixedSizeIntListItemSchema.getTypes().size()); + assertEquals(Schema.Type.INT, nullableFixedSizeIntListItemSchema.getTypes().get(0).getType()); + assertEquals(Schema.Type.NULL, nullableFixedSizeIntListItemSchema.getTypes().get(1).getType()); + + // Assertions for nullableFixedSizeDoubleList + assertEquals( + Schema.Type.UNION, schema.getField("nullableFixedSizeDoubleList").schema().getType()); + assertEquals(2, schema.getField("nullableFixedSizeDoubleList").schema().getTypes().size()); + assertEquals( + Schema.Type.ARRAY, + schema.getField("nullableFixedSizeDoubleList").schema().getTypes().get(0).getType()); + assertEquals( + Schema.Type.NULL, + schema.getField("nullableFixedSizeDoubleList").schema().getTypes().get(1).getType()); + Schema nullableFixedSizeDoubleListItemSchema = + schema.getField("nullableFixedSizeDoubleList").schema().getTypes().get(0).getElementType(); + assertEquals(Schema.Type.DOUBLE, nullableFixedSizeDoubleListItemSchema.getType()); + + // Assertions for nonNullableFixedSizeDecimalList + assertEquals( + Schema.Type.ARRAY, schema.getField("nonNullableFixedSizeDecimalList").schema().getType()); + Schema nonNullableFixedSizeDecimalListItemSchema = + schema.getField("nonNullableFixedSizeDecimalList").schema().getElementType(); + assertEquals(Schema.Type.UNION, nonNullableFixedSizeDecimalListItemSchema.getType()); + assertEquals(2, nonNullableFixedSizeDecimalListItemSchema.getTypes().size()); + Schema nullableDecimalSchema = nonNullableFixedSizeDecimalListItemSchema.getTypes().get(0); + assertEquals(Schema.Type.FIXED, nullableDecimalSchema.getType()); + assertEquals(16, nullableDecimalSchema.getFixedSize()); + assertEquals(LogicalTypes.decimal(10, 2), nullableDecimalSchema.getLogicalType()); + assertEquals(10, nullableDecimalSchema.getObjectProp("precision")); + assertEquals(2, nullableDecimalSchema.getObjectProp("scale")); + assertEquals( + Schema.Type.NULL, nonNullableFixedSizeDecimalListItemSchema.getTypes().get(1).getType()); + + // Assertions for nonNullableFixedSizeTimestampList + assertEquals( + Schema.Type.ARRAY, schema.getField("nonNullableFixedSizeTimestampList").schema().getType()); + Schema nonNullableFixedSizeTimestampListItemSchema = + schema.getField("nonNullableFixedSizeTimestampList").schema().getElementType(); + assertEquals(Schema.Type.LONG, nonNullableFixedSizeTimestampListItemSchema.getType()); + assertEquals( + LogicalTypes.timestampMillis(), + nonNullableFixedSizeTimestampListItemSchema.getLogicalType()); + } + + @Test + public void testConvertMapTypes() { + List fields = + Arrays.asList( + new Field( + "nullableMapWithNullableInt", + FieldType.nullable(new ArrowType.Map(false)), + Arrays.asList( + new Field( + "entries", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList( + new Field("key", FieldType.notNullable(new ArrowType.Utf8()), null), + new Field( + "value", FieldType.nullable(new ArrowType.Int(32, true)), null))))), + new Field( + "nullableMapWithNonNullableDouble", + FieldType.nullable(new ArrowType.Map(false)), + Arrays.asList( + new Field( + "entries", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList( + new Field("key", FieldType.notNullable(new ArrowType.Utf8()), null), + new Field( + "value", + FieldType.notNullable( + new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + null))))), + new Field( + "nonNullableMapWithNullableDecimal", + FieldType.notNullable(new ArrowType.Map(false)), + Arrays.asList( + new Field( + "entries", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList( + new Field("key", FieldType.notNullable(new ArrowType.Utf8()), null), + new Field( + "value", + FieldType.nullable(new ArrowType.Decimal(10, 2, 128)), + null))))), + new Field( + "nonNullableMapWithNonNullableTimestamp", + FieldType.notNullable(new ArrowType.Map(false)), + Arrays.asList( + new Field( + "entries", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList( + new Field("key", FieldType.notNullable(new ArrowType.Utf8()), null), + new Field( + "value", + FieldType.notNullable( + new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC")), + null)))))); + + Schema schema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord"); + + assertEquals(Schema.Type.RECORD, schema.getType()); + assertEquals(4, schema.getFields().size()); + + // Assertions for nullableMapWithNullableInt + assertEquals( + Schema.Type.UNION, schema.getField("nullableMapWithNullableInt").schema().getType()); + assertEquals(2, schema.getField("nullableMapWithNullableInt").schema().getTypes().size()); + assertEquals( + Schema.Type.MAP, + schema.getField("nullableMapWithNullableInt").schema().getTypes().get(0).getType()); + assertEquals( + Schema.Type.NULL, + schema.getField("nullableMapWithNullableInt").schema().getTypes().get(1).getType()); + Schema nullableMapWithNullableIntValueSchema = + schema.getField("nullableMapWithNullableInt").schema().getTypes().get(0).getValueType(); + assertEquals(Schema.Type.UNION, nullableMapWithNullableIntValueSchema.getType()); + assertEquals(2, nullableMapWithNullableIntValueSchema.getTypes().size()); + assertEquals( + Schema.Type.INT, nullableMapWithNullableIntValueSchema.getTypes().get(0).getType()); + assertEquals( + Schema.Type.NULL, nullableMapWithNullableIntValueSchema.getTypes().get(1).getType()); + + // Assertions for nullableMapWithNonNullableDouble + assertEquals( + Schema.Type.UNION, schema.getField("nullableMapWithNonNullableDouble").schema().getType()); + assertEquals(2, schema.getField("nullableMapWithNonNullableDouble").schema().getTypes().size()); + assertEquals( + Schema.Type.MAP, + schema.getField("nullableMapWithNonNullableDouble").schema().getTypes().get(0).getType()); + assertEquals( + Schema.Type.NULL, + schema.getField("nullableMapWithNonNullableDouble").schema().getTypes().get(1).getType()); + Schema nullableMapWithNonNullableDoubleValueSchema = + schema + .getField("nullableMapWithNonNullableDouble") + .schema() + .getTypes() + .get(0) + .getValueType(); + assertEquals(Schema.Type.DOUBLE, nullableMapWithNonNullableDoubleValueSchema.getType()); + + // Assertions for nonNullableMapWithNullableDecimal + assertEquals( + Schema.Type.MAP, schema.getField("nonNullableMapWithNullableDecimal").schema().getType()); + Schema nonNullableMapWithNullableDecimalValueSchema = + schema.getField("nonNullableMapWithNullableDecimal").schema().getValueType(); + assertEquals(Schema.Type.UNION, nonNullableMapWithNullableDecimalValueSchema.getType()); + assertEquals(2, nonNullableMapWithNullableDecimalValueSchema.getTypes().size()); + Schema nullableDecimalSchema = nonNullableMapWithNullableDecimalValueSchema.getTypes().get(0); + assertEquals(Schema.Type.FIXED, nullableDecimalSchema.getType()); + assertEquals(16, nullableDecimalSchema.getFixedSize()); + assertEquals(LogicalTypes.decimal(10, 2), nullableDecimalSchema.getLogicalType()); + assertEquals(10, nullableDecimalSchema.getObjectProp("precision")); + assertEquals(2, nullableDecimalSchema.getObjectProp("scale")); + assertEquals( + Schema.Type.NULL, nonNullableMapWithNullableDecimalValueSchema.getTypes().get(1).getType()); + + // Assertions for nonNullableMapWithNonNullableTimestamp + assertEquals( + Schema.Type.MAP, + schema.getField("nonNullableMapWithNonNullableTimestamp").schema().getType()); + Schema nonNullableMapWithNonNullableTimestampValueSchema = + schema.getField("nonNullableMapWithNonNullableTimestamp").schema().getValueType(); + assertEquals(Schema.Type.LONG, nonNullableMapWithNonNullableTimestampValueSchema.getType()); + assertEquals( + LogicalTypes.timestampMillis(), + nonNullableMapWithNonNullableTimestampValueSchema.getLogicalType()); + } + + @Test + public void testConvertRecordTypes() { + List fields = + Arrays.asList( + new Field( + "nullableRecord", + FieldType.nullable(new ArrowType.Struct()), + Arrays.asList( + new Field("field1", FieldType.nullable(new ArrowType.Int(32, true)), null), + new Field( + "field2", + FieldType.notNullable( + new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + null), + new Field( + "field3", FieldType.nullable(new ArrowType.Decimal(10, 2, 128)), null), + new Field( + "field4", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC")), + null))), + new Field( + "nonNullableRecord", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList( + new Field("field1", FieldType.nullable(new ArrowType.Int(32, true)), null), + new Field( + "field2", + FieldType.notNullable( + new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + null), + new Field( + "field3", FieldType.nullable(new ArrowType.Decimal(10, 2, 128)), null), + new Field( + "field4", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC")), + null)))); + + Schema schema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord"); + + assertEquals(Schema.Type.RECORD, schema.getType()); + assertEquals(2, schema.getFields().size()); + + // Assertions for nullableRecord + assertEquals(Schema.Type.UNION, schema.getField("nullableRecord").schema().getType()); + assertEquals(2, schema.getField("nullableRecord").schema().getTypes().size()); + assertEquals( + Schema.Type.RECORD, schema.getField("nullableRecord").schema().getTypes().get(0).getType()); + assertEquals( + Schema.Type.NULL, schema.getField("nullableRecord").schema().getTypes().get(1).getType()); + Schema nullableRecordSchema = schema.getField("nullableRecord").schema().getTypes().get(0); + assertEquals(4, nullableRecordSchema.getFields().size()); + assertEquals( + Schema.Type.INT, + nullableRecordSchema.getField("field1").schema().getTypes().get(0).getType()); + assertEquals( + Schema.Type.NULL, + nullableRecordSchema.getField("field1").schema().getTypes().get(1).getType()); + assertEquals(Schema.Type.DOUBLE, nullableRecordSchema.getField("field2").schema().getType()); + assertEquals( + Schema.Type.FIXED, + nullableRecordSchema.getField("field3").schema().getTypes().get(0).getType()); + assertEquals( + 16, nullableRecordSchema.getField("field3").schema().getTypes().get(0).getFixedSize()); + assertEquals( + LogicalTypes.decimal(10, 2), + nullableRecordSchema.getField("field3").schema().getTypes().get(0).getLogicalType()); + assertEquals( + 10, + nullableRecordSchema + .getField("field3") + .schema() + .getTypes() + .get(0) + .getObjectProp("precision")); + assertEquals( + 2, + nullableRecordSchema.getField("field3").schema().getTypes().get(0).getObjectProp("scale")); + assertEquals( + Schema.Type.NULL, + nullableRecordSchema.getField("field3").schema().getTypes().get(1).getType()); + assertEquals(Schema.Type.LONG, nullableRecordSchema.getField("field4").schema().getType()); + assertEquals( + LogicalTypes.timestampMillis(), + nullableRecordSchema.getField("field4").schema().getLogicalType()); + + // Assertions for nonNullableRecord + assertEquals(Schema.Type.RECORD, schema.getField("nonNullableRecord").schema().getType()); + Schema nonNullableRecordSchema = schema.getField("nonNullableRecord").schema(); + assertEquals(4, nonNullableRecordSchema.getFields().size()); + assertEquals( + Schema.Type.INT, + nonNullableRecordSchema.getField("field1").schema().getTypes().get(0).getType()); + assertEquals( + Schema.Type.NULL, + nonNullableRecordSchema.getField("field1").schema().getTypes().get(1).getType()); + assertEquals(Schema.Type.DOUBLE, nonNullableRecordSchema.getField("field2").schema().getType()); + assertEquals( + Schema.Type.FIXED, + nonNullableRecordSchema.getField("field3").schema().getTypes().get(0).getType()); + assertEquals( + 16, nullableRecordSchema.getField("field3").schema().getTypes().get(0).getFixedSize()); + assertEquals( + LogicalTypes.decimal(10, 2), + nonNullableRecordSchema.getField("field3").schema().getTypes().get(0).getLogicalType()); + assertEquals( + 10, + nonNullableRecordSchema + .getField("field3") + .schema() + .getTypes() + .get(0) + .getObjectProp("precision")); + assertEquals( + 2, + nonNullableRecordSchema + .getField("field3") + .schema() + .getTypes() + .get(0) + .getObjectProp("scale")); + assertEquals( + Schema.Type.NULL, + nonNullableRecordSchema.getField("field3").schema().getTypes().get(1).getType()); + assertEquals(Schema.Type.LONG, nonNullableRecordSchema.getField("field4").schema().getType()); + assertEquals( + LogicalTypes.timestampMillis(), + nonNullableRecordSchema.getField("field4").schema().getLogicalType()); + } + + @Test + public void testConvertUnionTypes() { + List fields = + Arrays.asList( + new Field( + "sparseUnionField", + FieldType.nullable( + new ArrowType.Union( + UnionMode.Sparse, + new int[] { + ArrowType.ArrowTypeID.Int.getFlatbufID(), + ArrowType.ArrowTypeID.FloatingPoint.getFlatbufID(), + ArrowType.ArrowTypeID.Utf8.getFlatbufID() + })), + Arrays.asList( + new Field( + "intMember", FieldType.notNullable(new ArrowType.Int(32, true)), null), + new Field( + "floatMember", + FieldType.notNullable( + new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)), + null), + new Field("stringMember", FieldType.notNullable(new ArrowType.Utf8()), null))), + new Field( + "denseUnionField", + FieldType.nullable( + new ArrowType.Union( + UnionMode.Dense, + new int[] { + ArrowType.ArrowTypeID.Int.getFlatbufID(), + ArrowType.ArrowTypeID.FloatingPoint.getFlatbufID(), + ArrowType.ArrowTypeID.Utf8.getFlatbufID() + })), + Arrays.asList( + new Field( + "intMember", FieldType.notNullable(new ArrowType.Int(32, true)), null), + new Field( + "floatMember", + FieldType.notNullable( + new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)), + null), + new Field("stringMember", FieldType.notNullable(new ArrowType.Utf8()), null))), + new Field( + "nullableSparseUnionField", + FieldType.nullable( + new ArrowType.Union( + UnionMode.Sparse, + new int[] { + ArrowType.ArrowTypeID.Int.getFlatbufID(), + ArrowType.ArrowTypeID.FloatingPoint.getFlatbufID(), + ArrowType.ArrowTypeID.Utf8.getFlatbufID() + })), + Arrays.asList( + new Field( + "nullableIntMember", FieldType.nullable(new ArrowType.Int(32, true)), null), + new Field( + "nullableFloatMember", + FieldType.nullable( + new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)), + null), + new Field( + "nullableStringMember", FieldType.nullable(new ArrowType.Utf8()), null))), + new Field( + "nullableDenseUnionField", + FieldType.nullable( + new ArrowType.Union( + UnionMode.Dense, + new int[] { + ArrowType.ArrowTypeID.Int.getFlatbufID(), + ArrowType.ArrowTypeID.FloatingPoint.getFlatbufID(), + ArrowType.ArrowTypeID.Utf8.getFlatbufID() + })), + Arrays.asList( + new Field( + "nullableIntMember", FieldType.nullable(new ArrowType.Int(32, true)), null), + new Field( + "nullableFloatMember", + FieldType.nullable( + new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)), + null), + new Field( + "nullableStringMember", FieldType.nullable(new ArrowType.Utf8()), null)))); + + Schema schema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord"); + + assertEquals(Schema.Type.RECORD, schema.getType()); + assertEquals(4, schema.getFields().size()); + + // Assertions for sparseUnionField + assertEquals(Schema.Type.UNION, schema.getField("sparseUnionField").schema().getType()); + assertEquals(3, schema.getField("sparseUnionField").schema().getTypes().size()); + assertEquals( + Schema.Type.INT, schema.getField("sparseUnionField").schema().getTypes().get(0).getType()); + assertEquals( + Schema.Type.FLOAT, + schema.getField("sparseUnionField").schema().getTypes().get(1).getType()); + assertEquals( + Schema.Type.STRING, + schema.getField("sparseUnionField").schema().getTypes().get(2).getType()); + + // Assertions for denseUnionField + assertEquals(Schema.Type.UNION, schema.getField("denseUnionField").schema().getType()); + assertEquals(3, schema.getField("denseUnionField").schema().getTypes().size()); + assertEquals( + Schema.Type.INT, schema.getField("denseUnionField").schema().getTypes().get(0).getType()); + assertEquals( + Schema.Type.FLOAT, schema.getField("denseUnionField").schema().getTypes().get(1).getType()); + assertEquals( + Schema.Type.STRING, + schema.getField("denseUnionField").schema().getTypes().get(2).getType()); + + // Assertions for sparseUnionField + assertEquals(Schema.Type.UNION, schema.getField("nullableSparseUnionField").schema().getType()); + assertEquals(4, schema.getField("nullableSparseUnionField").schema().getTypes().size()); + assertEquals( + Schema.Type.NULL, + schema.getField("nullableSparseUnionField").schema().getTypes().get(0).getType()); + assertEquals( + Schema.Type.INT, + schema.getField("nullableSparseUnionField").schema().getTypes().get(1).getType()); + assertEquals( + Schema.Type.FLOAT, + schema.getField("nullableSparseUnionField").schema().getTypes().get(2).getType()); + assertEquals( + Schema.Type.STRING, + schema.getField("nullableSparseUnionField").schema().getTypes().get(3).getType()); + + // Assertions for denseUnionField + assertEquals(Schema.Type.UNION, schema.getField("nullableDenseUnionField").schema().getType()); + assertEquals(4, schema.getField("nullableDenseUnionField").schema().getTypes().size()); + assertEquals( + Schema.Type.NULL, + schema.getField("nullableDenseUnionField").schema().getTypes().get(0).getType()); + assertEquals( + Schema.Type.INT, + schema.getField("nullableDenseUnionField").schema().getTypes().get(1).getType()); + assertEquals( + Schema.Type.FLOAT, + schema.getField("nullableDenseUnionField").schema().getTypes().get(2).getType()); + assertEquals( + Schema.Type.STRING, + schema.getField("nullableDenseUnionField").schema().getTypes().get(3).getType()); + } +} diff --git a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/TestWriteReadAvroRecord.java b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/TestWriteReadAvroRecord.java index c318214f5c..76e58a75ae 100644 --- a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/TestWriteReadAvroRecord.java +++ b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/TestWriteReadAvroRecord.java @@ -19,8 +19,22 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import org.apache.arrow.adapter.avro.consumers.CompositeAvroConsumer; +import org.apache.arrow.adapter.avro.producers.CompositeAvroProducer; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.avro.Schema; import org.apache.avro.file.DataFileReader; import org.apache.avro.file.DataFileWriter; @@ -28,10 +42,16 @@ import org.apache.avro.generic.GenericDatumReader; import org.apache.avro.generic.GenericDatumWriter; import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.BinaryEncoder; import org.apache.avro.io.DatumReader; import org.apache.avro.io.DatumWriter; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.EncoderFactory; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; public class TestWriteReadAvroRecord { @@ -82,4 +102,85 @@ public void testWriteAndRead() throws Exception { assertEquals(7, deUser2.get("favorite_number")); assertEquals("red", deUser2.get("favorite_color").toString()); } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testWriteAndReadVSR(boolean useSchemaFile) throws Exception { + + BufferAllocator allocator = new RootAllocator(); + FieldType stringNotNull = new FieldType(false, ArrowType.Utf8.INSTANCE, null); + FieldType stringNull = new FieldType(true, ArrowType.Utf8.INSTANCE, null); + FieldType intN32Null = new FieldType(true, new ArrowType.Int(32, true), null); + + List fields = new ArrayList<>(); + fields.add(new Field("name", stringNotNull, null)); + fields.add(new Field("favorite_number", intN32Null, null)); + fields.add(new Field("favorite_color", stringNull, null)); + + VarCharVector nameVector = new VarCharVector(fields.get(0), allocator); + nameVector.allocateNew(2); + nameVector.set(0, "Alyssa".getBytes(StandardCharsets.UTF_8)); + nameVector.set(1, "Ben".getBytes(StandardCharsets.UTF_8)); + + IntVector favNumberVector = new IntVector(fields.get(1), allocator); + favNumberVector.allocateNew(2); + favNumberVector.set(0, 256); + favNumberVector.set(1, 7); + + VarCharVector favColorVector = new VarCharVector(fields.get(2), allocator); + favColorVector.allocateNew(2); + favColorVector.setNull(0); + favColorVector.set(1, "red".getBytes(StandardCharsets.UTF_8)); + + List vectors = new ArrayList<>(); + vectors.add(nameVector); + vectors.add(favNumberVector); + vectors.add(favColorVector); + + Schema schema = + useSchemaFile + ? AvroTestBase.getSchema("test.avsc") + : ArrowToAvroUtils.createAvroSchema(fields); + + File dataFile = new File(TMP, "test_vsr.avro"); + AvroToArrowConfig config = new AvroToArrowConfigBuilder(allocator).build(); + + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = ArrowToAvroUtils.createCompositeProducer(vectors); + + producer.produce(encoder); + producer.produce(encoder); + + encoder.flush(); + } + + List roundTripFields = new ArrayList<>(); + List roundTripVectors = new ArrayList<>(); + + try (FileInputStream fis = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = new DecoderFactory().directBinaryDecoder(fis, null); + CompositeAvroConsumer consumer = AvroToArrowUtils.createCompositeConsumer(schema, config); + + consumer.getConsumers().forEach(c -> roundTripFields.add(c.getVector().getField())); + consumer.getConsumers().forEach(c -> roundTripVectors.add(c.getVector())); + consumer.consume(decoder); + consumer.consume(decoder); + } + + VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, 2); + VectorSchemaRoot roundTripRoot = new VectorSchemaRoot(roundTripFields, roundTripVectors, 2); + + assertEquals(root.getRowCount(), roundTripRoot.getRowCount()); + + for (int row = 0; row < 2; row++) { + for (int col = 0; col < 3; col++) { + FieldVector vector = root.getVector(col); + FieldVector roundTripVector = roundTripRoot.getVector(col); + assertEquals(vector.getObject(row), roundTripVector.getObject(row)); + } + } + } } From c29fcfc72bed84c1dc9926bc28294b5ff193937f Mon Sep 17 00:00:00 2001 From: Vishal Boddu <9461605+bodduv@users.noreply.github.com> Date: Thu, 3 Apr 2025 17:25:44 +0200 Subject: [PATCH 138/408] GH-692: Preserve nullability information while transfering DecimalVector and Decimal256Vector (#693) ## What's Changed This PR proposes to use the "from" `ValueVector`'s field while transferring `DecimalVector` and `Decimal256Vector` to preserve nullability information. This is to have similar behavior with all the other primitive value vectors. Note that the `FieldType` of the value vector has nullability information. Closes #692 . --- .../apache/arrow/vector/Decimal256Vector.java | 7 +++-- .../apache/arrow/vector/DecimalVector.java | 5 +++- .../arrow/vector/TestDecimal256Vector.java | 28 +++++++++++++++++++ .../arrow/vector/TestDecimalVector.java | 26 +++++++++++++++++ 4 files changed, 63 insertions(+), 3 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java b/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java index 90f67798dd..f9d7e5cb9e 100644 --- a/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java +++ b/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java @@ -567,8 +567,11 @@ private class TransferImpl implements TransferPair { public TransferImpl(String ref, BufferAllocator allocator) { to = - new Decimal256Vector( - ref, allocator, Decimal256Vector.this.precision, Decimal256Vector.this.scale); + (Decimal256Vector.this.field != null + && Decimal256Vector.this.field.getFieldType() != null) + ? new Decimal256Vector(ref, Decimal256Vector.this.field.getFieldType(), allocator) + : new Decimal256Vector( + ref, allocator, Decimal256Vector.this.precision, Decimal256Vector.this.scale); } public TransferImpl(Field field, BufferAllocator allocator) { diff --git a/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java b/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java index b4c55680b7..9bf1812cc6 100644 --- a/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java @@ -565,7 +565,10 @@ private class TransferImpl implements TransferPair { public TransferImpl(String ref, BufferAllocator allocator) { to = - new DecimalVector(ref, allocator, DecimalVector.this.precision, DecimalVector.this.scale); + (DecimalVector.this.field != null && DecimalVector.this.field.getFieldType() != null) + ? new DecimalVector(ref, DecimalVector.this.field.getFieldType(), allocator) + : new DecimalVector( + ref, allocator, DecimalVector.this.precision, DecimalVector.this.scale); } public TransferImpl(Field field, BufferAllocator allocator) { diff --git a/vector/src/test/java/org/apache/arrow/vector/TestDecimal256Vector.java b/vector/src/test/java/org/apache/arrow/vector/TestDecimal256Vector.java index c155ab98fa..b995dc5d92 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestDecimal256Vector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestDecimal256Vector.java @@ -26,6 +26,7 @@ import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.TransferPair; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -375,6 +376,33 @@ public void testGetTransferPairWithField() { assertSame(fromVector.getField(), toVector.getField()); } + @Test + public void testGetTransferPairWithoutField() { + final Decimal256Vector fromVector = new Decimal256Vector("decimal", allocator, 10, scale); + final TransferPair transferPair = + fromVector.getTransferPair(fromVector.getField().getName(), allocator); + final Decimal256Vector toVector = (Decimal256Vector) transferPair.getTo(); + // A new Field created inside a new vector should reuse the field type (should be the same in + // memory as the original Field's field type). + assertSame(fromVector.getField().getFieldType(), toVector.getField().getFieldType()); + } + + @Test + public void testGetTransferPairWithoutFieldNonNullable() { + final FieldType decimal256NonNullableType = + new FieldType( + false, new ArrowType.Decimal(10, scale, Decimal256Vector.TYPE_WIDTH * 8), null); + final Decimal256Vector fromVector = + new Decimal256Vector("decimal", decimal256NonNullableType, allocator); + final TransferPair transferPair = + fromVector.getTransferPair(fromVector.getField().getName(), allocator); + final Decimal256Vector toVector = (Decimal256Vector) transferPair.getTo(); + // A new Field created inside a new vector should reuse the field type (should be the same in + // memory as the original Field's field type). + assertSame(fromVector.getField().getFieldType(), toVector.getField().getFieldType()); + assertSame(decimal256NonNullableType, toVector.getField().getFieldType()); + } + private void verifyWritingArrowBufWithBigEndianBytes( Decimal256Vector decimalVector, ArrowBuf buf, BigDecimal[] expectedValues, int length) { decimalVector.allocateNew(); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java b/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java index d5310bad0e..85c11e8f3d 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java @@ -26,6 +26,7 @@ import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.TransferPair; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -371,6 +372,31 @@ public void testGetTransferPairWithField() { assertSame(fromVector.getField(), toVector.getField()); } + @Test + public void testGetTransferPairWithoutField() { + final DecimalVector fromVector = new DecimalVector("decimal", allocator, 10, scale); + final TransferPair transferPair = + fromVector.getTransferPair(fromVector.getField().getName(), allocator); + final DecimalVector toVector = (DecimalVector) transferPair.getTo(); + // A new Field created inside a new vector should reuse the field type (should be the same in + // memory as the original Field's field type). + assertSame(fromVector.getField().getFieldType(), toVector.getField().getFieldType()); + } + + @Test + public void testGetTransferPairWithoutFieldNonNullable() { + final FieldType decimalNonNullableType = + new FieldType(false, new ArrowType.Decimal(10, scale), null); + final DecimalVector fromVector = + new DecimalVector("decimal", decimalNonNullableType, allocator); + final TransferPair transferPair = + fromVector.getTransferPair(fromVector.getField().getName(), allocator); + final DecimalVector toVector = (DecimalVector) transferPair.getTo(); + // A new Field created inside a new vector should reuse the field type (should be the same in + // memory as the original Field's field type). + assertSame(fromVector.getField().getFieldType(), toVector.getField().getFieldType()); + } + private void verifyWritingArrowBufWithBigEndianBytes( DecimalVector decimalVector, ArrowBuf buf, BigDecimal[] expectedValues, int length) { decimalVector.allocateNew(); From 8ffc1d3eb58bbe4fc023d2222cc4439dace7e8ea Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 8 Apr 2025 12:17:11 +0900 Subject: [PATCH 139/408] MINOR: Fix JNI code after upstream DCHECK change (#706) ## What's Changed Upstream renamed the public DCHECK macros to ARROW_DCHECK (https://github.com/apache/arrow/pull/46015) --- .github/workflows/rc.yml | 5 +++++ dataset/src/main/cpp/jni_util.cc | 4 ++-- gandiva/src/main/cpp/expression_registry_helper.cc | 2 +- gandiva/src/main/cpp/jni_common.cc | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 5b78cc9395..e039306ec3 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -261,6 +261,11 @@ jobs: # bundled Protobuf. brew uninstall protobuf + # We need Flatbuffers 24, not the latest version + # Homebrew does not offer older versions, so remove the Homebrew + # package and rely on Arrow using a bundled version instead + brew uninstall flatbuffers + brew bundle --file=Brewfile - name: Prepare ccache run: | diff --git a/dataset/src/main/cpp/jni_util.cc b/dataset/src/main/cpp/jni_util.cc index 1fd15696e6..35bfb328f0 100644 --- a/dataset/src/main/cpp/jni_util.cc +++ b/dataset/src/main/cpp/jni_util.cc @@ -187,7 +187,7 @@ ReservationListenableMemoryPool::~ReservationListenableMemoryPool() {} std::string Describe(JNIEnv* env, jthrowable t) { jclass describer_class = env->FindClass("org/apache/arrow/dataset/jni/JniExceptionDescriber"); - DCHECK_NE(describer_class, nullptr); + ARROW_DCHECK_NE(describer_class, nullptr); jmethodID describe_method = env->GetStaticMethodID( describer_class, "describe", "(Ljava/lang/Throwable;)Ljava/lang/String;"); std::string description = JStringToCString( @@ -197,7 +197,7 @@ std::string Describe(JNIEnv* env, jthrowable t) { bool IsErrorInstanceOf(JNIEnv* env, jthrowable t, std::string class_name) { jclass java_class = env->FindClass(class_name.c_str()); - DCHECK_NE(java_class, nullptr) << "Could not find Java class " << class_name; + ARROW_DCHECK_NE(java_class, nullptr) << "Could not find Java class " << class_name; return env->IsInstanceOf(t, java_class); } diff --git a/gandiva/src/main/cpp/expression_registry_helper.cc b/gandiva/src/main/cpp/expression_registry_helper.cc index 66b97c8b9e..21077ff1db 100644 --- a/gandiva/src/main/cpp/expression_registry_helper.cc +++ b/gandiva/src/main/cpp/expression_registry_helper.cc @@ -138,7 +138,7 @@ void ArrowToProtobuf(DataTypePtr type, gandiva::types::ExtGandivaType* gandiva_d default: // un-supported types. test ensures that // when one of these are added build breaks. - DCHECK(false); + ARROW_DCHECK(false); } } diff --git a/gandiva/src/main/cpp/jni_common.cc b/gandiva/src/main/cpp/jni_common.cc index ec1bb76234..2851250072 100644 --- a/gandiva/src/main/cpp/jni_common.cc +++ b/gandiva/src/main/cpp/jni_common.cc @@ -751,7 +751,7 @@ Status JavaResizableBuffer::Resize(const int64_t new_size, bool shrink_to_fit) { } RETURN_NOT_OK(Reserve(new_size)); - DCHECK_GE(capacity_, new_size); + ARROW_DCHECK_GE(capacity_, new_size); size_ = new_size; return Status::OK(); } From 7bd2b0d3853b6ef1066aa2e9819743f441082dd8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Apr 2025 01:21:37 -0400 Subject: [PATCH 140/408] MINOR: Bump checker.framework.version from 3.49.1 to 3.49.2 (#703) Bumps `checker.framework.version` from 3.49.1 to 3.49.2. Updates `org.checkerframework:checker-qual` from 3.49.1 to 3.49.2 Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index df32105804..5462d1416e 100644 --- a/pom.xml +++ b/pom.xml @@ -109,7 +109,7 @@ under the License. 10.22.0 true 2.37.0 - 3.49.1 + 3.49.2 1.5.18 none -Xdoclint:none From 8d55fd1b52863bc0d2606c623d34e3eaa3be7b3d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Apr 2025 01:26:19 -0400 Subject: [PATCH 141/408] MINOR: Bump org.mockito:mockito-bom from 5.16.1 to 5.17.0 (#701) Bumps [org.mockito:mockito-bom](https://github.com/mockito/mockito) from 5.16.1 to 5.17.0. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5462d1416e..9e6c8b782e 100644 --- a/pom.xml +++ b/pom.xml @@ -103,7 +103,7 @@ under the License. 3.4.1 25.2.10 1.12.0 - 5.16.1 + 5.17.0 2 10.22.0 From 42a126fd88a1430b474fb4571b0c5b99c83ee770 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Apr 2025 01:47:44 -0400 Subject: [PATCH 142/408] MINOR: Bump com.puppycrawl.tools:checkstyle from 10.22.0 to 10.23.0 (#702) Bumps [com.puppycrawl.tools:checkstyle](https://github.com/checkstyle/checkstyle) from 10.22.0 to 10.23.0. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9e6c8b782e..cb189dc8f9 100644 --- a/pom.xml +++ b/pom.xml @@ -106,7 +106,7 @@ under the License. 5.17.0 2 - 10.22.0 + 10.23.0 true 2.37.0 3.49.2 From f92585c272ea4b2db5b1717f6642f3edc8f99903 Mon Sep 17 00:00:00 2001 From: Kristin Cowalcijk Date: Tue, 8 Apr 2025 14:11:32 +0800 Subject: [PATCH 143/408] GH-704: Fix initialization of offset buffer when exporting VarChar vectors through C Data Interface (#705) ## What's Changed This patch fixes the initialization of offset buffers when exporting variable width arrays through Arrow C Data Interface. The original code incorrectly mess up with the member `this.offsetBuffer` while we should actually initialize the newly allocated offsetBuffer. I think the diff itself will be quite self-explanatory. Closes #704 and probably #88 . --- .../org/apache/arrow/c/RoundtripTest.java | 23 +++++++++++++++++++ .../vector/BaseLargeVariableWidthVector.java | 2 +- .../arrow/vector/BaseVariableWidthVector.java | 2 +- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/c/src/test/java/org/apache/arrow/c/RoundtripTest.java b/c/src/test/java/org/apache/arrow/c/RoundtripTest.java index 67ab282de5..6d68449c0b 100644 --- a/c/src/test/java/org/apache/arrow/c/RoundtripTest.java +++ b/c/src/test/java/org/apache/arrow/c/RoundtripTest.java @@ -38,6 +38,8 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.memory.util.hash.ArrowBufHasher; +import org.apache.arrow.vector.BaseLargeVariableWidthVector; +import org.apache.arrow.vector.BaseVariableWidthVector; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.DateDayVector; @@ -181,6 +183,13 @@ boolean roundtrip(FieldVector vector, Class clazz) { clazz.isInstance(imported), String.format("expected %s but was %s", clazz, imported.getClass())); result = VectorEqualsVisitor.vectorEquals(vector, imported); + + if (imported instanceof BaseVariableWidthVector + || imported instanceof BaseLargeVariableWidthVector) { + ArrowBuf offsetBuffer = imported.getOffsetBuffer(); + assertTrue(offsetBuffer.capacity() > 0); + assertEquals(0, offsetBuffer.getInt(0)); + } } // Check that the ref counts of the buffers are the same after the roundtrip @@ -602,6 +611,13 @@ public void testVarCharVector() { } } + @Test + public void testEmptyVarCharVector() { + try (final VarCharVector vector = new VarCharVector("v", allocator)) { + assertTrue(roundtrip(vector, VarCharVector.class)); + } + } + @Test public void testLargeVarBinaryVector() { try (final LargeVarBinaryVector vector = new LargeVarBinaryVector("", allocator)) { @@ -635,6 +651,13 @@ public void testLargeVarCharVector() { } } + @Test + public void testEmptyLargeVarCharVector() { + try (final LargeVarCharVector vector = new LargeVarCharVector("v", allocator)) { + assertTrue(roundtrip(vector, LargeVarCharVector.class)); + } + } + @Test public void testListVector() { try (final ListVector vector = ListVector.empty("v", allocator)) { diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java index 552a896ea8..7e0d0affc6 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java @@ -496,7 +496,7 @@ private void allocateBytes(final long valueBufferSize, final int valueCount) { private ArrowBuf allocateOffsetBuffer(final long size) { ArrowBuf offsetBuffer = allocator.buffer(size); offsetBuffer.readerIndex(0); - initOffsetBuffer(); + offsetBuffer.setZero(0, offsetBuffer.capacity()); return offsetBuffer; } diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java index aaccec602f..7b8d2cdfda 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java @@ -514,7 +514,7 @@ private ArrowBuf allocateOffsetBuffer(final long size) { final int curSize = (int) size; ArrowBuf offsetBuffer = allocator.buffer(curSize); offsetBuffer.readerIndex(0); - initOffsetBuffer(); + offsetBuffer.setZero(0, offsetBuffer.capacity()); return offsetBuffer; } From 222f30e75d079ae5005cfc5764f4c905eaa2ca5c Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 8 Apr 2025 15:58:08 +0900 Subject: [PATCH 144/408] GH-494: [Flight] Allow configuring connect timeout in JDBC (#495) Allow configuring the connect timeout via `connectTimeoutMs` so that the driver doesn't wait so long before giving up on unreachable locations. Fixes #494. --- flight/flight-sql-jdbc-core/pom.xml | 15 ++ .../driver/jdbc/ArrowFlightConnection.java | 1 + .../client/ArrowFlightSqlClientHandler.java | 52 +++++-- .../ArrowFlightConnectionConfigImpl.java | 15 +- .../arrow/driver/jdbc/ResultSetTest.java | 140 +++++++++++++++++- ...rrowFlightSqlClientHandlerBuilderTest.java | 1 + .../ArrowFlightConnectionConfigImplTest.java | 32 +++- .../jdbc/utils/FallbackFlightSqlProducer.java | 10 ++ 8 files changed, 240 insertions(+), 26 deletions(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 15368e918d..3362b60a8e 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -47,6 +47,21 @@ under the License. + + io.grpc + grpc-api + + + + io.grpc + grpc-netty + + + + io.netty + netty-transport + + org.apache.arrow arrow-memory-core diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java index c1b1c8f8e6..cf9804d68b 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java @@ -113,6 +113,7 @@ private static ArrowFlightSqlClientHandler createNewClientHandler( .withRetainCookies(config.retainCookies()) .withRetainAuth(config.retainAuth()) .withCatalog(config.getCatalog()) + .withConnectTimeout(config.getConnectTimeout()) .build(); } catch (final SQLException e) { try { diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java index 0e9c79a090..cbbe223eb8 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java @@ -17,10 +17,13 @@ package org.apache.arrow.driver.jdbc.client; import com.google.common.collect.ImmutableMap; +import io.grpc.netty.NettyChannelBuilder; +import io.netty.channel.ChannelOption; import java.io.IOException; import java.net.URI; import java.security.GeneralSecurityException; import java.sql.SQLException; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -36,6 +39,7 @@ import org.apache.arrow.flight.FlightClient; import org.apache.arrow.flight.FlightClientMiddleware; import org.apache.arrow.flight.FlightEndpoint; +import org.apache.arrow.flight.FlightGrpcUtils; import org.apache.arrow.flight.FlightInfo; import org.apache.arrow.flight.FlightRuntimeException; import org.apache.arrow.flight.FlightStatusCode; @@ -50,6 +54,7 @@ import org.apache.arrow.flight.auth2.ClientIncomingAuthHeaderMiddleware; import org.apache.arrow.flight.client.ClientCookieMiddleware; import org.apache.arrow.flight.grpc.CredentialCallOption; +import org.apache.arrow.flight.grpc.NettyClientBuilder; import org.apache.arrow.flight.sql.FlightSqlClient; import org.apache.arrow.flight.sql.impl.FlightSql.SqlInfo; import org.apache.arrow.flight.sql.util.TableRef; @@ -138,12 +143,11 @@ public List getStreams(final FlightInfo flightInfo) // Clone the builder and then set the new endpoint on it. // GH-38574: Currently a new FlightClient will be made for each partition that returns a - // non-empty Location - // then disposed of. It may be better to cache clients because a server may report the - // same Locations. - // It would also be good to identify when the reported location is the same as the - // original connection's - // Location and skip creating a FlightClient in that scenario. + // non-empty Location then disposed of. It may be better to cache clients because a server + // may report the same Locations. It would also be good to identify when the reported + // location + // is the same as the original connection's Location and skip creating a FlightClient in + // that scenario. List exceptions = new ArrayList<>(); CloseableEndpointStreamPair stream = null; for (Location location : endpoint.getLocations()) { @@ -158,7 +162,8 @@ public List getStreams(final FlightInfo flightInfo) new Builder(ArrowFlightSqlClientHandler.this.builder) .withHost(endpointUri.getHost()) .withPort(endpointUri.getPort()) - .withEncryption(endpointUri.getScheme().equals(LocationSchemes.GRPC_TLS)); + .withEncryption(endpointUri.getScheme().equals(LocationSchemes.GRPC_TLS)) + .withConnectTimeout(builder.connectTimeout); ArrowFlightSqlClientHandler endpointHandler = null; try { @@ -177,6 +182,7 @@ public List getStreams(final FlightInfo flightInfo) exceptions.add(ex); continue; } + break; } if (stream != null) { @@ -543,6 +549,8 @@ public static final class Builder { @VisibleForTesting Optional catalog = Optional.empty(); + @VisibleForTesting @Nullable Duration connectTimeout; + // These two middleware are for internal use within build() and should not be exposed by builder // APIs. // Note that these middleware may not necessarily be registered. @@ -825,6 +833,19 @@ public Builder withCatalog(@Nullable final String catalog) { return this; } + public Builder withConnectTimeout(Duration connectTimeout) { + this.connectTimeout = connectTimeout; + return this; + } + + /** Get the location that this client will connect to. */ + public Location getLocation() { + if (useEncryption) { + return Location.forGrpcTls(host, port); + } + return Location.forGrpcInsecure(host, port); + } + /** * Builds a new {@link ArrowFlightSqlClientHandler} from the provided fields. * @@ -845,17 +866,15 @@ public ArrowFlightSqlClientHandler build() throws SQLException { if (isUsingUserPasswordAuth) { buildTimeMiddlewareFactories.add(authFactory); } - final FlightClient.Builder clientBuilder = FlightClient.builder().allocator(allocator); + final NettyClientBuilder clientBuilder = new NettyClientBuilder(); + clientBuilder.allocator(allocator); buildTimeMiddlewareFactories.add(new ClientCookieMiddleware.Factory()); buildTimeMiddlewareFactories.forEach(clientBuilder::intercept); - Location location; if (useEncryption) { - location = Location.forGrpcTls(host, port); clientBuilder.useTls(); - } else { - location = Location.forGrpcInsecure(host, port); } + Location location = getLocation(); clientBuilder.location(location); if (useEncryption) { @@ -883,7 +902,14 @@ public ArrowFlightSqlClientHandler build() throws SQLException { } } - client = clientBuilder.build(); + NettyChannelBuilder channelBuilder = clientBuilder.build(); + if (connectTimeout != null) { + channelBuilder.withOption( + ChannelOption.CONNECT_TIMEOUT_MILLIS, (int) connectTimeout.toMillis()); + } + client = + FlightGrpcUtils.createFlightClient( + allocator, channelBuilder.build(), clientBuilder.middleware()); final ArrayList credentialOptions = new ArrayList<>(); if (isUsingUserPasswordAuth) { // If the authFactory has already been used for a handshake, use the existing token. diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java index e8bae2a207..ab6a5898b7 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.driver.jdbc.utils; +import java.time.Duration; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -163,6 +164,16 @@ public String getCatalog() { return ArrowFlightConnectionProperty.CATALOG.getString(properties); } + /** The initial connect timeout. */ + public Duration getConnectTimeout() { + Integer timeout = ArrowFlightConnectionProperty.CONNECT_TIMEOUT_MILLIS.getInteger(properties); + if (timeout == null) { + return Duration.ofMillis( + (int) ArrowFlightConnectionProperty.CONNECT_TIMEOUT_MILLIS.defaultValue()); + } + return Duration.ofMillis(timeout); + } + /** * Gets the {@link CallOption}s from this {@link ConnectionConfig}. * @@ -213,7 +224,9 @@ public enum ArrowFlightConnectionProperty implements ConnectionProperty { TOKEN("token", null, Type.STRING, false), RETAIN_COOKIES("retainCookies", true, Type.BOOLEAN, false), RETAIN_AUTH("retainAuth", true, Type.BOOLEAN, false), - CATALOG("catalog", null, Type.STRING, false); + CATALOG("catalog", null, Type.STRING, false), + CONNECT_TIMEOUT_MILLIS("connectTimeoutMs", 10000, Type.NUMBER, false), + ; private final String camelName; private final Object defaultValue; diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java index a8d04dfc83..cd47408f52 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java @@ -25,12 +25,7 @@ import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assertions.*; import com.google.common.collect.ImmutableSet; import java.nio.charset.StandardCharsets; @@ -645,6 +640,139 @@ public void testFallbackSecondFlightServer() throws Exception { } } + @Test + public void testFallbackUnresolvableFlightServer() throws Exception { + final Schema schema = + new Schema( + Collections.singletonList(Field.nullable("int_column", Types.MinorType.INT.getType()))); + try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + VectorSchemaRoot resultData = VectorSchemaRoot.create(schema, allocator)) { + resultData.setRowCount(1); + ((IntVector) resultData.getVector(0)).set(0, 1); + + try (final FallbackFlightSqlProducer rootProducer = + new FallbackFlightSqlProducer(resultData); + FlightServer rootServer = + FlightServer.builder(allocator, forGrpcInsecure("localhost", 0), rootProducer) + .build() + .start(); + Connection newConnection = + DriverManager.getConnection( + String.format( + "jdbc:arrow-flight-sql://%s:%d/?useEncryption=false", + rootServer.getLocation().getUri().getHost(), rootServer.getPort()))) { + // This first attempt should take a measurable amount of time. + long start = System.nanoTime(); + try (Statement newStatement = newConnection.createStatement()) { + try (ResultSet result = newStatement.executeQuery("fallback with unresolvable")) { + List actualData = new ArrayList<>(); + while (result.next()) { + actualData.add(result.getInt(1)); + } + + // Assert + assertEquals(resultData.getRowCount(), actualData.size()); + assertTrue(actualData.contains(((IntVector) resultData.getVector(0)).get(0))); + } + } + long attempt1 = System.nanoTime(); + double elapsedMs = (attempt1 - start) / 1_000_000.; + assertTrue( + elapsedMs >= 5000., + String.format( + "Expected first attempt to hit the timeout, but only %f ms elapsed", elapsedMs)); + + // Once the client cache is implemented (GH-661), this second attempt should take less time, + // since the failure from before should be cached. + start = System.nanoTime(); + try (Statement newStatement = newConnection.createStatement()) { + try (ResultSet result = newStatement.executeQuery("fallback with unresolvable")) { + List actualData = new ArrayList<>(); + while (result.next()) { + actualData.add(result.getInt(1)); + } + + // Assert + assertEquals(resultData.getRowCount(), actualData.size()); + assertTrue(actualData.contains(((IntVector) resultData.getVector(0)).get(0))); + } + } + attempt1 = System.nanoTime(); + elapsedMs = (attempt1 - start) / 1_000_000.; + // TODO(GH-661): this assertion should be flipped to assertTrue. + assertFalse( + elapsedMs < 5000., + String.format("Expected second attempt to be the same, but %f ms elapsed", elapsedMs)); + } + } + } + + @Test + public void testFallbackUnresolvableFlightServerDisableCache() throws Exception { + final Schema schema = + new Schema( + Collections.singletonList(Field.nullable("int_column", Types.MinorType.INT.getType()))); + try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + VectorSchemaRoot resultData = VectorSchemaRoot.create(schema, allocator)) { + resultData.setRowCount(1); + ((IntVector) resultData.getVector(0)).set(0, 1); + + try (final FallbackFlightSqlProducer rootProducer = + new FallbackFlightSqlProducer(resultData); + FlightServer rootServer = + FlightServer.builder(allocator, forGrpcInsecure("localhost", 0), rootProducer) + .build() + .start(); + Connection newConnection = + DriverManager.getConnection( + String.format( + "jdbc:arrow-flight-sql://%s:%d/?useEncryption=false&useClientCache=false", + rootServer.getLocation().getUri().getHost(), rootServer.getPort()))) { + // This first attempt should take a measurable amount of time. + long start = System.nanoTime(); + try (Statement newStatement = newConnection.createStatement()) { + try (ResultSet result = newStatement.executeQuery("fallback with unresolvable")) { + List actualData = new ArrayList<>(); + while (result.next()) { + actualData.add(result.getInt(1)); + } + + // Assert + assertEquals(resultData.getRowCount(), actualData.size()); + assertTrue(actualData.contains(((IntVector) resultData.getVector(0)).get(0))); + } + } + long attempt1 = System.nanoTime(); + double elapsedMs = (attempt1 - start) / 1_000_000.; + assertTrue( + elapsedMs >= 5000., + String.format( + "Expected first attempt to hit the timeout, but only %f ms elapsed", elapsedMs)); + + // This second attempt should take a long time still, since we disabled the cache. + start = System.nanoTime(); + try (Statement newStatement = newConnection.createStatement()) { + try (ResultSet result = newStatement.executeQuery("fallback with unresolvable")) { + List actualData = new ArrayList<>(); + while (result.next()) { + actualData.add(result.getInt(1)); + } + + // Assert + assertEquals(resultData.getRowCount(), actualData.size()); + assertTrue(actualData.contains(((IntVector) resultData.getVector(0)).get(0))); + } + } + attempt1 = System.nanoTime(); + elapsedMs = (attempt1 - start) / 1_000_000.; + assertTrue( + elapsedMs >= 5000., + String.format( + "Expected second attempt to hit the timeout, but only %f ms elapsed", elapsedMs)); + } + } + } + @Test public void testShouldRunSelectQueryWithEmptyVectorsEmbedded() throws Exception { try (Statement statement = connection.createStatement(); diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerBuilderTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerBuilderTest.java index 6beaba8236..7b416638e1 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerBuilderTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerBuilderTest.java @@ -147,6 +147,7 @@ public void testDefaults() { assertNull(builder.clientCertificatePath); assertNull(builder.clientKeyPath); assertEquals(Optional.empty(), builder.catalog); + assertNull(builder.connectTimeout); } @Test diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImplTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImplTest.java index 4a46b5f5be..c780d53fab 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImplTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImplTest.java @@ -18,6 +18,7 @@ import static java.lang.Runtime.getRuntime; import static org.apache.arrow.driver.jdbc.utils.ArrowFlightConnectionConfigImpl.ArrowFlightConnectionProperty.CATALOG; +import static org.apache.arrow.driver.jdbc.utils.ArrowFlightConnectionConfigImpl.ArrowFlightConnectionProperty.CONNECT_TIMEOUT_MILLIS; import static org.apache.arrow.driver.jdbc.utils.ArrowFlightConnectionConfigImpl.ArrowFlightConnectionProperty.HOST; import static org.apache.arrow.driver.jdbc.utils.ArrowFlightConnectionConfigImpl.ArrowFlightConnectionProperty.PASSWORD; import static org.apache.arrow.driver.jdbc.utils.ArrowFlightConnectionConfigImpl.ArrowFlightConnectionProperty.PORT; @@ -27,6 +28,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +import java.time.Duration; import java.util.Properties; import java.util.Random; import java.util.function.Function; @@ -59,49 +61,67 @@ public void setUp() { public void testGetProperty( ArrowFlightConnectionProperty property, Object value, + Object expected, Function configFunction) { properties.put(property.camelName(), value); arrowFlightConnectionConfigFunction = configFunction; - assertThat(configFunction.apply(arrowFlightConnectionConfig), is(value)); - assertThat(arrowFlightConnectionConfigFunction.apply(arrowFlightConnectionConfig), is(value)); + assertThat(configFunction.apply(arrowFlightConnectionConfig), is(expected)); + assertThat( + arrowFlightConnectionConfigFunction.apply(arrowFlightConnectionConfig), is(expected)); } public static Stream provideParameters() { + int port = RANDOM.nextInt(Short.toUnsignedInt(Short.MAX_VALUE)); + boolean useEncryption = RANDOM.nextBoolean(); + int threadPoolSize = RANDOM.nextInt(getRuntime().availableProcessors()); return Stream.of( Arguments.of( HOST, "host", + "host", (Function) ArrowFlightConnectionConfigImpl::getHost), Arguments.of( PORT, - RANDOM.nextInt(Short.toUnsignedInt(Short.MAX_VALUE)), + port, + port, (Function) ArrowFlightConnectionConfigImpl::getPort), Arguments.of( USER, "user", + "user", (Function) ArrowFlightConnectionConfigImpl::getUser), Arguments.of( PASSWORD, "password", + "password", (Function) ArrowFlightConnectionConfigImpl::getPassword), Arguments.of( USE_ENCRYPTION, - RANDOM.nextBoolean(), + useEncryption, + useEncryption, (Function) ArrowFlightConnectionConfigImpl::useEncryption), Arguments.of( THREAD_POOL_SIZE, - RANDOM.nextInt(getRuntime().availableProcessors()), + threadPoolSize, + threadPoolSize, (Function) ArrowFlightConnectionConfigImpl::threadPoolSize), Arguments.of( CATALOG, "catalog", + "catalog", + (Function) + ArrowFlightConnectionConfigImpl::getCatalog), + Arguments.of( + CONNECT_TIMEOUT_MILLIS, + 5000, + Duration.ofMillis(5000), (Function) - ArrowFlightConnectionConfigImpl::getCatalog)); + ArrowFlightConnectionConfigImpl::getConnectTimeout)); } } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/FallbackFlightSqlProducer.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/FallbackFlightSqlProducer.java index 9aa257172c..670b9e3be0 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/FallbackFlightSqlProducer.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/FallbackFlightSqlProducer.java @@ -109,6 +109,16 @@ private FlightInfo getFlightInfo(FlightDescriptor descriptor, String query) { Location.forGrpcInsecure("localhost", 9999), Location.reuseConnection()) .build()); + } else if (query.equals("fallback with unresolvable")) { + endpoints = + Collections.singletonList( + FlightEndpoint.builder( + ticket, + // Inaccessible IP + // https://stackoverflow.com/questions/10456044/what-is-a-good-invalid-ip-address-to-use-for-unit-tests + Location.forGrpcInsecure("203.0.113.0", 9999), + Location.reuseConnection()) + .build()); } else { throw CallStatus.UNIMPLEMENTED.withDescription(query).toRuntimeException(); } From 437612c07e04fb0283010961a836de22521fa759 Mon Sep 17 00:00:00 2001 From: Ivan Chesnov Date: Wed, 9 Apr 2025 15:19:25 +0300 Subject: [PATCH 145/408] GH-87: [Vector] Add ExtensionWriter (#697) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Based on changes from https://github.com/apache/arrow/pull/41731. ## What's Changed Added writer ExtensionWriter with 3 methods: - write method for writing values from Extension holders; - writeExtensionType method for writing values (arguments is Object because we don't know exact type); - addExtensionTypeFactory method - because the exact vector and value type are unknown, the user should create their own extension type vector, write for it, and ExtensionTypeFactory, which should map the vector and writer. Closes #87. Co-authored-by: Finn Völkel --- .../templates/AbstractFieldWriter.java | 22 ++++ .../AbstractPromotableFieldWriter.java | 10 ++ .../main/codegen/templates/BaseWriter.java | 31 +++++ .../codegen/templates/PromotableWriter.java | 14 +++ .../main/codegen/templates/StructWriters.java | 26 ++++ .../codegen/templates/UnionListWriter.java | 23 ++++ .../codegen/templates/UnionMapWriter.java | 12 ++ .../main/codegen/templates/UnionWriter.java | 20 +++ .../impl/AbstractExtensionTypeWriter.java | 66 ++++++++++ .../impl/ExtensionTypeWriterFactory.java | 38 ++++++ .../complex/impl/UnionExtensionWriter.java | 79 ++++++++++++ .../vector/complex/writer/FieldWriter.java | 4 +- .../arrow/vector/holders/ExtensionHolder.java | 22 ++++ .../apache/arrow/vector/TestStructVector.java | 37 ++++++ .../org/apache/arrow/vector/UuidVector.java | 114 ++++++++++++++++++ .../complex/impl/TestPromotableWriter.java | 29 +++++ .../complex/impl/UuidWriterFactory.java | 31 +++++ .../vector/complex/impl/UuidWriterImpl.java | 47 ++++++++ .../complex/writer/TestSimpleWriter.java | 20 +++ .../arrow/vector/holder/UuidHolder.java | 23 ++++ .../vector/types/pojo/TestExtensionType.java | 70 +---------- .../arrow/vector/types/pojo/UuidType.java | 60 +++++++++ 22 files changed, 728 insertions(+), 70 deletions(-) create mode 100644 vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractExtensionTypeWriter.java create mode 100644 vector/src/main/java/org/apache/arrow/vector/complex/impl/ExtensionTypeWriterFactory.java create mode 100644 vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionExtensionWriter.java create mode 100644 vector/src/main/java/org/apache/arrow/vector/holders/ExtensionHolder.java create mode 100644 vector/src/test/java/org/apache/arrow/vector/UuidVector.java create mode 100644 vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java create mode 100644 vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java create mode 100644 vector/src/test/java/org/apache/arrow/vector/holder/UuidHolder.java create mode 100644 vector/src/test/java/org/apache/arrow/vector/types/pojo/UuidType.java diff --git a/vector/src/main/codegen/templates/AbstractFieldWriter.java b/vector/src/main/codegen/templates/AbstractFieldWriter.java index cc2cc618d8..ae5b97faef 100644 --- a/vector/src/main/codegen/templates/AbstractFieldWriter.java +++ b/vector/src/main/codegen/templates/AbstractFieldWriter.java @@ -107,6 +107,16 @@ public void endEntry() { throw new IllegalStateException(String.format("You tried to end a map entry when you are using a ValueWriter of type %s.", this.getClass().getSimpleName())); } + public void write(ExtensionHolder var1) { + this.fail("ExtensionType"); + } + public void writeExtension(Object var1) { + this.fail("ExtensionType"); + } + public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory var1) { + this.fail("ExtensionType"); + } + <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) /> @@ -241,6 +251,18 @@ public MapWriter map(String name, boolean keysSorted) { fail("Map"); return null; } + + @Override + public ExtensionWriter extension(String name, ArrowType arrowType) { + fail("Extension"); + return null; + } + + @Override + public ExtensionWriter extension(ArrowType arrowType) { + fail("Extension"); + return null; + } <#list vv.types as type><#list type.minor as minor> <#assign lowerName = minor.class?uncap_first /> <#if lowerName == "int" ><#assign lowerName = "integer" /> diff --git a/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java b/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java index 06cb235f7d..951edd5eee 100644 --- a/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java +++ b/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java @@ -293,6 +293,11 @@ public MapWriter map(boolean keysSorted) { return getWriter(MinorType.MAP, new ArrowType.Map(keysSorted)); } + @Override + public ExtensionWriter extension(ArrowType arrowType) { + return getWriter(MinorType.EXTENSIONTYPE).extension(arrowType); + } + @Override public StructWriter struct(String name) { return getWriter(MinorType.STRUCT).struct(name); @@ -318,6 +323,11 @@ public MapWriter map(String name, boolean keysSorted) { return getWriter(MinorType.STRUCT).map(name, keysSorted); } + @Override + public ExtensionWriter extension(String name, ArrowType arrowType) { + return getWriter(MinorType.EXTENSIONTYPE).extension(name, arrowType); + } + <#list vv.types as type><#list type.minor as minor> <#assign lowerName = minor.class?uncap_first /> <#if lowerName == "int" ><#assign lowerName = "integer" /> diff --git a/vector/src/main/codegen/templates/BaseWriter.java b/vector/src/main/codegen/templates/BaseWriter.java index e952d46f1f..78da7fddc3 100644 --- a/vector/src/main/codegen/templates/BaseWriter.java +++ b/vector/src/main/codegen/templates/BaseWriter.java @@ -61,6 +61,7 @@ public interface StructWriter extends BaseWriter { void copyReaderToField(String name, FieldReader reader); StructWriter struct(String name); + ExtensionWriter extension(String name, ArrowType arrowType); ListWriter list(String name); ListWriter listView(String name); MapWriter map(String name); @@ -79,6 +80,7 @@ public interface ListWriter extends BaseWriter { ListWriter listView(); MapWriter map(); MapWriter map(boolean keysSorted); + ExtensionWriter extension(ArrowType arrowType); void copyReader(FieldReader reader); <#list vv.types as type><#list type.minor as minor> @@ -101,6 +103,35 @@ public interface MapWriter extends ListWriter { MapWriter value(); } + public interface ExtensionWriter extends BaseWriter { + + /** + * Writes a null value. + */ + void writeNull(); + + /** + * Writes value from the given extension holder. + * + * @param holder the extension holder to write + */ + void write(ExtensionHolder holder); + + /** + * Writes the given extension type value. + * + * @param value the extension type value to write + */ + void writeExtension(Object value); + + /** + * Adds the given extension type factory. This factory allows configuring writer implementations for specific ExtensionTypeVector. + * + * @param factory the extension type factory to add + */ + void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory factory); + } + public interface ScalarWriter extends <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> ${name}Writer, BaseWriter {} diff --git a/vector/src/main/codegen/templates/PromotableWriter.java b/vector/src/main/codegen/templates/PromotableWriter.java index c0e686f317..8d7d57bb9d 100644 --- a/vector/src/main/codegen/templates/PromotableWriter.java +++ b/vector/src/main/codegen/templates/PromotableWriter.java @@ -285,6 +285,9 @@ protected void setWriter(ValueVector v) { case UNION: writer = new UnionWriter((UnionVector) vector, nullableStructWriterFactory); break; + case EXTENSIONTYPE: + writer = new UnionExtensionWriter((ExtensionTypeVector) vector); + break; default: writer = type.getNewFieldWriter(vector); break; @@ -316,6 +319,7 @@ protected boolean requiresArrowType(MinorType type) { || type == MinorType.MAP || type == MinorType.DURATION || type == MinorType.FIXEDSIZEBINARY + || type == MinorType.EXTENSIONTYPE || (type.name().startsWith("TIMESTAMP") && type.name().endsWith("TZ")); } @@ -536,6 +540,16 @@ public void writeLargeVarChar(String value) { getWriter(MinorType.LARGEVARCHAR).writeLargeVarChar(value); } + @Override + public void writeExtension(Object value) { + getWriter(MinorType.EXTENSIONTYPE).writeExtension(value); + } + + @Override + public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory factory) { + getWriter(MinorType.EXTENSIONTYPE).addExtensionTypeWriterFactory(factory); + } + @Override public void allocate() { getWriter().allocate(); diff --git a/vector/src/main/codegen/templates/StructWriters.java b/vector/src/main/codegen/templates/StructWriters.java index 3e6b9fd773..413f707c70 100644 --- a/vector/src/main/codegen/templates/StructWriters.java +++ b/vector/src/main/codegen/templates/StructWriters.java @@ -83,6 +83,9 @@ public class ${mode}StructWriter extends AbstractFieldWriter { fields.put(handleCase(child.getName()), writer); break; } + case EXTENSIONTYPE: + extension(child.getName(), child.getType()); + break; case UNION: FieldType fieldType = new FieldType(addVectorAsNullable, MinorType.UNION.getType(), null, null); UnionWriter writer = new UnionWriter(container.addOrGet(child.getName(), fieldType, UnionVector.class), getNullableStructWriterFactory()); @@ -159,6 +162,29 @@ public StructWriter struct(String name) { return writer; } + @Override + public ExtensionWriter extension(String name, ArrowType arrowType) { + String finalName = handleCase(name); + FieldWriter writer = fields.get(finalName); + if(writer == null){ + int vectorCount=container.size(); + FieldType fieldType = new FieldType(addVectorAsNullable, arrowType, null, null); + ExtensionTypeVector vector = container.addOrGet(name, fieldType, ExtensionTypeVector.class); + writer = new PromotableWriter(vector, container, getNullableStructWriterFactory()); + if(vectorCount != container.size()) { + writer.allocate(); + } + writer.setPosition(idx()); + fields.put(finalName, writer); + } else { + if (writer instanceof PromotableWriter) { + // ensure writers are initialized + ((PromotableWriter)writer).getWriter(MinorType.EXTENSIONTYPE, arrowType); + } + } + return (ExtensionWriter) writer; + } + @Override public void close() throws Exception { clear(); diff --git a/vector/src/main/codegen/templates/UnionListWriter.java b/vector/src/main/codegen/templates/UnionListWriter.java index 3962e1d073..9424533f29 100644 --- a/vector/src/main/codegen/templates/UnionListWriter.java +++ b/vector/src/main/codegen/templates/UnionListWriter.java @@ -201,6 +201,17 @@ public MapWriter map(String name, boolean keysSorted) { return mapWriter; } + @Override + public ExtensionWriter extension(ArrowType arrowType) { + writer.extension(arrowType); + return writer; + } + @Override + public ExtensionWriter extension(String name, ArrowType arrowType) { + ExtensionWriter extensionWriter = writer.extension(name, arrowType); + return extensionWriter; + } + <#if listName == "LargeList"> @Override public void startList() { @@ -323,6 +334,18 @@ public void writeNull() { } } + @Override + public void writeExtension(Object value) { + writer.writeExtension(value); + } + @Override + public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory var1) { + writer.addExtensionTypeWriterFactory(var1); + } + public void write(ExtensionHolder var1) { + writer.write(var1); + } + <#list vv.types as type> <#list type.minor as minor> <#assign name = minor.class?cap_first /> diff --git a/vector/src/main/codegen/templates/UnionMapWriter.java b/vector/src/main/codegen/templates/UnionMapWriter.java index 90b55cb65e..8b2f091215 100644 --- a/vector/src/main/codegen/templates/UnionMapWriter.java +++ b/vector/src/main/codegen/templates/UnionMapWriter.java @@ -231,4 +231,16 @@ public MapWriter map() { return super.map(); } } + + @Override + public ExtensionWriter extension(ArrowType type) { + switch (mode) { + case KEY: + return entryWriter.extension(MapVector.KEY_NAME, type); + case VALUE: + return entryWriter.extension(MapVector.VALUE_NAME, type); + default: + return super.extension(type); + } + } } diff --git a/vector/src/main/codegen/templates/UnionWriter.java b/vector/src/main/codegen/templates/UnionWriter.java index bfe97e2770..272edab17c 100644 --- a/vector/src/main/codegen/templates/UnionWriter.java +++ b/vector/src/main/codegen/templates/UnionWriter.java @@ -213,6 +213,10 @@ public MapWriter asMap(ArrowType arrowType) { return getMapWriter(arrowType); } + private ExtensionWriter getExtensionWriter(ArrowType arrowType) { + throw new UnsupportedOperationException("ExtensionTypes are not supported yet."); + } + BaseWriter getWriter(MinorType minorType) { return getWriter(minorType, null); } @@ -227,6 +231,8 @@ BaseWriter getWriter(MinorType minorType, ArrowType arrowType) { return getListViewWriter(); case MAP: return getMapWriter(arrowType); + case EXTENSIONTYPE: + return getExtensionWriter(arrowType); <#list vv.types as type> <#list type.minor as minor> <#assign name = minor.class?cap_first /> @@ -460,6 +466,20 @@ public MapWriter map(String name, boolean keysSorted) { return getStructWriter().map(name, keysSorted); } + @Override + public ExtensionWriter extension(ArrowType arrowType) { + data.setType(idx(), MinorType.EXTENSIONTYPE); + getListWriter().setPosition(idx()); + return getListWriter().extension(arrowType); + } + + @Override + public ExtensionWriter extension(String name, ArrowType arrowType) { + data.setType(idx(), MinorType.EXTENSIONTYPE); + getStructWriter().setPosition(idx()); + return getStructWriter().extension(name, arrowType); + } + <#list vv.types as type><#list type.minor as minor> <#assign lowerName = minor.class?uncap_first /> <#if lowerName == "int" ><#assign lowerName = "integer" /> diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractExtensionTypeWriter.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractExtensionTypeWriter.java new file mode 100644 index 0000000000..fccff6c21f --- /dev/null +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractExtensionTypeWriter.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.complex.impl; + +import org.apache.arrow.vector.ExtensionTypeVector; +import org.apache.arrow.vector.types.pojo.Field; + +/** + * Base {@link AbstractFieldWriter} class for an {@link + * org.apache.arrow.vector.ExtensionTypeVector}. + * + * @param a specific {@link ExtensionTypeVector}. + */ +public class AbstractExtensionTypeWriter + extends AbstractFieldWriter { + protected final T vector; + + public AbstractExtensionTypeWriter(T vector) { + this.vector = vector; + } + + @Override + public Field getField() { + return this.vector.getField(); + } + + @Override + public int getValueCapacity() { + return this.vector.getValueCapacity(); + } + + @Override + public void allocate() { + this.vector.allocateNew(); + } + + @Override + public void close() { + this.vector.close(); + } + + @Override + public void clear() { + this.vector.clear(); + } + + @Override + public void writeNull() { + this.vector.setNull(getPosition()); + this.vector.setValueCount(getPosition() + 1); + } +} diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/ExtensionTypeWriterFactory.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/ExtensionTypeWriterFactory.java new file mode 100644 index 0000000000..09f0314c5f --- /dev/null +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/ExtensionTypeWriterFactory.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.complex.impl; + +import org.apache.arrow.vector.ExtensionTypeVector; +import org.apache.arrow.vector.complex.writer.FieldWriter; + +/** + * A factory interface for creating instances of {@link ExtensionTypeWriter}. This factory allows + * configuring writer implementations for specific {@link ExtensionTypeVector}. + * + * @param the type of writer implementation for a specific {@link ExtensionTypeVector}. + */ +public interface ExtensionTypeWriterFactory { + + /** + * Returns an instance of the writer implementation for the given {@link ExtensionTypeVector}. + * + * @param vector the {@link ExtensionTypeVector} for which the writer implementation is to be + * returned. + * @return an instance of the writer implementation for the given {@link ExtensionTypeVector}. + */ + T getWriterImpl(ExtensionTypeVector vector); +} diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionExtensionWriter.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionExtensionWriter.java new file mode 100644 index 0000000000..d341384bd9 --- /dev/null +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionExtensionWriter.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.complex.impl; + +import org.apache.arrow.vector.ExtensionTypeVector; +import org.apache.arrow.vector.complex.writer.FieldWriter; +import org.apache.arrow.vector.holders.ExtensionHolder; +import org.apache.arrow.vector.types.pojo.Field; + +public class UnionExtensionWriter extends AbstractFieldWriter { + protected ExtensionTypeVector vector; + protected FieldWriter writer; + + public UnionExtensionWriter(ExtensionTypeVector vector) { + this.vector = vector; + } + + @Override + public void allocate() { + vector.allocateNew(); + } + + @Override + public void clear() { + vector.clear(); + } + + @Override + public int getValueCapacity() { + return vector.getValueCapacity(); + } + + @Override + public Field getField() { + return vector.getField(); + } + + @Override + public void close() throws Exception { + vector.close(); + } + + @Override + public void writeExtension(Object var1) { + this.writer.writeExtension(var1); + } + + @Override + public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory factory) { + this.writer = factory.getWriterImpl(vector); + this.writer.setPosition(idx()); + } + + public void write(ExtensionHolder holder) { + this.writer.write(holder); + } + + @Override + public void setPosition(int index) { + super.setPosition(index); + if (this.writer != null) { + this.writer.setPosition(index); + } + } +} diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/writer/FieldWriter.java b/vector/src/main/java/org/apache/arrow/vector/complex/writer/FieldWriter.java index 949eb35d8e..51bf106685 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/writer/FieldWriter.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/writer/FieldWriter.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.vector.complex.writer; +import org.apache.arrow.vector.complex.writer.BaseWriter.ExtensionWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.ScalarWriter; @@ -25,7 +26,8 @@ * Composite of all writer types. Writers are convenience classes for incrementally adding values to * {@linkplain org.apache.arrow.vector.ValueVector}s. */ -public interface FieldWriter extends StructWriter, ListWriter, MapWriter, ScalarWriter { +public interface FieldWriter + extends StructWriter, ListWriter, MapWriter, ScalarWriter, ExtensionWriter { void allocate(); void clear(); diff --git a/vector/src/main/java/org/apache/arrow/vector/holders/ExtensionHolder.java b/vector/src/main/java/org/apache/arrow/vector/holders/ExtensionHolder.java new file mode 100644 index 0000000000..fc7ed85878 --- /dev/null +++ b/vector/src/main/java/org/apache/arrow/vector/holders/ExtensionHolder.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.holders; + +/** Base {@link ValueHolder} class for a {@link org.apache.arrow.vector.ExtensionTypeVector}. */ +public abstract class ExtensionHolder implements ValueHolder { + public int isSet; +} diff --git a/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java b/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java index 4ef0fbe2d9..d40af9ae89 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java @@ -26,6 +26,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.complex.AbstractStructVector; import org.apache.arrow.vector.complex.ListVector; @@ -37,9 +38,11 @@ import org.apache.arrow.vector.holders.ComplexHolder; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.Types.MinorType; +import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.ArrowType.Struct; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.UuidType; import org.apache.arrow.vector.util.TransferPair; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -336,6 +339,40 @@ public void testGetTransferPairWithFieldAndCallBack() { } } + @Test + public void testStructVectorWithExtensionTypes() { + UuidType uuidType = new UuidType(); + Field uuidField = new Field("struct_child", FieldType.nullable(uuidType), null); + Field structField = + new Field("struct", FieldType.nullable(new ArrowType.Struct()), List.of(uuidField)); + StructVector s1 = new StructVector(structField, allocator, null); + StructVector s2 = (StructVector) structField.createVector(allocator); + s1.close(); + s2.close(); + } + + @Test + public void testStructVectorTransferPairWithExtensionType() { + UuidType uuidType = new UuidType(); + Field uuidField = new Field("uuid_child", FieldType.nullable(uuidType), null); + Field structField = + new Field("struct", FieldType.nullable(new ArrowType.Struct()), List.of(uuidField)); + + StructVector s1 = (StructVector) structField.createVector(allocator); + UuidVector uuidVector = + s1.addOrGet("uuid_child", FieldType.nullable(uuidType), UuidVector.class); + s1.setValueCount(1); + uuidVector.set(0, new UUID(1, 2)); + s1.setIndexDefined(0); + + TransferPair tp = s1.getTransferPair(structField, allocator); + final StructVector toVector = (StructVector) tp.getTo(); + assertEquals(s1.getField(), toVector.getField()); + + s1.close(); + toVector.close(); + } + private StructVector simpleStructVector(String name, BufferAllocator allocator) { final String INT_COL = "struct_int_child"; final String FLT_COL = "struct_flt_child"; diff --git a/vector/src/test/java/org/apache/arrow/vector/UuidVector.java b/vector/src/test/java/org/apache/arrow/vector/UuidVector.java new file mode 100644 index 0000000000..5c90d45f60 --- /dev/null +++ b/vector/src/test/java/org/apache/arrow/vector/UuidVector.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector; + +import java.nio.ByteBuffer; +import java.util.UUID; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.util.hash.ArrowBufHasher; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.UuidType; +import org.apache.arrow.vector.util.TransferPair; + +public class UuidVector extends ExtensionTypeVector + implements ValueIterableVector { + private final Field field; + + public UuidVector( + String name, BufferAllocator allocator, FixedSizeBinaryVector underlyingVector) { + super(name, allocator, underlyingVector); + this.field = new Field(name, FieldType.nullable(new UuidType()), null); + } + + public UuidVector(String name, BufferAllocator allocator) { + super(name, allocator, new FixedSizeBinaryVector(name, allocator, 16)); + this.field = new Field(name, FieldType.nullable(new UuidType()), null); + } + + @Override + public UUID getObject(int index) { + final ByteBuffer bb = ByteBuffer.wrap(getUnderlyingVector().getObject(index)); + return new UUID(bb.getLong(), bb.getLong()); + } + + @Override + public int hashCode(int index) { + return hashCode(index, null); + } + + @Override + public int hashCode(int index, ArrowBufHasher hasher) { + return getUnderlyingVector().hashCode(index, hasher); + } + + public void set(int index, UUID uuid) { + ByteBuffer bb = ByteBuffer.allocate(16); + bb.putLong(uuid.getMostSignificantBits()); + bb.putLong(uuid.getLeastSignificantBits()); + getUnderlyingVector().set(index, bb.array()); + } + + @Override + public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { + getUnderlyingVector() + .copyFromSafe(fromIndex, thisIndex, ((UuidVector) from).getUnderlyingVector()); + } + + @Override + public Field getField() { + return field; + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((UuidVector) to); + } + + public void setSafe(int index, byte[] value) { + getUnderlyingVector().setIndexDefined(index); + getUnderlyingVector().setSafe(index, value); + } + + public class TransferImpl implements TransferPair { + UuidVector to; + ValueVector targetUnderlyingVector; + TransferPair tp; + + public TransferImpl(UuidVector to) { + this.to = to; + targetUnderlyingVector = this.to.getUnderlyingVector(); + tp = getUnderlyingVector().makeTransferPair(targetUnderlyingVector); + } + + public UuidVector getTo() { + return this.to; + } + + public void transfer() { + tp.transfer(); + } + + public void splitAndTransfer(int startIndex, int length) { + tp.splitAndTransfer(startIndex, length); + } + + public void copyValueSafe(int fromIndex, int toIndex) { + tp.copyValueSafe(fromIndex, toIndex); + } + } +} diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java index a791e55135..1556852c5a 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java @@ -26,12 +26,14 @@ import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; import java.util.Objects; +import java.util.UUID; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.DecimalVector; import org.apache.arrow.vector.DirtyRootAllocator; import org.apache.arrow.vector.LargeVarBinaryVector; import org.apache.arrow.vector.LargeVarCharVector; +import org.apache.arrow.vector.UuidVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.complex.ListVector; @@ -52,6 +54,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.UuidType; import org.apache.arrow.vector.util.DecimalUtility; import org.apache.arrow.vector.util.Text; import org.junit.jupiter.api.AfterEach; @@ -776,4 +779,30 @@ public void testPromoteToUnionFromDecimal() throws Exception { assertEquals(1, intHolder.value); } } + + @Test + public void testExtensionType() throws Exception { + try (final NonNullableStructVector container = + NonNullableStructVector.empty(EMPTY_SCHEMA_PATH, allocator); + final UuidVector v = + container.addOrGet("uuid", FieldType.nullable(new UuidType()), UuidVector.class); + final PromotableWriter writer = new PromotableWriter(v, container)) { + UUID u1 = UUID.randomUUID(); + UUID u2 = UUID.randomUUID(); + container.allocateNew(); + container.setValueCount(1); + writer.addExtensionTypeWriterFactory(new UuidWriterFactory()); + + writer.setPosition(0); + writer.writeExtension(u1); + writer.setPosition(1); + writer.writeExtension(u2); + + container.setValueCount(2); + + UuidVector uuidVector = (UuidVector) container.getChild("uuid"); + assertEquals(u1, uuidVector.getObject(0)); + assertEquals(u2, uuidVector.getObject(1)); + } + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java new file mode 100644 index 0000000000..1b1bf4e6e4 --- /dev/null +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.complex.impl; + +import org.apache.arrow.vector.ExtensionTypeVector; +import org.apache.arrow.vector.UuidVector; + +public class UuidWriterFactory implements ExtensionTypeWriterFactory { + + @Override + public AbstractFieldWriter getWriterImpl(ExtensionTypeVector extensionTypeVector) { + if (extensionTypeVector instanceof UuidVector) { + return new UuidWriterImpl((UuidVector) extensionTypeVector); + } + return null; + } +} diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java new file mode 100644 index 0000000000..68029b1df5 --- /dev/null +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.complex.impl; + +import java.nio.ByteBuffer; +import java.util.UUID; +import org.apache.arrow.vector.UuidVector; +import org.apache.arrow.vector.holder.UuidHolder; +import org.apache.arrow.vector.holders.ExtensionHolder; + +public class UuidWriterImpl extends AbstractExtensionTypeWriter { + + public UuidWriterImpl(UuidVector vector) { + super(vector); + } + + @Override + public void writeExtension(Object value) { + UUID uuid = (UUID) value; + ByteBuffer bb = ByteBuffer.allocate(16); + bb.putLong(uuid.getMostSignificantBits()); + bb.putLong(uuid.getLeastSignificantBits()); + vector.setSafe(getPosition(), bb.array()); + vector.setValueCount(getPosition() + 1); + } + + @Override + public void write(ExtensionHolder holder) { + UuidHolder uuidHolder = (UuidHolder) holder; + vector.setSafe(getPosition(), uuidHolder.value); + vector.setValueCount(getPosition() + 1); + } +} diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java index 5bb5962704..bf1b9b0dfa 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java @@ -20,16 +20,20 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import java.nio.ByteBuffer; +import java.util.UUID; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.LargeVarBinaryVector; import org.apache.arrow.vector.LargeVarCharVector; +import org.apache.arrow.vector.UuidVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.complex.impl.LargeVarBinaryWriterImpl; import org.apache.arrow.vector.complex.impl.LargeVarCharWriterImpl; +import org.apache.arrow.vector.complex.impl.UuidWriterImpl; import org.apache.arrow.vector.complex.impl.VarBinaryWriterImpl; import org.apache.arrow.vector.complex.impl.VarCharWriterImpl; +import org.apache.arrow.vector.holder.UuidHolder; import org.apache.arrow.vector.util.Text; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -184,4 +188,20 @@ public void testWriteTextToLargeVarChar() throws Exception { assertEquals(input, result); } } + + @Test + public void testWriteToExtensionVector() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator); + UuidWriterImpl writer = new UuidWriterImpl(vector)) { + UUID uuid = UUID.randomUUID(); + ByteBuffer bb = ByteBuffer.allocate(16); + bb.putLong(uuid.getMostSignificantBits()); + bb.putLong(uuid.getLeastSignificantBits()); + UuidHolder holder = new UuidHolder(); + holder.value = bb.array(); + writer.write(holder); + UUID result = vector.getObject(0); + assertEquals(uuid, result); + } + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/holder/UuidHolder.java b/vector/src/test/java/org/apache/arrow/vector/holder/UuidHolder.java new file mode 100644 index 0000000000..207b0951a7 --- /dev/null +++ b/vector/src/test/java/org/apache/arrow/vector/holder/UuidHolder.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.holder; + +import org.apache.arrow.vector.holders.ExtensionHolder; + +public class UuidHolder extends ExtensionHolder { + public byte[] value; +} diff --git a/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java b/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java index 8f54a6e5d7..d24708d66c 100644 --- a/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java +++ b/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java @@ -41,6 +41,7 @@ import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.FixedSizeBinaryVector; import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.UuidVector; import org.apache.arrow.vector.ValueIterableVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.compare.Range; @@ -295,75 +296,6 @@ public void testVectorCompare() { } } - static class UuidType extends ExtensionType { - - @Override - public ArrowType storageType() { - return new ArrowType.FixedSizeBinary(16); - } - - @Override - public String extensionName() { - return "uuid"; - } - - @Override - public boolean extensionEquals(ExtensionType other) { - return other instanceof UuidType; - } - - @Override - public ArrowType deserialize(ArrowType storageType, String serializedData) { - if (!storageType.equals(storageType())) { - throw new UnsupportedOperationException( - "Cannot construct UuidType from underlying type " + storageType); - } - return new UuidType(); - } - - @Override - public String serialize() { - return ""; - } - - @Override - public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator) { - return new UuidVector(name, allocator, new FixedSizeBinaryVector(name, allocator, 16)); - } - } - - static class UuidVector extends ExtensionTypeVector - implements ValueIterableVector { - - public UuidVector( - String name, BufferAllocator allocator, FixedSizeBinaryVector underlyingVector) { - super(name, allocator, underlyingVector); - } - - @Override - public UUID getObject(int index) { - final ByteBuffer bb = ByteBuffer.wrap(getUnderlyingVector().getObject(index)); - return new UUID(bb.getLong(), bb.getLong()); - } - - @Override - public int hashCode(int index) { - return hashCode(index, null); - } - - @Override - public int hashCode(int index, ArrowBufHasher hasher) { - return getUnderlyingVector().hashCode(index, hasher); - } - - public void set(int index, UUID uuid) { - ByteBuffer bb = ByteBuffer.allocate(16); - bb.putLong(uuid.getMostSignificantBits()); - bb.putLong(uuid.getLeastSignificantBits()); - getUnderlyingVector().set(index, bb.array()); - } - } - static class LocationType extends ExtensionType { @Override diff --git a/vector/src/test/java/org/apache/arrow/vector/types/pojo/UuidType.java b/vector/src/test/java/org/apache/arrow/vector/types/pojo/UuidType.java new file mode 100644 index 0000000000..5e2bd8881b --- /dev/null +++ b/vector/src/test/java/org/apache/arrow/vector/types/pojo/UuidType.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.types.pojo; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.FixedSizeBinaryVector; +import org.apache.arrow.vector.UuidVector; +import org.apache.arrow.vector.types.pojo.ArrowType.ExtensionType; + +public class UuidType extends ExtensionType { + + @Override + public ArrowType storageType() { + return new ArrowType.FixedSizeBinary(16); + } + + @Override + public String extensionName() { + return "uuid"; + } + + @Override + public boolean extensionEquals(ExtensionType other) { + return other instanceof UuidType; + } + + @Override + public ArrowType deserialize(ArrowType storageType, String serializedData) { + if (!storageType.equals(storageType())) { + throw new UnsupportedOperationException( + "Cannot construct UuidType from underlying type " + storageType); + } + return new UuidType(); + } + + @Override + public String serialize() { + return ""; + } + + @Override + public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator) { + return new UuidVector(name, allocator, new FixedSizeBinaryVector(name, allocator, 16)); + } +} From 3e135f4f24551cd932c7ac0646ca58aa99c07a90 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 14 Apr 2025 11:37:26 +0900 Subject: [PATCH 146/408] GH-708: Enable GitHub discussions (#710) ## What's Changed Enable GitHub Discussions like other Arrow repositories. Fixes #708. --------- Co-authored-by: Sutou Kouhei --- .asf.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.asf.yaml b/.asf.yaml index ead2581149..13bd59c224 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -29,12 +29,14 @@ github: rebase: false squash: true features: + discussions: true issues: true protected_branches: main: required_linear_history: true notifications: commits: commits@arrow.apache.org + discussions: user@arrow.apache.org issues_status: issues@arrow.apache.org issues_comment: github@arrow.apache.org pullrequests: github@arrow.apache.org From 74e8981d5ba0646f2ee1dbc99364766650ad084f Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Mon, 14 Apr 2025 07:46:56 +0200 Subject: [PATCH 147/408] GH-709: Correct length calculation of value buffers of variable-sized arrays (#707) ## What's Changed For variable-size binary layout arrays, BufferImportTypeVisitor currently derives the length of the value buffer by calculating the difference between the last and first offset. When the first offset is not zero, this is actually incorrect and leads to out of bounds errors when attempting to read values from the imported array. Instead, BufferImportTypeVisitor should simply use the last offset value as the length of the value buffer. This PR makes that change. Just FYI, I bumped into this issue when attempting to import an array originating from DataFusion. A test query of the form `SELECT column1 FROM VALUES ('a'), ('b'), ('c'), ('d') LIMIT 2 OFFSET 1;` returns a slice of the full set of values. The values buffer contains all the original values, and the offsets buffer contains 1 and 2 as values to handle the offset from the query. Closes #709. --- .../org/apache/arrow/c/BufferImportTypeVisitor.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/c/src/main/java/org/apache/arrow/c/BufferImportTypeVisitor.java b/c/src/main/java/org/apache/arrow/c/BufferImportTypeVisitor.java index 5ca398c9f9..10f690fc87 100644 --- a/c/src/main/java/org/apache/arrow/c/BufferImportTypeVisitor.java +++ b/c/src/main/java/org/apache/arrow/c/BufferImportTypeVisitor.java @@ -228,9 +228,8 @@ public List visit(ArrowType.Utf8 type) { type, start, end); - final int len = end - start; offsets.getReferenceManager().retain(); - return Arrays.asList(maybeImportBitmap(type), offsets, importData(type, len)); + return Arrays.asList(maybeImportBitmap(type), offsets, importData(type, end)); } } @@ -279,9 +278,8 @@ public List visit(ArrowType.LargeUtf8 type) { type, start, end); - final long len = end - start; offsets.getReferenceManager().retain(); - return Arrays.asList(maybeImportBitmap(type), offsets, importData(type, len)); + return Arrays.asList(maybeImportBitmap(type), offsets, importData(type, end)); } } @@ -296,9 +294,8 @@ public List visit(ArrowType.Binary type) { type, start, end); - final int len = end - start; offsets.getReferenceManager().retain(); - return Arrays.asList(maybeImportBitmap(type), offsets, importData(type, len)); + return Arrays.asList(maybeImportBitmap(type), offsets, importData(type, end)); } } @@ -320,9 +317,8 @@ public List visit(ArrowType.LargeBinary type) { type, start, end); - final long len = end - start; offsets.getReferenceManager().retain(); - return Arrays.asList(maybeImportBitmap(type), offsets, importData(type, len)); + return Arrays.asList(maybeImportBitmap(type), offsets, importData(type, end)); } } From 4b4d92851c8ecba1876a8f452279122e88e5ff8e Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 14 Apr 2025 21:47:30 +0900 Subject: [PATCH 148/408] MINOR: Don't uninstall Flatbuffers in CI (#711) ## What's Changed There is no more need after https://github.com/apache/arrow/commit/32641ecfd97e302ea8119b1147d77aced96d097b Also, disable building tests to avoid issues with Boost/Homebrew. They aren't executed anyways. --- .github/workflows/rc.yml | 5 ----- ci/scripts/jni_macos_build.sh | 2 +- ci/scripts/jni_manylinux_build.sh | 2 +- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index e039306ec3..5b78cc9395 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -261,11 +261,6 @@ jobs: # bundled Protobuf. brew uninstall protobuf - # We need Flatbuffers 24, not the latest version - # Homebrew does not offer older versions, so remove the Homebrew - # package and rely on Arrow using a bundled version instead - brew uninstall flatbuffers - brew bundle --file=Brewfile - name: Prepare ccache run: | diff --git a/ci/scripts/jni_macos_build.sh b/ci/scripts/jni_macos_build.sh index 65b255e2dc..f7543b6f7a 100755 --- a/ci/scripts/jni_macos_build.sh +++ b/ci/scripts/jni_macos_build.sh @@ -61,7 +61,7 @@ github_actions_group_begin "Building Arrow C++ libraries" install_dir="${build_dir}/cpp-install" : "${ARROW_ACERO:=ON}" export ARROW_ACERO -: "${ARROW_BUILD_TESTS:=ON}" +: "${ARROW_BUILD_TESTS:=OFF}" export ARROW_BUILD_TESTS : "${ARROW_DATASET:=ON}" export ARROW_DATASET diff --git a/ci/scripts/jni_manylinux_build.sh b/ci/scripts/jni_manylinux_build.sh index 148d2e02f6..a34ec0f420 100755 --- a/ci/scripts/jni_manylinux_build.sh +++ b/ci/scripts/jni_manylinux_build.sh @@ -57,7 +57,7 @@ devtoolset_version="$(rpm -qa "devtoolset-*-gcc" --queryformat '%{VERSION}' | gr devtoolset_include_cpp="/opt/rh/devtoolset-${devtoolset_version}/root/usr/include/c++/${devtoolset_version}" : "${ARROW_ACERO:=ON}" export ARROW_ACERO -: "${ARROW_BUILD_TESTS:=ON}" +: "${ARROW_BUILD_TESTS:=OFF}" export ARROW_BUILD_TESTS : "${ARROW_DATASET:=ON}" export ARROW_DATASET From 4af464cc80361fdbfcb2024d3ff848b76c27524f Mon Sep 17 00:00:00 2001 From: Gabor Szadovszky Date: Wed, 23 Apr 2025 01:38:35 +0200 Subject: [PATCH 149/408] GH-721: Allow using 1GB+ data buffers in variable width vectors (#722) ## What's Changed Allow actually reaching MAX_BUFFER_SIZE at reallocating variable width vectors instead of exceeding it calculating the next power of 2. For unit testing the maximum allocation size has be increased to 2MB - 1byte to simulate the default maximum behavior. Due to this change needed some updates in existing unit tests because of the round ups used at calculating the required buffer sizes. Closes #721. --- pom.xml | 4 ++-- .../arrow/vector/BaseVariableWidthVector.java | 7 +++++-- .../arrow/vector/BaseVariableWidthViewVector.java | 14 ++++++++++---- .../org/apache/arrow/vector/TestValueVector.java | 2 +- .../org/apache/arrow/vector/TestVectorReAlloc.java | 12 ++++++++++++ .../arrow/vector/util/TestVectorAppender.java | 11 ++++++++++- 6 files changed, 40 insertions(+), 10 deletions(-) diff --git a/pom.xml b/pom.xml index cb189dc8f9..ab988c7d05 100644 --- a/pom.xml +++ b/pom.xml @@ -327,8 +327,8 @@ under the License. true UTC - 1048576 + which in turn can cause OOM. Using 2MB - 1byte to simulate the defaul limit of 2^31 - 1 bytes. --> + 2097151 false diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java index 7b8d2cdfda..1609e64ca5 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java @@ -571,10 +571,13 @@ public void reallocDataBuffer(long desiredAllocSize) { return; } - final long newAllocationSize = CommonUtil.nextPowerOfTwo(desiredAllocSize); + final long newAllocationSize = + Math.min(CommonUtil.nextPowerOfTwo(desiredAllocSize), MAX_BUFFER_SIZE); assert newAllocationSize >= 1; - checkDataBufferSize(newAllocationSize); + if (newAllocationSize < desiredAllocSize) { + checkDataBufferSize(desiredAllocSize); + } final ArrowBuf newBuf = allocator.buffer(newAllocationSize); newBuf.setBytes(0, valueBuffer, 0, valueBuffer.capacity()); diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java index e0e16762f2..beda91dc3f 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java @@ -550,15 +550,18 @@ public void reallocViewBuffer(long desiredAllocSize) { if (desiredAllocSize == 0) { return; } - long newAllocationSize = CommonUtil.nextPowerOfTwo(desiredAllocSize); + long newAllocationSize = Math.min(CommonUtil.nextPowerOfTwo(desiredAllocSize), MAX_BUFFER_SIZE); assert newAllocationSize >= 1; - checkDataBufferSize(newAllocationSize); // for each set operation, we have to allocate 16 bytes // here we are adjusting the desired allocation-based allocation size // to align with the 16bytes requirement. newAllocationSize = roundUpToMultipleOf16(newAllocationSize); + if (newAllocationSize < desiredAllocSize) { + checkDataBufferSize(desiredAllocSize); + } + final ArrowBuf newBuf = allocator.buffer(newAllocationSize); newBuf.setBytes(0, viewBuffer, 0, viewBuffer.capacity()); @@ -587,10 +590,13 @@ public void reallocViewDataBuffer(long desiredAllocSize) { return; } - final long newAllocationSize = CommonUtil.nextPowerOfTwo(desiredAllocSize); + final long newAllocationSize = + Math.min(CommonUtil.nextPowerOfTwo(desiredAllocSize), MAX_BUFFER_SIZE); assert newAllocationSize >= 1; - checkDataBufferSize(newAllocationSize); + if (newAllocationSize < desiredAllocSize) { + checkDataBufferSize(desiredAllocSize); + } final ArrowBuf newBuf = allocator.buffer(newAllocationSize); dataBuffers.add(newBuf); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index 83e470ae25..daec331831 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -95,7 +95,7 @@ public void init() { private static final byte[] STR5 = "EEE5".getBytes(utf8Charset); private static final byte[] STR6 = "FFFFF6".getBytes(utf8Charset); private static final int MAX_VALUE_COUNT = - (int) (Integer.getInteger("arrow.vector.max_allocation_bytes", Integer.MAX_VALUE) / 7); + (int) (Integer.getInteger("arrow.vector.max_allocation_bytes", Integer.MAX_VALUE) / 9); private static final int MAX_VALUE_COUNT_8BYTE = (int) (MAX_VALUE_COUNT / 2); @AfterEach diff --git a/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java index f5ec42c71c..bc47150376 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java @@ -24,6 +24,7 @@ import java.nio.charset.StandardCharsets; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.memory.util.CommonUtil; import org.apache.arrow.vector.complex.DenseUnionVector; import org.apache.arrow.vector.complex.FixedSizeListVector; import org.apache.arrow.vector.complex.ListVector; @@ -222,6 +223,17 @@ public void testVariableAllocateAfterReAlloc() throws Exception { } } + @Test + public void testVariableReAllocAbove1GB() throws Exception { + try (final VarCharVector vector = new VarCharVector("", allocator)) { + long desiredSizeAboveLastPowerOf2 = + CommonUtil.nextPowerOfTwo(BaseVariableWidthVector.MAX_ALLOCATION_SIZE) / 2 + 1; + vector.reallocDataBuffer(desiredSizeAboveLastPowerOf2); + + assertTrue(vector.getDataBuffer().capacity() >= desiredSizeAboveLastPowerOf2); + } + } + @Test public void testLargeVariableAllocateAfterReAlloc() throws Exception { try (final LargeVarCharVector vector = new LargeVarCharVector("", allocator)) { diff --git a/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java b/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java index e1b3889d85..4ee9630a4d 100644 --- a/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java +++ b/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java @@ -28,6 +28,7 @@ import java.util.stream.Stream; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.memory.util.CommonUtil; import org.apache.arrow.vector.BaseValueVector; import org.apache.arrow.vector.BaseVariableWidthViewVector; import org.apache.arrow.vector.BigIntVector; @@ -309,7 +310,15 @@ public void testAppendEmptyVariableWidthVector() { @Test public void testAppendLargeAndSmallVariableVectorsWithinLimit() { - int sixteenthOfMaxAllocation = Math.toIntExact(BaseValueVector.MAX_ALLOCATION_SIZE / 16); + // Using the max power of 2 allocation size to avoid hitting the max limit at round ups + long maxPowerOfTwoAllocationSize = + CommonUtil.nextPowerOfTwo(BaseValueVector.MAX_ALLOCATION_SIZE); + if (maxPowerOfTwoAllocationSize > BaseValueVector.MAX_ALLOCATION_SIZE) { + maxPowerOfTwoAllocationSize = + CommonUtil.nextPowerOfTwo(BaseValueVector.MAX_ALLOCATION_SIZE / 2); + } + + int sixteenthOfMaxAllocation = Math.toIntExact(maxPowerOfTwoAllocationSize / 16); try (VarCharVector target = makeVarCharVec(1, sixteenthOfMaxAllocation); VarCharVector delta = makeVarCharVec(sixteenthOfMaxAllocation, 1)) { new VectorAppender(delta).visit(target, null); From d2465c3fec94ff25a966490827b57b5eafb1b637 Mon Sep 17 00:00:00 2001 From: Martin Traverse Date: Wed, 23 Apr 2025 02:19:24 +0100 Subject: [PATCH 150/408] GH-698: Improve and fix Avro read consumers (#718) ## What's Changed This PR relates to #698 and is the second in a series intended to provide full Avro read / write support in native Java. It adds round-trip tests for both schemas (Arrow schema -> Avro -> Arrow) and data (Arrow VSR -> Avro block -> Arrow VSR). It also adds a number of fixes and improvements to the Avro Consumers so that data arrives back in its original form after a round trip. The main changes are: * Added a top level method in AvroToArrow to convert Avro schema directly to Arrow schema (this may exist elsewhere, but is needed to provide an API that matches the logic of this implementation) * Avro unions of [ type, null ] or [ null, type ] now have special handling, these are interpreted as a single nullable type rather than a union. Setting legacyMode = false in the AvroToArrowConfig object is required to enable this behaviour, otherwise unions are interpreted literally. Unions with more than 2 elements are always interpreted literally (but, per #108, in practice Java's current Union implementation is probably not usable with Avro atm). * Added support for new logical types (decimal 256, timestamp nano and 3 local timestamp types) * Existing timestamp-mills and timestamp-micros times now interpreted as zone-aware (previously they were interpreted as local, but now the local timestamp types are interpreted as local - I think this is correct per the [Avro spec](https://avro.apache.org/docs/1.12.0/specification/#timestamps)). Requires setting legacyMode = false. * Removed namespaces from generated Arrow field names in complex types. E.g. the Avro field myNamepsace.outerRecord.structField.intField should be called just "intField" inside the Arrow struct. This doesn't affect the skip field logic, which still works using the qualified names. This requires setting legacyMode = false. * Remove unexpected metadata in generated Arrow fields (empty alias lists and attributes interpreted as part of the field schema). This requires setting legacyMode = false. * Use the expected child vector names for Arrow LIST and MAP types when reading. For LIST, the default child vector is called "$data$" which is illegal in Avro, so the child field name is also changed to "item" in the producers. This requires setting legacyMode = false. Breaking changes have been removed from this PR. Per discussion below, all breaking changes are now behind a "legacyMode" flag in the AvroToArrowConfig object, which is enabled by default in all the original code paths. Closes #698 . This change is meant to allow for round trip of schemas and individual Avro data blocks (one Avro data block -> one VSR). File-level capabilities are not included. I have not included anything to recycle the VSR as part of the read API, this feels like it belongs with the file-level piece. Also I have not done anything specific for enums / dict encoding as of yet. --- .../arrow/adapter/avro/ArrowToAvroUtils.java | 12 +- .../arrow/adapter/avro/AvroToArrow.java | 19 + .../arrow/adapter/avro/AvroToArrowConfig.java | 41 + .../arrow/adapter/avro/AvroToArrowUtils.java | 414 ++++- .../avro/consumers/AvroNullableConsumer.java | 82 + .../logical/AvroDecimal256Consumer.java | 74 + .../logical/AvroTimestampMicrosConsumer.java | 2 +- .../AvroTimestampMicrosTzConsumer.java | 39 + .../logical/AvroTimestampMillisConsumer.java | 2 +- .../AvroTimestampMillisTzConsumer.java | 39 + .../logical/AvroTimestampNanosConsumer.java | 39 + .../logical/AvroTimestampNanosTzConsumer.java | 39 + .../avro/producers/AvroNullableProducer.java | 4 +- .../adapter/avro/ArrowToAvroSchemaTest.java | 24 +- .../adapter/avro/AvroLogicalTypesTest.java | 2 +- .../arrow/adapter/avro/RoundTripDataTest.java | 1606 +++++++++++++++++ .../adapter/avro/RoundTripSchemaTest.java | 443 +++++ .../schema/logical/test_decimal_invalid1.avsc | 2 +- .../logical/test_local_timestamp_micros.avsc | 23 + .../logical/test_local_timestamp_millis.avsc | 23 + .../logical/test_local_timestamp_nanos.avsc | 23 + .../schema/logical/test_timestamp_nanos.avsc | 23 + 22 files changed, 2870 insertions(+), 105 deletions(-) create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroNullableConsumer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDecimal256Consumer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMicrosTzConsumer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMillisTzConsumer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampNanosConsumer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampNanosTzConsumer.java create mode 100644 adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripDataTest.java create mode 100644 adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripSchemaTest.java create mode 100644 adapter/avro/src/test/resources/schema/logical/test_local_timestamp_micros.avsc create mode 100644 adapter/avro/src/test/resources/schema/logical/test_local_timestamp_millis.avsc create mode 100644 adapter/avro/src/test/resources/schema/logical/test_local_timestamp_nanos.avsc create mode 100644 adapter/avro/src/test/resources/schema/logical/test_timestamp_nanos.avsc diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/ArrowToAvroUtils.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/ArrowToAvroUtils.java index 6f0cb5cffc..87b594af9e 100644 --- a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/ArrowToAvroUtils.java +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/ArrowToAvroUtils.java @@ -332,7 +332,17 @@ private static T buildBaseTypeSchema( case List: case FixedSizeList: - return buildArraySchema(builder.array(), field, namespace); + // Arrow uses "$data$" as the field name for list items, that is not a valid Avro name + Field itemField = field.getChildren().get(0); + if (ListVector.DATA_VECTOR_NAME.equals(itemField.getName())) { + Field safeItemField = + new Field("item", itemField.getFieldType(), itemField.getChildren()); + Field safeListField = + new Field(field.getName(), field.getFieldType(), List.of(safeItemField)); + return buildArraySchema(builder.array(), safeListField, namespace); + } else { + return buildArraySchema(builder.array(), field, namespace); + } case Map: return buildMapSchema(builder.map(), field, namespace); diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrow.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrow.java index 2392c36f94..2a28ad393b 100644 --- a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrow.java +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrow.java @@ -59,4 +59,23 @@ public static AvroToArrowVectorIterator avroToArrowIterator( return AvroToArrowVectorIterator.create(decoder, schema, config); } + + /** + * Convert an Avro schema to its Arrow equivalent. + * + *

The resulting set of Arrow fields matches what would be set in the VSR after calling + * avroToArrow() or avroToArrowIterator(), respecting the configuration in the config parameter. + * + * @param schema The Avro schema to convert + * @param config Configuration options for conversion + * @return The equivalent Arrow schema + */ + public static org.apache.arrow.vector.types.pojo.Schema avroToAvroSchema( + Schema schema, AvroToArrowConfig config) { + + Preconditions.checkNotNull(schema, "Avro schema object cannot be null"); + Preconditions.checkNotNull(config, "config cannot be null"); + + return AvroToArrowUtils.createArrowSchema(schema, config); + } } diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowConfig.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowConfig.java index bd70c2b8ba..5596138586 100644 --- a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowConfig.java +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowConfig.java @@ -41,6 +41,12 @@ public class AvroToArrowConfig { /** The field names which to skip when reading decoder values. */ private final Set skipFieldNames; + /** + * Use legacy-mode to keep compatibility with old behavior (pre-2025), enabled by default. This + * affects how the AvroToArrow code interprets the Avro schema. + */ + private final boolean legacyMode; + /** * Instantiate an instance. * @@ -64,6 +70,37 @@ public class AvroToArrowConfig { this.targetBatchSize = targetBatchSize; this.provider = provider; this.skipFieldNames = skipFieldNames; + + // Default values for optional parameters + legacyMode = true; // Keep compatibility with old behavior by default + } + + /** + * Instantiate an instance. + * + * @param allocator The memory allocator to construct the Arrow vectors with. + * @param targetBatchSize The maximum rowCount to read each time when partially convert data. + * @param provider The dictionary provider used for enum type, adapter will update this provider. + * @param skipFieldNames Field names which to skip. + * @param legacyMode Keep compatibility with old behavior (pre-2025) + */ + AvroToArrowConfig( + BufferAllocator allocator, + int targetBatchSize, + DictionaryProvider.MapDictionaryProvider provider, + Set skipFieldNames, + boolean legacyMode) { + + Preconditions.checkArgument( + targetBatchSize == AvroToArrowVectorIterator.NO_LIMIT_BATCH_SIZE || targetBatchSize > 0, + "invalid targetBatchSize: %s", + targetBatchSize); + + this.allocator = allocator; + this.targetBatchSize = targetBatchSize; + this.provider = provider; + this.skipFieldNames = skipFieldNames; + this.legacyMode = legacyMode; } public BufferAllocator getAllocator() { @@ -81,4 +118,8 @@ public DictionaryProvider.MapDictionaryProvider getProvider() { public Set getSkipFieldNames() { return skipFieldNames; } + + public boolean isLegacyMode() { + return legacyMode; + } } diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java index ed7642aabd..aedef7732e 100644 --- a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java @@ -41,6 +41,7 @@ import org.apache.arrow.adapter.avro.consumers.AvroLongConsumer; import org.apache.arrow.adapter.avro.consumers.AvroMapConsumer; import org.apache.arrow.adapter.avro.consumers.AvroNullConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroNullableConsumer; import org.apache.arrow.adapter.avro.consumers.AvroStringConsumer; import org.apache.arrow.adapter.avro.consumers.AvroStructConsumer; import org.apache.arrow.adapter.avro.consumers.AvroUnionsConsumer; @@ -49,17 +50,23 @@ import org.apache.arrow.adapter.avro.consumers.SkipConsumer; import org.apache.arrow.adapter.avro.consumers.SkipFunction; import org.apache.arrow.adapter.avro.consumers.logical.AvroDateConsumer; +import org.apache.arrow.adapter.avro.consumers.logical.AvroDecimal256Consumer; import org.apache.arrow.adapter.avro.consumers.logical.AvroDecimalConsumer; import org.apache.arrow.adapter.avro.consumers.logical.AvroTimeMicroConsumer; import org.apache.arrow.adapter.avro.consumers.logical.AvroTimeMillisConsumer; import org.apache.arrow.adapter.avro.consumers.logical.AvroTimestampMicrosConsumer; +import org.apache.arrow.adapter.avro.consumers.logical.AvroTimestampMicrosTzConsumer; import org.apache.arrow.adapter.avro.consumers.logical.AvroTimestampMillisConsumer; +import org.apache.arrow.adapter.avro.consumers.logical.AvroTimestampMillisTzConsumer; +import org.apache.arrow.adapter.avro.consumers.logical.AvroTimestampNanosConsumer; +import org.apache.arrow.adapter.avro.consumers.logical.AvroTimestampNanosTzConsumer; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.BaseIntVector; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.DateDayVector; +import org.apache.arrow.vector.Decimal256Vector; import org.apache.arrow.vector.DecimalVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.FixedSizeBinaryVector; @@ -69,8 +76,12 @@ import org.apache.arrow.vector.NullVector; import org.apache.arrow.vector.TimeMicroVector; import org.apache.arrow.vector.TimeMilliVector; +import org.apache.arrow.vector.TimeStampMicroTZVector; import org.apache.arrow.vector.TimeStampMicroVector; +import org.apache.arrow.vector.TimeStampMilliTZVector; import org.apache.arrow.vector.TimeStampMilliVector; +import org.apache.arrow.vector.TimeStampNanoTZVector; +import org.apache.arrow.vector.TimeStampNanoVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; @@ -169,42 +180,69 @@ private static Consumer createConsumer( switch (type) { case UNION: - consumer = createUnionConsumer(schema, name, config, consumerVector); + boolean nullableUnion = + schema.getTypes().stream().anyMatch(t -> t.getType() == Schema.Type.NULL); + if (schema.getTypes().size() == 2 && nullableUnion && !config.isLegacyMode()) { + // For a simple nullable (null | type), interpret the union as a single nullable field. + // Not available in legacy mode, which uses the literal interpretation instead + int nullIndex = schema.getTypes().get(0).getType() == Schema.Type.NULL ? 0 : 1; + int childIndex = nullIndex == 0 ? 1 : 0; + Schema childSchema = schema.getTypes().get(childIndex); + Consumer childConsumer = + createConsumer(childSchema, name, true, config, consumerVector); + consumer = new AvroNullableConsumer<>(childConsumer, nullIndex); + } else { + // Literal interpretation of a union, which may or may not include a null element. + consumer = createUnionConsumer(schema, name, nullableUnion, config, consumerVector); + } break; case ARRAY: - consumer = createArrayConsumer(schema, name, config, consumerVector); + consumer = createArrayConsumer(schema, name, nullable, config, consumerVector); break; case MAP: - consumer = createMapConsumer(schema, name, config, consumerVector); + consumer = createMapConsumer(schema, name, nullable, config, consumerVector); break; case RECORD: - consumer = createStructConsumer(schema, name, config, consumerVector); + consumer = createStructConsumer(schema, name, nullable, config, consumerVector); break; case ENUM: - consumer = createEnumConsumer(schema, name, config, consumerVector); + consumer = createEnumConsumer(schema, name, nullable, config, consumerVector); break; case STRING: arrowType = new ArrowType.Utf8(); - fieldType = new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); + fieldType = + new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroStringConsumer((VarCharVector) vector); break; case FIXED: - Map extProps = createExternalProps(schema); + Map extProps = createExternalProps(schema, config); if (logicalType instanceof LogicalTypes.Decimal) { - arrowType = createDecimalArrowType((LogicalTypes.Decimal) logicalType); + arrowType = createDecimalArrowType((LogicalTypes.Decimal) logicalType, schema); fieldType = new FieldType( - nullable, arrowType, /* dictionary= */ null, getMetaData(schema, extProps)); + nullable, + arrowType, + /* dictionary= */ null, + getMetaData(schema, extProps, config)); vector = createVector(consumerVector, fieldType, name, allocator); - consumer = - new AvroDecimalConsumer.FixedDecimalConsumer( - (DecimalVector) vector, schema.getFixedSize()); + if (schema.getFixedSize() <= 16) { + consumer = + new AvroDecimalConsumer.FixedDecimalConsumer( + (DecimalVector) vector, schema.getFixedSize()); + } else { + consumer = + new AvroDecimal256Consumer.FixedDecimal256Consumer( + (Decimal256Vector) vector, schema.getFixedSize()); + } } else { arrowType = new ArrowType.FixedSizeBinary(schema.getFixedSize()); fieldType = new FieldType( - nullable, arrowType, /* dictionary= */ null, getMetaData(schema, extProps)); + nullable, + arrowType, + /* dictionary= */ null, + getMetaData(schema, extProps, config)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroFixedConsumer((FixedSizeBinaryVector) vector, schema.getFixedSize()); } @@ -213,26 +251,30 @@ private static Consumer createConsumer( if (logicalType instanceof LogicalTypes.Date) { arrowType = new ArrowType.Date(DateUnit.DAY); fieldType = - new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); + new FieldType( + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroDateConsumer((DateDayVector) vector); } else if (logicalType instanceof LogicalTypes.TimeMillis) { arrowType = new ArrowType.Time(TimeUnit.MILLISECOND, 32); fieldType = - new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); + new FieldType( + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroTimeMillisConsumer((TimeMilliVector) vector); } else { arrowType = new ArrowType.Int(32, /* isSigned= */ true); fieldType = - new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); + new FieldType( + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroIntConsumer((IntVector) vector); } break; case BOOLEAN: arrowType = new ArrowType.Bool(); - fieldType = new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); + fieldType = + new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroBooleanConsumer((BitVector) vector); break; @@ -240,60 +282,109 @@ private static Consumer createConsumer( if (logicalType instanceof LogicalTypes.TimeMicros) { arrowType = new ArrowType.Time(TimeUnit.MICROSECOND, 64); fieldType = - new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); + new FieldType( + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroTimeMicroConsumer((TimeMicroVector) vector); - } else if (logicalType instanceof LogicalTypes.TimestampMillis) { + } else if (logicalType instanceof LogicalTypes.TimestampMillis && !config.isLegacyMode()) { + // In legacy mode the timestamp-xxx types are treated as local, there is no zone aware + // type + arrowType = new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC"); + fieldType = + new FieldType( + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); + vector = createVector(consumerVector, fieldType, name, allocator); + consumer = new AvroTimestampMillisTzConsumer((TimeStampMilliTZVector) vector); + } else if (logicalType instanceof LogicalTypes.TimestampMicros && !config.isLegacyMode()) { + arrowType = new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC"); + fieldType = + new FieldType( + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); + vector = createVector(consumerVector, fieldType, name, allocator); + consumer = new AvroTimestampMicrosTzConsumer((TimeStampMicroTZVector) vector); + } else if (logicalType instanceof LogicalTypes.TimestampNanos && !config.isLegacyMode()) { + arrowType = new ArrowType.Timestamp(TimeUnit.NANOSECOND, "UTC"); + fieldType = + new FieldType( + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); + vector = createVector(consumerVector, fieldType, name, allocator); + consumer = new AvroTimestampNanosTzConsumer((TimeStampNanoTZVector) vector); + } else if (logicalType instanceof LogicalTypes.LocalTimestampMillis + || (logicalType instanceof LogicalTypes.TimestampMillis && config.isLegacyMode())) { arrowType = new ArrowType.Timestamp(TimeUnit.MILLISECOND, null); fieldType = - new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); + new FieldType( + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroTimestampMillisConsumer((TimeStampMilliVector) vector); - } else if (logicalType instanceof LogicalTypes.TimestampMicros) { + } else if (logicalType instanceof LogicalTypes.LocalTimestampMicros + || (logicalType instanceof LogicalTypes.TimestampMicros && config.isLegacyMode())) { + // In legacy mode the timestamp-xxx types are treated as local arrowType = new ArrowType.Timestamp(TimeUnit.MICROSECOND, null); fieldType = - new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); + new FieldType( + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroTimestampMicrosConsumer((TimeStampMicroVector) vector); + } else if (logicalType instanceof LogicalTypes.LocalTimestampNanos + || (logicalType instanceof LogicalTypes.TimestampNanos && config.isLegacyMode())) { + arrowType = new ArrowType.Timestamp(TimeUnit.NANOSECOND, null); + fieldType = + new FieldType( + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); + vector = createVector(consumerVector, fieldType, name, allocator); + consumer = new AvroTimestampNanosConsumer((TimeStampNanoVector) vector); } else { arrowType = new ArrowType.Int(64, /* isSigned= */ true); fieldType = - new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); + new FieldType( + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroLongConsumer((BigIntVector) vector); } break; case FLOAT: arrowType = new ArrowType.FloatingPoint(SINGLE); - fieldType = new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); + fieldType = + new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroFloatConsumer((Float4Vector) vector); break; case DOUBLE: arrowType = new ArrowType.FloatingPoint(DOUBLE); - fieldType = new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); + fieldType = + new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroDoubleConsumer((Float8Vector) vector); break; case BYTES: if (logicalType instanceof LogicalTypes.Decimal) { - arrowType = createDecimalArrowType((LogicalTypes.Decimal) logicalType); + LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType; + arrowType = createDecimalArrowType(decimalType, schema); fieldType = - new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); + new FieldType( + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); vector = createVector(consumerVector, fieldType, name, allocator); - consumer = new AvroDecimalConsumer.BytesDecimalConsumer((DecimalVector) vector); + if (decimalType.getPrecision() <= 38) { + consumer = new AvroDecimalConsumer.BytesDecimalConsumer((DecimalVector) vector); + } else { + consumer = + new AvroDecimal256Consumer.BytesDecimal256Consumer((Decimal256Vector) vector); + } } else { arrowType = new ArrowType.Binary(); fieldType = - new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); + new FieldType( + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroBytesConsumer((VarBinaryVector) vector); } break; case NULL: arrowType = new ArrowType.Null(); - fieldType = new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); - vector = fieldType.createNewSingleVector(name, allocator, /* schemaCallBack= */ null); + fieldType = + new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); + vector = new NullVector(name, fieldType); // Respect nullability defined in fieldType consumer = new AvroNullConsumer((NullVector) vector); break; default: @@ -304,19 +395,31 @@ private static Consumer createConsumer( return consumer; } - private static ArrowType createDecimalArrowType(LogicalTypes.Decimal logicalType) { + private static ArrowType createDecimalArrowType(LogicalTypes.Decimal logicalType, Schema schema) { final int scale = logicalType.getScale(); final int precision = logicalType.getPrecision(); Preconditions.checkArgument( - precision > 0 && precision <= 38, "Precision must be in range of 1 to 38"); - Preconditions.checkArgument(scale >= 0 && scale <= 38, "Scale must be in range of 0 to 38."); + precision > 0 && precision <= 76, "Precision must be in range of 1 to 76"); + Preconditions.checkArgument(scale >= 0 && scale <= 76, "Scale must be in range of 0 to 76."); Preconditions.checkArgument( scale <= precision, "Invalid decimal scale: %s (greater than precision: %s)", scale, precision); - return new ArrowType.Decimal(precision, scale, 128); + if (schema.getType() == Schema.Type.FIXED) { + if (schema.getFixedSize() <= 16) { + return new ArrowType.Decimal(precision, scale, 128); + } else { + return new ArrowType.Decimal(precision, scale, 256); + } + } else { + if (precision <= 38) { + return new ArrowType.Decimal(precision, scale, 128); + } else { + return new ArrowType.Decimal(precision, scale, 256); + } + } } private static Consumer createSkipConsumer(Schema schema) { @@ -406,6 +509,30 @@ private static Consumer createSkipConsumer(Schema schema) { return new SkipConsumer(skipFunction); } + static org.apache.arrow.vector.types.pojo.Schema createArrowSchema( + Schema schema, AvroToArrowConfig config) { + + // Create an Arrow schema matching the structure of vectors built by createCompositeConsumer() + + Set skipFieldNames = config.getSkipFieldNames(); + List arrowFields = new ArrayList<>(schema.getFields().size()); + + Schema.Type type = schema.getType(); + if (type == Schema.Type.RECORD) { + for (Schema.Field field : schema.getFields()) { + if (!skipFieldNames.contains(field.name())) { + Field arrowField = avroSchemaToField(field.schema(), field.name(), config); + arrowFields.add(arrowField); + } + } + } else { + Field arrowField = avroSchemaToField(schema, schema.getName(), config); + arrowFields.add(arrowField); + } + + return new org.apache.arrow.vector.types.pojo.Schema(arrowFields); + } + static CompositeAvroConsumer createCompositeConsumer(Schema schema, AvroToArrowConfig config) { List consumers = new ArrayList<>(); @@ -442,11 +569,20 @@ private static String getDefaultFieldName(ArrowType type) { } private static Field avroSchemaToField(Schema schema, String name, AvroToArrowConfig config) { - return avroSchemaToField(schema, name, config, null); + return avroSchemaToField(schema, name, false, config, null); } private static Field avroSchemaToField( Schema schema, String name, AvroToArrowConfig config, Map externalProps) { + return avroSchemaToField(schema, name, false, config, externalProps); + } + + private static Field avroSchemaToField( + Schema schema, + String name, + boolean nullable, + AvroToArrowConfig config, + Map externalProps) { final Schema.Type type = schema.getType(); final LogicalType logicalType = schema.getLogicalType(); @@ -455,33 +591,53 @@ private static Field avroSchemaToField( switch (type) { case UNION: - for (int i = 0; i < schema.getTypes().size(); i++) { - Schema childSchema = schema.getTypes().get(i); - // Union child vector should use default name - children.add(avroSchemaToField(childSchema, null, config)); + boolean nullableUnion = + schema.getTypes().stream().anyMatch(t -> t.getType() == Schema.Type.NULL); + if (nullableUnion && schema.getTypes().size() == 2 && !config.isLegacyMode()) { + // For a simple nullable (null | type), interpret the union as a single nullable field. + // Not available in legacy mode, which uses the literal interpretation instead + Schema childSchema = + schema.getTypes().get(0).getType() == Schema.Type.NULL + ? schema.getTypes().get(1) + : schema.getTypes().get(0); + return avroSchemaToField(childSchema, name, true, config, externalProps); + } else { + // Literal interpretation of a union, which may or may not include a null element. + for (int i = 0; i < schema.getTypes().size(); i++) { + Schema childSchema = schema.getTypes().get(i); + // Union child vector should use default name + children.add(avroSchemaToField(childSchema, null, nullableUnion, config, null)); + } + fieldType = + createFieldType( + new ArrowType.Union(UnionMode.Sparse, null), schema, externalProps, config); } - fieldType = - createFieldType(new ArrowType.Union(UnionMode.Sparse, null), schema, externalProps); break; case ARRAY: Schema elementSchema = schema.getElementType(); - children.add(avroSchemaToField(elementSchema, elementSchema.getName(), config)); - fieldType = createFieldType(new ArrowType.List(), schema, externalProps); + children.add(avroSchemaToField(elementSchema, ListVector.DATA_VECTOR_NAME, config)); + fieldType = createFieldType(nullable, new ArrowType.List(), schema, externalProps, config); break; case MAP: // MapVector internal struct field and key field should be non-nullable FieldType keyFieldType = new FieldType(/* nullable= */ false, new ArrowType.Utf8(), /* dictionary= */ null); - Field keyField = new Field("key", keyFieldType, /* children= */ null); - Field valueField = avroSchemaToField(schema.getValueType(), "value", config); + Field keyField = new Field(MapVector.KEY_NAME, keyFieldType, /* children= */ null); + Field valueField = avroSchemaToField(schema.getValueType(), MapVector.VALUE_NAME, config); FieldType structFieldType = new FieldType(false, new ArrowType.Struct(), /* dictionary= */ null); Field structField = - new Field("internal", structFieldType, Arrays.asList(keyField, valueField)); + new Field( + MapVector.DATA_VECTOR_NAME, structFieldType, Arrays.asList(keyField, valueField)); children.add(structField); fieldType = - createFieldType(new ArrowType.Map(/* keysSorted= */ false), schema, externalProps); + createFieldType( + nullable, + new ArrowType.Map(/* keysSorted= */ false), + schema, + externalProps, + config); break; case RECORD: final Set skipFieldNames = config.getSkipFieldNames(); @@ -496,13 +652,14 @@ private static Field avroSchemaToField( if (doc != null) { extProps.put("doc", doc); } - if (aliases != null) { + if (aliases != null && (!aliases.isEmpty() || config.isLegacyMode())) { extProps.put("aliases", convertAliases(aliases)); } children.add(avroSchemaToField(childSchema, fullChildName, config, extProps)); } } - fieldType = createFieldType(new ArrowType.Struct(), schema, externalProps); + fieldType = + createFieldType(nullable, new ArrowType.Struct(), schema, externalProps, config); break; case ENUM: DictionaryProvider.MapDictionaryProvider provider = config.getProvider(); @@ -512,23 +669,25 @@ private static Field avroSchemaToField( fieldType = createFieldType( + nullable, indexType, schema, externalProps, - new DictionaryEncoding(current, /* ordered= */ false, /* indexType= */ indexType)); + new DictionaryEncoding(current, /* ordered= */ false, /* indexType= */ indexType), + config); break; case STRING: - fieldType = createFieldType(new ArrowType.Utf8(), schema, externalProps); + fieldType = createFieldType(nullable, new ArrowType.Utf8(), schema, externalProps, config); break; case FIXED: final ArrowType fixedArrowType; if (logicalType instanceof LogicalTypes.Decimal) { - fixedArrowType = createDecimalArrowType((LogicalTypes.Decimal) logicalType); + fixedArrowType = createDecimalArrowType((LogicalTypes.Decimal) logicalType, schema); } else { fixedArrowType = new ArrowType.FixedSizeBinary(schema.getFixedSize()); } - fieldType = createFieldType(fixedArrowType, schema, externalProps); + fieldType = createFieldType(nullable, fixedArrowType, schema, externalProps, config); break; case INT: final ArrowType intArrowType; @@ -539,41 +698,62 @@ private static Field avroSchemaToField( } else { intArrowType = new ArrowType.Int(32, /* isSigned= */ true); } - fieldType = createFieldType(intArrowType, schema, externalProps); + fieldType = createFieldType(nullable, intArrowType, schema, externalProps, config); break; case BOOLEAN: - fieldType = createFieldType(new ArrowType.Bool(), schema, externalProps); + fieldType = createFieldType(nullable, new ArrowType.Bool(), schema, externalProps, config); break; case LONG: final ArrowType longArrowType; if (logicalType instanceof LogicalTypes.TimeMicros) { longArrowType = new ArrowType.Time(TimeUnit.MICROSECOND, 64); } else if (logicalType instanceof LogicalTypes.TimestampMillis) { - longArrowType = new ArrowType.Timestamp(TimeUnit.MILLISECOND, null); + // In legacy mode the timestamp-xxx types are treated as local + String tz = config.isLegacyMode() ? null : "UTC"; + longArrowType = new ArrowType.Timestamp(TimeUnit.MILLISECOND, tz); } else if (logicalType instanceof LogicalTypes.TimestampMicros) { + String tz = config.isLegacyMode() ? null : "UTC"; + longArrowType = new ArrowType.Timestamp(TimeUnit.MICROSECOND, tz); + } else if (logicalType instanceof LogicalTypes.TimestampNanos) { + String tz = config.isLegacyMode() ? null : "UTC"; + longArrowType = new ArrowType.Timestamp(TimeUnit.NANOSECOND, tz); + } else if (logicalType instanceof LogicalTypes.LocalTimestampMillis + && !config.isLegacyMode()) { + // In legacy mode the local-timestamp-xxx types are not recognized (result is just type = + // long) + longArrowType = new ArrowType.Timestamp(TimeUnit.MILLISECOND, null); + } else if (logicalType instanceof LogicalTypes.LocalTimestampMicros + && !config.isLegacyMode()) { longArrowType = new ArrowType.Timestamp(TimeUnit.MICROSECOND, null); + } else if (logicalType instanceof LogicalTypes.LocalTimestampNanos + && !config.isLegacyMode()) { + longArrowType = new ArrowType.Timestamp(TimeUnit.NANOSECOND, null); } else { longArrowType = new ArrowType.Int(64, /* isSigned= */ true); } - fieldType = createFieldType(longArrowType, schema, externalProps); + fieldType = createFieldType(nullable, longArrowType, schema, externalProps, config); break; case FLOAT: - fieldType = createFieldType(new ArrowType.FloatingPoint(SINGLE), schema, externalProps); + fieldType = + createFieldType( + nullable, new ArrowType.FloatingPoint(SINGLE), schema, externalProps, config); break; case DOUBLE: - fieldType = createFieldType(new ArrowType.FloatingPoint(DOUBLE), schema, externalProps); + fieldType = + createFieldType( + nullable, new ArrowType.FloatingPoint(DOUBLE), schema, externalProps, config); break; case BYTES: final ArrowType bytesArrowType; if (logicalType instanceof LogicalTypes.Decimal) { - bytesArrowType = createDecimalArrowType((LogicalTypes.Decimal) logicalType); + bytesArrowType = createDecimalArrowType((LogicalTypes.Decimal) logicalType, schema); } else { bytesArrowType = new ArrowType.Binary(); } - fieldType = createFieldType(bytesArrowType, schema, externalProps); + fieldType = createFieldType(nullable, bytesArrowType, schema, externalProps, config); break; case NULL: - fieldType = createFieldType(ArrowType.Null.INSTANCE, schema, externalProps); + fieldType = createFieldType(ArrowType.Null.INSTANCE, schema, externalProps, config); break; default: // no-op, shouldn't get here @@ -583,15 +763,24 @@ private static Field avroSchemaToField( if (name == null) { name = getDefaultFieldName(fieldType.getType()); } + if (name.contains(".") && !config.isLegacyMode()) { + // Do not include namespace as part of the field name + name = name.substring(name.lastIndexOf(".") + 1); + } return new Field(name, fieldType, children.size() == 0 ? null : children); } private static Consumer createArrayConsumer( - Schema schema, String name, AvroToArrowConfig config, FieldVector consumerVector) { + Schema schema, + String name, + boolean nullable, + AvroToArrowConfig config, + FieldVector consumerVector) { ListVector listVector; if (consumerVector == null) { - final Field field = avroSchemaToField(schema, name, config); + final Field field = + avroSchemaToField(schema, name, nullable, config, /* externalProps= */ null); listVector = (ListVector) field.createVector(config.getAllocator()); } else { listVector = (ListVector) consumerVector; @@ -607,13 +796,18 @@ private static Consumer createArrayConsumer( } private static Consumer createStructConsumer( - Schema schema, String name, AvroToArrowConfig config, FieldVector consumerVector) { + Schema schema, + String name, + boolean nullable, + AvroToArrowConfig config, + FieldVector consumerVector) { final Set skipFieldNames = config.getSkipFieldNames(); StructVector structVector; if (consumerVector == null) { - final Field field = avroSchemaToField(schema, name, config, createExternalProps(schema)); + final Field field = + avroSchemaToField(schema, name, nullable, config, createExternalProps(schema, config)); structVector = (StructVector) field.createVector(config.getAllocator()); } else { structVector = (StructVector) consumerVector; @@ -644,11 +838,16 @@ private static Consumer createStructConsumer( } private static Consumer createEnumConsumer( - Schema schema, String name, AvroToArrowConfig config, FieldVector consumerVector) { + Schema schema, + String name, + boolean nullable, + AvroToArrowConfig config, + FieldVector consumerVector) { BaseIntVector indexVector; if (consumerVector == null) { - final Field field = avroSchemaToField(schema, name, config, createExternalProps(schema)); + final Field field = + avroSchemaToField(schema, name, nullable, config, createExternalProps(schema, config)); indexVector = (BaseIntVector) field.createVector(config.getAllocator()); } else { indexVector = (BaseIntVector) consumerVector; @@ -668,11 +867,16 @@ private static Consumer createEnumConsumer( } private static Consumer createMapConsumer( - Schema schema, String name, AvroToArrowConfig config, FieldVector consumerVector) { + Schema schema, + String name, + boolean nullable, + AvroToArrowConfig config, + FieldVector consumerVector) { MapVector mapVector; if (consumerVector == null) { - final Field field = avroSchemaToField(schema, name, config); + final Field field = + avroSchemaToField(schema, name, nullable, config, /* externalProps= */ null); mapVector = (MapVector) field.createVector(config.getAllocator()); } else { mapVector = (MapVector) consumerVector; @@ -698,12 +902,13 @@ private static Consumer createMapConsumer( } private static Consumer createUnionConsumer( - Schema schema, String name, AvroToArrowConfig config, FieldVector consumerVector) { + Schema schema, + String name, + boolean nullableUnion, + AvroToArrowConfig config, + FieldVector consumerVector) { final int size = schema.getTypes().size(); - final boolean nullable = - schema.getTypes().stream().anyMatch(t -> t.getType() == Schema.Type.NULL); - UnionVector unionVector; if (consumerVector == null) { final Field field = avroSchemaToField(schema, name, config); @@ -720,7 +925,8 @@ private static Consumer createUnionConsumer( for (int i = 0; i < size; i++) { FieldVector child = childVectors.get(i); Schema subSchema = schema.getTypes().get(i); - Consumer delegate = createConsumer(subSchema, subSchema.getName(), nullable, config, child); + Consumer delegate = + createConsumer(subSchema, subSchema.getName(), nullableUnion, config, child); delegates[i] = delegate; types[i] = child.getMinorType(); } @@ -785,14 +991,24 @@ static VectorSchemaRoot avroToArrowVectors( return root; } - private static Map getMetaData(Schema schema) { + // Do not include props that are part of the Avro format itself as field metadata + // These are already represented in the field / type structure and are not custom attributes + private static final List AVRO_FORMAT_METADATA = + Arrays.asList("logicalType", "precision", "scale"); + + private static Map getMetaData(Schema schema, AvroToArrowConfig config) { Map metadata = new HashMap<>(); - schema.getObjectProps().forEach((k, v) -> metadata.put(k, v.toString())); + for (Map.Entry prop : schema.getObjectProps().entrySet()) { + if (!AVRO_FORMAT_METADATA.contains(prop.getKey()) || config.isLegacyMode()) { + metadata.put(prop.getKey(), prop.getValue().toString()); + } + } return metadata; } - private static Map getMetaData(Schema schema, Map externalProps) { - Map metadata = getMetaData(schema); + private static Map getMetaData( + Schema schema, Map externalProps, AvroToArrowConfig config) { + Map metadata = getMetaData(schema, config); if (externalProps != null) { metadata.putAll(externalProps); } @@ -800,32 +1016,58 @@ private static Map getMetaData(Schema schema, Map createExternalProps(Schema schema) { + private static Map createExternalProps(Schema schema, AvroToArrowConfig config) { final Map extProps = new HashMap<>(); String doc = schema.getDoc(); Set aliases = schema.getAliases(); if (doc != null) { extProps.put("doc", doc); } - if (aliases != null) { + if (aliases != null && (!aliases.isEmpty() || config.isLegacyMode())) { extProps.put("aliases", convertAliases(aliases)); } return extProps; } private static FieldType createFieldType( - ArrowType arrowType, Schema schema, Map externalProps) { - return createFieldType(arrowType, schema, externalProps, /* dictionary= */ null); + ArrowType arrowType, + Schema schema, + Map externalProps, + AvroToArrowConfig config) { + return createFieldType(arrowType, schema, externalProps, /* dictionary= */ null, config); + } + + private static FieldType createFieldType( + boolean nullable, + ArrowType arrowType, + Schema schema, + Map externalProps, + AvroToArrowConfig config) { + return createFieldType( + nullable, arrowType, schema, externalProps, /* dictionary= */ null, config); } private static FieldType createFieldType( ArrowType arrowType, Schema schema, Map externalProps, - DictionaryEncoding dictionary) { + DictionaryEncoding dictionary, + AvroToArrowConfig config) { + + return createFieldType( + /* nullable= */ false, arrowType, schema, externalProps, dictionary, config); + } + + private static FieldType createFieldType( + boolean nullable, + ArrowType arrowType, + Schema schema, + Map externalProps, + DictionaryEncoding dictionary, + AvroToArrowConfig config) { return new FieldType( - /* nullable= */ false, arrowType, dictionary, getMetaData(schema, externalProps)); + nullable, arrowType, dictionary, getMetaData(schema, externalProps, config)); } private static String convertAliases(Set aliases) { diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroNullableConsumer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroNullableConsumer.java new file mode 100644 index 0000000000..b67819cb9d --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroNullableConsumer.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.consumers; + +import java.io.IOException; +import org.apache.arrow.vector.FieldVector; +import org.apache.avro.io.Decoder; + +/** + * Consumer wrapper which consumes nullable type values from avro decoder. Write the data to the + * underlying {@link FieldVector}. + * + * @param The vector within consumer or its delegate. + */ +public class AvroNullableConsumer extends BaseAvroConsumer { + + private final Consumer delegate; + private final int nullIndex; + + /** Instantiate a AvroNullableConsumer. */ + @SuppressWarnings("unchecked") + public AvroNullableConsumer(Consumer delegate, int nullIndex) { + super((T) delegate.getVector()); + this.delegate = delegate; + this.nullIndex = nullIndex; + } + + @Override + public void consume(Decoder decoder) throws IOException { + int typeIndex = decoder.readInt(); + if (typeIndex == nullIndex) { + decoder.readNull(); + delegate.addNull(); + } else { + delegate.consume(decoder); + } + currentIndex++; + } + + @Override + public void addNull() { + // Can be called by containers of nullable types + delegate.addNull(); + currentIndex++; + } + + @Override + public void setPosition(int index) { + if (index < 0 || index > vector.getValueCount()) { + throw new IllegalArgumentException("Index out of bounds"); + } + delegate.setPosition(index); + super.setPosition(index); + } + + @Override + public boolean resetValueVector(T vector) { + boolean delegateOk = delegate.resetValueVector(vector); + boolean thisOk = super.resetValueVector(vector); + return thisOk && delegateOk; + } + + @Override + public void close() throws Exception { + super.close(); + delegate.close(); + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDecimal256Consumer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDecimal256Consumer.java new file mode 100644 index 0000000000..12652833a1 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDecimal256Consumer.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.consumers.logical; + +import java.io.IOException; +import java.nio.ByteBuffer; +import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; +import org.apache.arrow.util.Preconditions; +import org.apache.arrow.vector.Decimal256Vector; +import org.apache.avro.io.Decoder; + +/** + * Consumer which consume 256-bit decimal type values from avro decoder. Write the data to {@link + * Decimal256Vector}. + */ +public abstract class AvroDecimal256Consumer extends BaseAvroConsumer { + + protected AvroDecimal256Consumer(Decimal256Vector vector) { + super(vector); + } + + /** Consumer for decimal logical type with 256 bit width and original bytes type. */ + public static class BytesDecimal256Consumer extends AvroDecimal256Consumer { + + private ByteBuffer cacheBuffer; + + /** Instantiate a BytesDecimal256Consumer. */ + public BytesDecimal256Consumer(Decimal256Vector vector) { + super(vector); + } + + @Override + public void consume(Decoder decoder) throws IOException { + cacheBuffer = decoder.readBytes(cacheBuffer); + byte[] bytes = new byte[cacheBuffer.limit()]; + Preconditions.checkArgument(bytes.length <= 32, "Decimal bytes length should <= 32."); + cacheBuffer.get(bytes); + vector.setBigEndian(currentIndex++, bytes); + } + } + + /** Consumer for decimal logical type with 256 bit width and original fixed type. */ + public static class FixedDecimal256Consumer extends AvroDecimal256Consumer { + + private final byte[] reuseBytes; + + /** Instantiate a FixedDecimal256Consumer. */ + public FixedDecimal256Consumer(Decimal256Vector vector, int size) { + super(vector); + Preconditions.checkArgument(size <= 32, "Decimal bytes length should <= 32."); + reuseBytes = new byte[size]; + } + + @Override + public void consume(Decoder decoder) throws IOException { + decoder.readFixed(reuseBytes); + vector.setBigEndian(currentIndex++, reuseBytes); + } + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMicrosConsumer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMicrosConsumer.java index 88acf7b329..5af40ed17d 100644 --- a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMicrosConsumer.java +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMicrosConsumer.java @@ -22,7 +22,7 @@ import org.apache.avro.io.Decoder; /** - * Consumer which consume date timestamp-micro values from avro decoder. Write the data to {@link + * Consumer which consumes local-timestamp-micros values from avro decoder. Write the data to {@link * TimeStampMicroVector}. */ public class AvroTimestampMicrosConsumer extends BaseAvroConsumer { diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMicrosTzConsumer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMicrosTzConsumer.java new file mode 100644 index 0000000000..a5dede4988 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMicrosTzConsumer.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.consumers.logical; + +import java.io.IOException; +import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; +import org.apache.arrow.vector.TimeStampMicroTZVector; +import org.apache.avro.io.Decoder; + +/** + * Consumer which consumes timestamp-micros values from avro decoder. Write the data to {@link + * TimeStampMicroTZVector}. + */ +public class AvroTimestampMicrosTzConsumer extends BaseAvroConsumer { + + /** Instantiate a AvroTimestampMicrosTzConsumer. */ + public AvroTimestampMicrosTzConsumer(TimeStampMicroTZVector vector) { + super(vector); + } + + @Override + public void consume(Decoder decoder) throws IOException { + vector.set(currentIndex++, decoder.readLong()); + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMillisConsumer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMillisConsumer.java index ec50d79023..bc451bd1dc 100644 --- a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMillisConsumer.java +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMillisConsumer.java @@ -22,7 +22,7 @@ import org.apache.avro.io.Decoder; /** - * Consumer which consume date timestamp-millis values from avro decoder. Write the data to {@link + * Consumer which consume local-timestamp-millis values from avro decoder. Write the data to {@link * TimeStampMilliVector}. */ public class AvroTimestampMillisConsumer extends BaseAvroConsumer { diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMillisTzConsumer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMillisTzConsumer.java new file mode 100644 index 0000000000..255fe501fb --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMillisTzConsumer.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.consumers.logical; + +import java.io.IOException; +import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; +import org.apache.arrow.vector.TimeStampMilliTZVector; +import org.apache.avro.io.Decoder; + +/** + * Consumer which consume timestamp-millis values from avro decoder. Write the data to {@link + * TimeStampMilliTZVector}. + */ +public class AvroTimestampMillisTzConsumer extends BaseAvroConsumer { + + /** Instantiate a AvroTimestampMillisTzConsumer. */ + public AvroTimestampMillisTzConsumer(TimeStampMilliTZVector vector) { + super(vector); + } + + @Override + public void consume(Decoder decoder) throws IOException { + vector.set(currentIndex++, decoder.readLong()); + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampNanosConsumer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampNanosConsumer.java new file mode 100644 index 0000000000..b5044d221f --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampNanosConsumer.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.consumers.logical; + +import java.io.IOException; +import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; +import org.apache.arrow.vector.TimeStampNanoVector; +import org.apache.avro.io.Decoder; + +/** + * Consumer which consume local-timestamp-nanos values from avro decoder. Write the data to {@link + * TimeStampNanoVector}. + */ +public class AvroTimestampNanosConsumer extends BaseAvroConsumer { + + /** Instantiate a AvroTimestampNanosConsumer. */ + public AvroTimestampNanosConsumer(TimeStampNanoVector vector) { + super(vector); + } + + @Override + public void consume(Decoder decoder) throws IOException { + vector.set(currentIndex++, decoder.readLong()); + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampNanosTzConsumer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampNanosTzConsumer.java new file mode 100644 index 0000000000..3f42b7ccbb --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampNanosTzConsumer.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.consumers.logical; + +import java.io.IOException; +import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; +import org.apache.arrow.vector.TimeStampNanoTZVector; +import org.apache.avro.io.Decoder; + +/** + * Consumer which consume timestamp-nanos values from avro decoder. Write the data to {@link + * TimeStampNanoTZVector}. + */ +public class AvroTimestampNanosTzConsumer extends BaseAvroConsumer { + + /** Instantiate a AvroTimestampNanosConsumer. */ + public AvroTimestampNanosTzConsumer(TimeStampNanoTZVector vector) { + super(vector); + } + + @Override + public void consume(Decoder decoder) throws IOException { + vector.set(currentIndex++, decoder.readLong()); + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroNullableProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroNullableProducer.java index f4215dbf84..5f8b314f49 100644 --- a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroNullableProducer.java +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroNullableProducer.java @@ -21,8 +21,8 @@ import org.apache.avro.io.Encoder; /** - * Producer wrapper which producers nullable types to an avro encoder. Write the data to the - * underlying {@link FieldVector}. + * Producer wrapper which produces nullable types to an avro encoder. Read data from the underlying + * {@link FieldVector}. * * @param The vector within producer or its delegate, used for partially produce purpose. */ diff --git a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroSchemaTest.java b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroSchemaTest.java index a05bbc1653..d3e12e763a 100644 --- a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroSchemaTest.java +++ b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroSchemaTest.java @@ -319,10 +319,10 @@ public void testConvertDecimalTypes() { FieldType.notNullable(new ArrowType.Decimal(20, 10, 128)), null), new Field( - "nullableDecimal256", FieldType.nullable(new ArrowType.Decimal(20, 4, 256)), null), + "nullableDecimal256", FieldType.nullable(new ArrowType.Decimal(55, 15, 256)), null), new Field( "nonNullableDecimal2561", - FieldType.notNullable(new ArrowType.Decimal(20, 4, 256)), + FieldType.notNullable(new ArrowType.Decimal(55, 25, 256)), null), new Field( "nonNullableDecimal2562", @@ -330,7 +330,7 @@ public void testConvertDecimalTypes() { null), new Field( "nonNullableDecimal2563", - FieldType.notNullable(new ArrowType.Decimal(30, 15, 256)), + FieldType.notNullable(new ArrowType.Decimal(60, 50, 256)), null)); Schema schema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord"); @@ -383,9 +383,9 @@ public void testConvertDecimalTypes() { schema.getField("nullableDecimal256").schema().getTypes().get(0); assertEquals(Schema.Type.FIXED, nullableDecimal256Schema.getType()); assertEquals(32, nullableDecimal256Schema.getFixedSize()); - assertEquals(LogicalTypes.decimal(20, 4), nullableDecimal256Schema.getLogicalType()); - assertEquals(20, nullableDecimal256Schema.getObjectProp("precision")); - assertEquals(4, nullableDecimal256Schema.getObjectProp("scale")); + assertEquals(LogicalTypes.decimal(55, 15), nullableDecimal256Schema.getLogicalType()); + assertEquals(55, nullableDecimal256Schema.getObjectProp("precision")); + assertEquals(15, nullableDecimal256Schema.getObjectProp("scale")); assertEquals( Schema.Type.NULL, schema.getField("nullableDecimal256").schema().getTypes().get(1).getType()); @@ -394,9 +394,9 @@ public void testConvertDecimalTypes() { Schema nonNullableDecimal2561Schema = schema.getField("nonNullableDecimal2561").schema(); assertEquals(Schema.Type.FIXED, nonNullableDecimal2561Schema.getType()); assertEquals(32, nonNullableDecimal2561Schema.getFixedSize()); - assertEquals(LogicalTypes.decimal(20, 4), nonNullableDecimal2561Schema.getLogicalType()); - assertEquals(20, nonNullableDecimal2561Schema.getObjectProp("precision")); - assertEquals(4, nonNullableDecimal2561Schema.getObjectProp("scale")); + assertEquals(LogicalTypes.decimal(55, 25), nonNullableDecimal2561Schema.getLogicalType()); + assertEquals(55, nonNullableDecimal2561Schema.getObjectProp("precision")); + assertEquals(25, nonNullableDecimal2561Schema.getObjectProp("scale")); // Assertions for nonNullableDecimal2562 Schema nonNullableDecimal2562Schema = schema.getField("nonNullableDecimal2562").schema(); @@ -410,9 +410,9 @@ public void testConvertDecimalTypes() { Schema nonNullableDecimal2563Schema = schema.getField("nonNullableDecimal2563").schema(); assertEquals(Schema.Type.FIXED, nonNullableDecimal2563Schema.getType()); assertEquals(32, nonNullableDecimal2563Schema.getFixedSize()); - assertEquals(LogicalTypes.decimal(30, 15), nonNullableDecimal2563Schema.getLogicalType()); - assertEquals(30, nonNullableDecimal2563Schema.getObjectProp("precision")); - assertEquals(15, nonNullableDecimal2563Schema.getObjectProp("scale")); + assertEquals(LogicalTypes.decimal(60, 50), nonNullableDecimal2563Schema.getLogicalType()); + assertEquals(60, nonNullableDecimal2563Schema.getObjectProp("precision")); + assertEquals(50, nonNullableDecimal2563Schema.getObjectProp("scale")); } @Test diff --git a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroLogicalTypesTest.java b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroLogicalTypesTest.java index 173cc855b1..801456d79b 100644 --- a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroLogicalTypesTest.java +++ b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroLogicalTypesTest.java @@ -173,7 +173,7 @@ public void testInvalidDecimalPrecision() throws Exception { IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> writeAndRead(schema, data)); - assertTrue(e.getMessage().contains("Precision must be in range of 1 to 38")); + assertTrue(e.getMessage().contains("Precision must be in range of 1 to 76")); } @Test diff --git a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripDataTest.java b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripDataTest.java new file mode 100644 index 0000000000..85e6a960b0 --- /dev/null +++ b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripDataTest.java @@ -0,0 +1,1606 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZonedDateTime; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.apache.arrow.adapter.avro.producers.CompositeAvroProducer; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.DateDayVector; +import org.apache.arrow.vector.Decimal256Vector; +import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.FixedSizeBinaryVector; +import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.NullVector; +import org.apache.arrow.vector.TimeMicroVector; +import org.apache.arrow.vector.TimeMilliVector; +import org.apache.arrow.vector.TimeStampMicroTZVector; +import org.apache.arrow.vector.TimeStampMicroVector; +import org.apache.arrow.vector.TimeStampMilliTZVector; +import org.apache.arrow.vector.TimeStampMilliVector; +import org.apache.arrow.vector.TimeStampNanoTZVector; +import org.apache.arrow.vector.TimeStampNanoVector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.StructVector; +import org.apache.arrow.vector.complex.writer.BaseWriter; +import org.apache.arrow.vector.complex.writer.FieldWriter; +import org.apache.arrow.vector.types.DateUnit; +import org.apache.arrow.vector.types.FloatingPointPrecision; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.avro.Schema; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.EncoderFactory; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +public class RoundTripDataTest { + + @TempDir public static File TMP; + + private static AvroToArrowConfig basicConfig(BufferAllocator allocator) { + return new AvroToArrowConfig(allocator, 1000, null, Collections.emptySet(), false); + } + + private static VectorSchemaRoot readDataFile( + Schema schema, File dataFile, BufferAllocator allocator) throws Exception { + + try (FileInputStream fis = new FileInputStream(dataFile)) { + BinaryDecoder decoder = new DecoderFactory().directBinaryDecoder(fis, null); + return AvroToArrow.avroToArrow(schema, decoder, basicConfig(allocator)); + } + } + + private static void roundTripTest( + VectorSchemaRoot root, BufferAllocator allocator, File dataFile, int rowCount) + throws Exception { + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = + ArrowToAvroUtils.createCompositeProducer(root.getFieldVectors()); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Generate AVRO schema + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + + // Read back in and compare + try (VectorSchemaRoot roundTrip = readDataFile(schema, dataFile, allocator)) { + + assertEquals(root.getSchema(), roundTrip.getSchema()); + assertEquals(rowCount, roundTrip.getRowCount()); + + // Read and check values + for (int row = 0; row < rowCount; row++) { + assertEquals(root.getVector(0).getObject(row), roundTrip.getVector(0).getObject(row)); + } + } + } + + private static void roundTripByteArrayTest( + VectorSchemaRoot root, BufferAllocator allocator, File dataFile, int rowCount) + throws Exception { + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = + ArrowToAvroUtils.createCompositeProducer(root.getFieldVectors()); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Generate AVRO schema + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + + // Read back in and compare + try (VectorSchemaRoot roundTrip = readDataFile(schema, dataFile, allocator)) { + + assertEquals(root.getSchema(), roundTrip.getSchema()); + assertEquals(rowCount, roundTrip.getRowCount()); + + // Read and check values + for (int row = 0; row < rowCount; row++) { + byte[] rootBytes = (byte[]) root.getVector(0).getObject(row); + byte[] roundTripBytes = (byte[]) roundTrip.getVector(0).getObject(row); + assertArrayEquals(rootBytes, roundTripBytes); + } + } + } + + // Data round trip for primitive types, nullable and non-nullable + + @Test + public void testRoundTripNullColumn() throws Exception { + + // The current read implementation expects EOF, which never happens for a single null vector + // Include a boolean vector with this test for now, so that EOF exception will be triggered + + // Field definition + FieldType nullField = new FieldType(false, new ArrowType.Null(), null); + FieldType booleanField = new FieldType(false, new ArrowType.Bool(), null); + + // Create empty vector + BufferAllocator allocator = new RootAllocator(); + NullVector nullVector = new NullVector(new Field("nullColumn", nullField, null)); + BitVector booleanVector = new BitVector(new Field("boolean", booleanField, null), allocator); + + int rowCount = 10; + + // Set up VSR + List vectors = Arrays.asList(nullVector, booleanVector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set all values to null + for (int row = 0; row < rowCount; row++) { + nullVector.setNull(row); + booleanVector.set(row, 0); + } + + File dataFile = new File(TMP, "testRoundTripNullColumn.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripBooleans() throws Exception { + + // Field definition + FieldType booleanField = new FieldType(false, new ArrowType.Bool(), null); + + // Create empty vector + BufferAllocator allocator = new RootAllocator(); + BitVector booleanVector = new BitVector(new Field("boolean", booleanField, null), allocator); + + // Set up VSR + List vectors = Arrays.asList(booleanVector); + int rowCount = 10; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + for (int row = 0; row < rowCount; row++) { + booleanVector.set(row, row % 2 == 0 ? 1 : 0); + } + + File dataFile = new File(TMP, "testRoundTripBooleans.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripNullableBooleans() throws Exception { + + // Field definition + FieldType booleanField = new FieldType(true, new ArrowType.Bool(), null); + + // Create empty vector + BufferAllocator allocator = new RootAllocator(); + BitVector booleanVector = new BitVector(new Field("boolean", booleanField, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = Arrays.asList(booleanVector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Null value + booleanVector.setNull(0); + + // False value + booleanVector.set(1, 0); + + // True value + booleanVector.set(2, 1); + + File dataFile = new File(TMP, "testRoundTripNullableBooleans.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripIntegers() throws Exception { + + // Field definitions + FieldType int32Field = new FieldType(false, new ArrowType.Int(32, true), null); + FieldType int64Field = new FieldType(false, new ArrowType.Int(64, true), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + IntVector int32Vector = new IntVector(new Field("int32", int32Field, null), allocator); + BigIntVector int64Vector = new BigIntVector(new Field("int64", int64Field, null), allocator); + + // Set up VSR + List vectors = Arrays.asList(int32Vector, int64Vector); + + int rowCount = 12; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + for (int row = 0; row < 10; row++) { + int32Vector.set(row, 513 * row * (row % 2 == 0 ? 1 : -1)); + int64Vector.set(row, 3791L * row * (row % 2 == 0 ? 1 : -1)); + } + + // Min values + int32Vector.set(10, Integer.MIN_VALUE); + int64Vector.set(10, Long.MIN_VALUE); + + // Max values + int32Vector.set(11, Integer.MAX_VALUE); + int64Vector.set(11, Long.MAX_VALUE); + + File dataFile = new File(TMP, "testRoundTripIntegers.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripNullableIntegers() throws Exception { + + // Field definitions + FieldType int32Field = new FieldType(true, new ArrowType.Int(32, true), null); + FieldType int64Field = new FieldType(true, new ArrowType.Int(64, true), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + IntVector int32Vector = new IntVector(new Field("int32", int32Field, null), allocator); + BigIntVector int64Vector = new BigIntVector(new Field("int64", int64Field, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = Arrays.asList(int32Vector, int64Vector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Null values + int32Vector.setNull(0); + int64Vector.setNull(0); + + // Zero values + int32Vector.set(1, 0); + int64Vector.set(1, 0); + + // Non-zero values + int32Vector.set(2, Integer.MAX_VALUE); + int64Vector.set(2, Long.MAX_VALUE); + + File dataFile = new File(TMP, "testRoundTripNullableIntegers.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripFloatingPoints() throws Exception { + + // Field definitions + FieldType float32Field = + new FieldType(false, new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE), null); + FieldType float64Field = + new FieldType(false, new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + Float4Vector float32Vector = + new Float4Vector(new Field("float32", float32Field, null), allocator); + Float8Vector float64Vector = + new Float8Vector(new Field("float64", float64Field, null), allocator); + + // Set up VSR + List vectors = Arrays.asList(float32Vector, float64Vector); + int rowCount = 15; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + for (int row = 0; row < 10; row++) { + float32Vector.set(row, 37.6f * row * (row % 2 == 0 ? 1 : -1)); + float64Vector.set(row, 37.6d * row * (row % 2 == 0 ? 1 : -1)); + } + + float32Vector.set(10, Float.MIN_VALUE); + float64Vector.set(10, Double.MIN_VALUE); + + float32Vector.set(11, Float.MAX_VALUE); + float64Vector.set(11, Double.MAX_VALUE); + + float32Vector.set(12, Float.NaN); + float64Vector.set(12, Double.NaN); + + float32Vector.set(13, Float.POSITIVE_INFINITY); + float64Vector.set(13, Double.POSITIVE_INFINITY); + + float32Vector.set(14, Float.NEGATIVE_INFINITY); + float64Vector.set(14, Double.NEGATIVE_INFINITY); + + File dataFile = new File(TMP, "testRoundTripFloatingPoints.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripNullableFloatingPoints() throws Exception { + + // Field definitions + FieldType float32Field = + new FieldType(true, new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE), null); + FieldType float64Field = + new FieldType(true, new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + Float4Vector float32Vector = + new Float4Vector(new Field("float32", float32Field, null), allocator); + Float8Vector float64Vector = + new Float8Vector(new Field("float64", float64Field, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = Arrays.asList(float32Vector, float64Vector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Null values + float32Vector.setNull(0); + float64Vector.setNull(0); + + // Zero values + float32Vector.set(1, 0.0f); + float64Vector.set(1, 0.0); + + // Non-zero values + float32Vector.set(2, 1.0f); + float64Vector.set(2, 1.0); + + File dataFile = new File(TMP, "testRoundTripNullableFloatingPoints.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripStrings() throws Exception { + + // Field definition + FieldType stringField = new FieldType(false, new ArrowType.Utf8(), null); + + // Create empty vector + BufferAllocator allocator = new RootAllocator(); + VarCharVector stringVector = + new VarCharVector(new Field("string", stringField, null), allocator); + + // Set up VSR + List vectors = Arrays.asList(stringVector); + int rowCount = 5; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + stringVector.setSafe(0, "Hello world!".getBytes()); + stringVector.setSafe(1, "<%**\r\n\t\\abc\0$$>".getBytes()); + stringVector.setSafe(2, "你好世界".getBytes()); + stringVector.setSafe(3, "مرحبا بالعالم".getBytes()); + stringVector.setSafe(4, "(P ∧ P ⇒ Q) ⇒ Q".getBytes()); + + File dataFile = new File(TMP, "testRoundTripStrings.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripNullableStrings() throws Exception { + + // Field definition + FieldType stringField = new FieldType(true, new ArrowType.Utf8(), null); + + // Create empty vector + BufferAllocator allocator = new RootAllocator(); + VarCharVector stringVector = + new VarCharVector(new Field("string", stringField, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = Arrays.asList(stringVector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + stringVector.setNull(0); + stringVector.setSafe(1, "".getBytes()); + stringVector.setSafe(2, "not empty".getBytes()); + + File dataFile = new File(TMP, "testRoundTripNullableStrings.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripBinary() throws Exception { + + // Field definition + FieldType binaryField = new FieldType(false, new ArrowType.Binary(), null); + FieldType fixedField = new FieldType(false, new ArrowType.FixedSizeBinary(5), null); + + // Create empty vector + BufferAllocator allocator = new RootAllocator(); + VarBinaryVector binaryVector = + new VarBinaryVector(new Field("binary", binaryField, null), allocator); + FixedSizeBinaryVector fixedVector = + new FixedSizeBinaryVector(new Field("fixed", fixedField, null), allocator); + + // Set up VSR + List vectors = Arrays.asList(binaryVector, fixedVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + binaryVector.setSafe(0, new byte[] {1, 2, 3}); + binaryVector.setSafe(1, new byte[] {4, 5, 6, 7}); + binaryVector.setSafe(2, new byte[] {8, 9}); + + fixedVector.setSafe(0, new byte[] {1, 2, 3, 4, 5}); + fixedVector.setSafe(1, new byte[] {4, 5, 6, 7, 8, 9}); + fixedVector.setSafe(2, new byte[] {8, 9, 10, 11, 12}); + + File dataFile = new File(TMP, "testRoundTripBinary.avro"); + + roundTripByteArrayTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripNullableBinary() throws Exception { + + // Field definition + FieldType binaryField = new FieldType(true, new ArrowType.Binary(), null); + FieldType fixedField = new FieldType(true, new ArrowType.FixedSizeBinary(5), null); + + // Create empty vector + BufferAllocator allocator = new RootAllocator(); + VarBinaryVector binaryVector = + new VarBinaryVector(new Field("binary", binaryField, null), allocator); + FixedSizeBinaryVector fixedVector = + new FixedSizeBinaryVector(new Field("fixed", fixedField, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = Arrays.asList(binaryVector, fixedVector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + binaryVector.setNull(0); + binaryVector.setSafe(1, new byte[] {}); + binaryVector.setSafe(2, new byte[] {10, 11, 12}); + + fixedVector.setNull(0); + fixedVector.setSafe(1, new byte[] {0, 0, 0, 0, 0}); + fixedVector.setSafe(2, new byte[] {10, 11, 12, 13, 14}); + + File dataFile = new File(TMP, "testRoundTripNullableBinary.avro"); + + roundTripByteArrayTest(root, allocator, dataFile, rowCount); + } + } + + // Data round trip for logical types, nullable and non-nullable + + @Test + public void testRoundTripDecimals() throws Exception { + + // Field definitions + FieldType decimal128Field1 = new FieldType(false, new ArrowType.Decimal(38, 10, 128), null); + FieldType decimal128Field2 = new FieldType(false, new ArrowType.Decimal(38, 5, 128), null); + FieldType decimal256Field1 = new FieldType(false, new ArrowType.Decimal(76, 20, 256), null); + FieldType decimal256Field2 = new FieldType(false, new ArrowType.Decimal(76, 10, 256), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + DecimalVector decimal128Vector1 = + new DecimalVector(new Field("decimal128_1", decimal128Field1, null), allocator); + DecimalVector decimal128Vector2 = + new DecimalVector(new Field("decimal128_2", decimal128Field2, null), allocator); + Decimal256Vector decimal256Vector1 = + new Decimal256Vector(new Field("decimal256_1", decimal256Field1, null), allocator); + Decimal256Vector decimal256Vector2 = + new Decimal256Vector(new Field("decimal256_2", decimal256Field2, null), allocator); + + // Set up VSR + List vectors = + Arrays.asList(decimal128Vector1, decimal128Vector2, decimal256Vector1, decimal256Vector2); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + decimal128Vector1.setSafe( + 0, new BigDecimal("12345.67890").setScale(10, RoundingMode.UNNECESSARY)); + decimal128Vector1.setSafe( + 1, new BigDecimal("-98765.43210").setScale(10, RoundingMode.UNNECESSARY)); + decimal128Vector1.setSafe( + 2, new BigDecimal("54321.09876").setScale(10, RoundingMode.UNNECESSARY)); + + decimal128Vector2.setSafe( + 0, new BigDecimal("12345.67890").setScale(5, RoundingMode.UNNECESSARY)); + decimal128Vector2.setSafe( + 1, new BigDecimal("-98765.43210").setScale(5, RoundingMode.UNNECESSARY)); + decimal128Vector2.setSafe( + 2, new BigDecimal("54321.09876").setScale(5, RoundingMode.UNNECESSARY)); + + decimal256Vector1.setSafe( + 0, + new BigDecimal("12345678901234567890.12345678901234567890") + .setScale(20, RoundingMode.UNNECESSARY)); + decimal256Vector1.setSafe( + 1, + new BigDecimal("-98765432109876543210.98765432109876543210") + .setScale(20, RoundingMode.UNNECESSARY)); + decimal256Vector1.setSafe( + 2, + new BigDecimal("54321098765432109876.54321098765432109876") + .setScale(20, RoundingMode.UNNECESSARY)); + + decimal256Vector2.setSafe( + 0, + new BigDecimal("12345678901234567890.1234567890").setScale(10, RoundingMode.UNNECESSARY)); + decimal256Vector2.setSafe( + 1, + new BigDecimal("-98765432109876543210.9876543210") + .setScale(10, RoundingMode.UNNECESSARY)); + decimal256Vector2.setSafe( + 2, + new BigDecimal("54321098765432109876.5432109876").setScale(10, RoundingMode.UNNECESSARY)); + + File dataFile = new File(TMP, "testRoundTripDecimals.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripNullableDecimals() throws Exception { + + // Field definitions + FieldType decimal128Field1 = new FieldType(true, new ArrowType.Decimal(38, 10, 128), null); + FieldType decimal128Field2 = new FieldType(true, new ArrowType.Decimal(38, 5, 128), null); + FieldType decimal256Field1 = new FieldType(true, new ArrowType.Decimal(76, 20, 256), null); + FieldType decimal256Field2 = new FieldType(true, new ArrowType.Decimal(76, 10, 256), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + DecimalVector decimal128Vector1 = + new DecimalVector(new Field("decimal128_1", decimal128Field1, null), allocator); + DecimalVector decimal128Vector2 = + new DecimalVector(new Field("decimal128_2", decimal128Field2, null), allocator); + Decimal256Vector decimal256Vector1 = + new Decimal256Vector(new Field("decimal256_1", decimal256Field1, null), allocator); + Decimal256Vector decimal256Vector2 = + new Decimal256Vector(new Field("decimal256_2", decimal256Field2, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = + Arrays.asList(decimal128Vector1, decimal128Vector2, decimal256Vector1, decimal256Vector2); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + decimal128Vector1.setNull(0); + decimal128Vector1.setSafe(1, BigDecimal.ZERO.setScale(10, RoundingMode.UNNECESSARY)); + decimal128Vector1.setSafe( + 2, new BigDecimal("12345.67890").setScale(10, RoundingMode.UNNECESSARY)); + + decimal128Vector2.setNull(0); + decimal128Vector2.setSafe(1, BigDecimal.ZERO.setScale(5, RoundingMode.UNNECESSARY)); + decimal128Vector2.setSafe( + 2, new BigDecimal("98765.43210").setScale(5, RoundingMode.UNNECESSARY)); + + decimal256Vector1.setNull(0); + decimal256Vector1.setSafe(1, BigDecimal.ZERO.setScale(20, RoundingMode.UNNECESSARY)); + decimal256Vector1.setSafe( + 2, + new BigDecimal("12345678901234567890.12345678901234567890") + .setScale(20, RoundingMode.UNNECESSARY)); + + decimal256Vector2.setNull(0); + decimal256Vector2.setSafe(1, BigDecimal.ZERO.setScale(10, RoundingMode.UNNECESSARY)); + decimal256Vector2.setSafe( + 2, + new BigDecimal("98765432109876543210.9876543210").setScale(10, RoundingMode.UNNECESSARY)); + + File dataFile = new File(TMP, "testRoundTripNullableDecimals.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripDates() throws Exception { + + // Field definitions + FieldType dateDayField = new FieldType(false, new ArrowType.Date(DateUnit.DAY), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + DateDayVector dateDayVector = + new DateDayVector(new Field("dateDay", dateDayField, null), allocator); + + // Set up VSR + List vectors = Arrays.asList(dateDayVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + dateDayVector.setSafe(0, (int) LocalDate.now().toEpochDay()); + dateDayVector.setSafe(1, (int) LocalDate.now().toEpochDay() + 1); + dateDayVector.setSafe(2, (int) LocalDate.now().toEpochDay() + 2); + + File dataFile = new File(TMP, "testRoundTripDates.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripNullableDates() throws Exception { + + // Field definitions + FieldType dateDayField = new FieldType(true, new ArrowType.Date(DateUnit.DAY), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + DateDayVector dateDayVector = + new DateDayVector(new Field("dateDay", dateDayField, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = Arrays.asList(dateDayVector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + dateDayVector.setNull(0); + dateDayVector.setSafe(1, 0); + dateDayVector.setSafe(2, (int) LocalDate.now().toEpochDay()); + + File dataFile = new File(TMP, "testRoundTripNullableDates.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripTimes() throws Exception { + + // Field definitions + FieldType timeMillisField = + new FieldType(false, new ArrowType.Time(TimeUnit.MILLISECOND, 32), null); + FieldType timeMicrosField = + new FieldType(false, new ArrowType.Time(TimeUnit.MICROSECOND, 64), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + TimeMilliVector timeMillisVector = + new TimeMilliVector(new Field("timeMillis", timeMillisField, null), allocator); + TimeMicroVector timeMicrosVector = + new TimeMicroVector(new Field("timeMicros", timeMicrosField, null), allocator); + + // Set up VSR + List vectors = Arrays.asList(timeMillisVector, timeMicrosVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + timeMillisVector.setSafe( + 0, (int) (ZonedDateTime.now().toLocalTime().toNanoOfDay() / 1000000)); + timeMillisVector.setSafe( + 1, (int) (ZonedDateTime.now().toLocalTime().toNanoOfDay() / 1000000) - 1000); + timeMillisVector.setSafe( + 2, (int) (ZonedDateTime.now().toLocalTime().toNanoOfDay() / 1000000) - 2000); + + timeMicrosVector.setSafe(0, ZonedDateTime.now().toLocalTime().toNanoOfDay() / 1000); + timeMicrosVector.setSafe(1, ZonedDateTime.now().toLocalTime().toNanoOfDay() / 1000 - 1000000); + timeMicrosVector.setSafe(2, ZonedDateTime.now().toLocalTime().toNanoOfDay() / 1000 - 2000000); + + File dataFile = new File(TMP, "testRoundTripTimes.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripNullableTimes() throws Exception { + + // Field definitions + FieldType timeMillisField = + new FieldType(true, new ArrowType.Time(TimeUnit.MILLISECOND, 32), null); + FieldType timeMicrosField = + new FieldType(true, new ArrowType.Time(TimeUnit.MICROSECOND, 64), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + TimeMilliVector timeMillisVector = + new TimeMilliVector(new Field("timeMillis", timeMillisField, null), allocator); + TimeMicroVector timeMicrosVector = + new TimeMicroVector(new Field("timeMicros", timeMicrosField, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = Arrays.asList(timeMillisVector, timeMicrosVector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + timeMillisVector.setNull(0); + timeMillisVector.setSafe(1, 0); + timeMillisVector.setSafe( + 2, (int) (ZonedDateTime.now().toLocalTime().toNanoOfDay() / 1000000)); + + timeMicrosVector.setNull(0); + timeMicrosVector.setSafe(1, 0); + timeMicrosVector.setSafe(2, ZonedDateTime.now().toLocalTime().toNanoOfDay() / 1000); + + File dataFile = new File(TMP, "testRoundTripNullableTimes.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripZoneAwareTimestamps() throws Exception { + + // Field definitions + FieldType timestampMillisField = + new FieldType(false, new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC"), null); + FieldType timestampMicrosField = + new FieldType(false, new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC"), null); + FieldType timestampNanosField = + new FieldType(false, new ArrowType.Timestamp(TimeUnit.NANOSECOND, "UTC"), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + TimeStampMilliTZVector timestampMillisVector = + new TimeStampMilliTZVector( + new Field("timestampMillis", timestampMillisField, null), allocator); + TimeStampMicroTZVector timestampMicrosVector = + new TimeStampMicroTZVector( + new Field("timestampMicros", timestampMicrosField, null), allocator); + TimeStampNanoTZVector timestampNanosVector = + new TimeStampNanoTZVector( + new Field("timestampNanos", timestampNanosField, null), allocator); + + // Set up VSR + List vectors = + Arrays.asList(timestampMillisVector, timestampMicrosVector, timestampNanosVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + timestampMillisVector.setSafe(0, (int) Instant.now().toEpochMilli()); + timestampMillisVector.setSafe(1, (int) Instant.now().toEpochMilli() - 1000); + timestampMillisVector.setSafe(2, (int) Instant.now().toEpochMilli() - 2000); + + timestampMicrosVector.setSafe(0, Instant.now().toEpochMilli() * 1000); + timestampMicrosVector.setSafe(1, (Instant.now().toEpochMilli() - 1000) * 1000); + timestampMicrosVector.setSafe(2, (Instant.now().toEpochMilli() - 2000) * 1000); + + timestampNanosVector.setSafe(0, Instant.now().toEpochMilli() * 1000000); + timestampNanosVector.setSafe(1, (Instant.now().toEpochMilli() - 1000) * 1000000); + timestampNanosVector.setSafe(2, (Instant.now().toEpochMilli() - 2000) * 1000000); + + File dataFile = new File(TMP, "testRoundTripZoneAwareTimestamps.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripNullableZoneAwareTimestamps() throws Exception { + + // Field definitions + FieldType timestampMillisField = + new FieldType(true, new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC"), null); + FieldType timestampMicrosField = + new FieldType(true, new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC"), null); + FieldType timestampNanosField = + new FieldType(true, new ArrowType.Timestamp(TimeUnit.NANOSECOND, "UTC"), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + TimeStampMilliTZVector timestampMillisVector = + new TimeStampMilliTZVector( + new Field("timestampMillis", timestampMillisField, null), allocator); + TimeStampMicroTZVector timestampMicrosVector = + new TimeStampMicroTZVector( + new Field("timestampMicros", timestampMicrosField, null), allocator); + TimeStampNanoTZVector timestampNanosVector = + new TimeStampNanoTZVector( + new Field("timestampNanos", timestampNanosField, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = + Arrays.asList(timestampMillisVector, timestampMicrosVector, timestampNanosVector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + timestampMillisVector.setNull(0); + timestampMillisVector.setSafe(1, 0); + timestampMillisVector.setSafe(2, (int) Instant.now().toEpochMilli()); + + timestampMicrosVector.setNull(0); + timestampMicrosVector.setSafe(1, 0); + timestampMicrosVector.setSafe(2, Instant.now().toEpochMilli() * 1000); + + timestampNanosVector.setNull(0); + timestampNanosVector.setSafe(1, 0); + timestampNanosVector.setSafe(2, Instant.now().toEpochMilli() * 1000000); + + File dataFile = new File(TMP, "testRoundTripNullableZoneAwareTimestamps.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripLocalTimestamps() throws Exception { + + // Field definitions + FieldType timestampMillisField = + new FieldType(false, new ArrowType.Timestamp(TimeUnit.MILLISECOND, null), null); + FieldType timestampMicrosField = + new FieldType(false, new ArrowType.Timestamp(TimeUnit.MICROSECOND, null), null); + FieldType timestampNanosField = + new FieldType(false, new ArrowType.Timestamp(TimeUnit.NANOSECOND, null), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + TimeStampMilliVector timestampMillisVector = + new TimeStampMilliVector( + new Field("timestampMillis", timestampMillisField, null), allocator); + TimeStampMicroVector timestampMicrosVector = + new TimeStampMicroVector( + new Field("timestampMicros", timestampMicrosField, null), allocator); + TimeStampNanoVector timestampNanosVector = + new TimeStampNanoVector(new Field("timestampNanos", timestampNanosField, null), allocator); + + // Set up VSR + List vectors = + Arrays.asList(timestampMillisVector, timestampMicrosVector, timestampNanosVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + timestampMillisVector.setSafe(0, (int) Instant.now().toEpochMilli()); + timestampMillisVector.setSafe(1, (int) Instant.now().toEpochMilli() - 1000); + timestampMillisVector.setSafe(2, (int) Instant.now().toEpochMilli() - 2000); + + timestampMicrosVector.setSafe(0, Instant.now().toEpochMilli() * 1000); + timestampMicrosVector.setSafe(1, (Instant.now().toEpochMilli() - 1000) * 1000); + timestampMicrosVector.setSafe(2, (Instant.now().toEpochMilli() - 2000) * 1000); + + timestampNanosVector.setSafe(0, Instant.now().toEpochMilli() * 1000000); + timestampNanosVector.setSafe(1, (Instant.now().toEpochMilli() - 1000) * 1000000); + timestampNanosVector.setSafe(2, (Instant.now().toEpochMilli() - 2000) * 1000000); + + File dataFile = new File(TMP, "testRoundTripLocalTimestamps.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripNullableLocalTimestamps() throws Exception { + + // Field definitions + FieldType timestampMillisField = + new FieldType(true, new ArrowType.Timestamp(TimeUnit.MILLISECOND, null), null); + FieldType timestampMicrosField = + new FieldType(true, new ArrowType.Timestamp(TimeUnit.MICROSECOND, null), null); + FieldType timestampNanosField = + new FieldType(true, new ArrowType.Timestamp(TimeUnit.NANOSECOND, null), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + TimeStampMilliVector timestampMillisVector = + new TimeStampMilliVector( + new Field("timestampMillis", timestampMillisField, null), allocator); + TimeStampMicroVector timestampMicrosVector = + new TimeStampMicroVector( + new Field("timestampMicros", timestampMicrosField, null), allocator); + TimeStampNanoVector timestampNanosVector = + new TimeStampNanoVector(new Field("timestampNanos", timestampNanosField, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = + Arrays.asList(timestampMillisVector, timestampMicrosVector, timestampNanosVector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + timestampMillisVector.setNull(0); + timestampMillisVector.setSafe(1, 0); + timestampMillisVector.setSafe(2, (int) Instant.now().toEpochMilli()); + + timestampMicrosVector.setNull(0); + timestampMicrosVector.setSafe(1, 0); + timestampMicrosVector.setSafe(2, Instant.now().toEpochMilli() * 1000); + + timestampNanosVector.setNull(0); + timestampNanosVector.setSafe(1, 0); + timestampNanosVector.setSafe(2, Instant.now().toEpochMilli() * 1000000); + + File dataFile = new File(TMP, "testRoundTripNullableLocalTimestamps.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + // Data round trip for containers of primitive and logical types, nullable and non-nullable + + @Test + public void testRoundTripLists() throws Exception { + + // Field definitions + FieldType intListField = new FieldType(false, new ArrowType.List(), null); + FieldType stringListField = new FieldType(false, new ArrowType.List(), null); + FieldType dateListField = new FieldType(false, new ArrowType.List(), null); + + Field intField = new Field("item", FieldType.notNullable(new ArrowType.Int(32, true)), null); + Field stringField = new Field("item", FieldType.notNullable(new ArrowType.Utf8()), null); + Field dateField = + new Field("item", FieldType.notNullable(new ArrowType.Date(DateUnit.DAY)), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + ListVector intListVector = new ListVector("intList", allocator, intListField, null); + ListVector stringListVector = new ListVector("stringList", allocator, stringListField, null); + ListVector dateListVector = new ListVector("dateList", allocator, dateListField, null); + + intListVector.initializeChildrenFromFields(Arrays.asList(intField)); + stringListVector.initializeChildrenFromFields(Arrays.asList(stringField)); + dateListVector.initializeChildrenFromFields(Arrays.asList(dateField)); + + // Set up VSR + List vectors = Arrays.asList(intListVector, stringListVector, dateListVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + FieldWriter intListWriter = intListVector.getWriter(); + FieldWriter stringListWriter = stringListVector.getWriter(); + FieldWriter dateListWriter = dateListVector.getWriter(); + + // Set test data for intList + for (int i = 0; i < rowCount; i++) { + intListWriter.startList(); + for (int j = 0; j < 5 - i; j++) { + intListWriter.writeInt(j); + } + intListWriter.endList(); + } + + // Set test data for stringList + for (int i = 0; i < rowCount; i++) { + stringListWriter.startList(); + for (int j = 0; j < 5 - i; j++) { + stringListWriter.writeVarChar("string" + j); + } + stringListWriter.endList(); + } + + // Set test data for dateList + for (int i = 0; i < rowCount; i++) { + dateListWriter.startList(); + for (int j = 0; j < 5 - i; j++) { + dateListWriter.writeDateDay((int) LocalDate.now().plusDays(j).toEpochDay()); + } + dateListWriter.endList(); + } + + // Update count for the vectors + intListVector.setValueCount(rowCount); + stringListVector.setValueCount(rowCount); + dateListVector.setValueCount(rowCount); + + File dataFile = new File(TMP, "testRoundTripLists.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripNullableLists() throws Exception { + + // Field definitions + FieldType nullListType = new FieldType(true, new ArrowType.List(), null); + FieldType nonNullListType = new FieldType(false, new ArrowType.List(), null); + + Field nullFieldType = new Field("item", FieldType.nullable(new ArrowType.Int(32, true)), null); + Field nonNullFieldType = + new Field("item", FieldType.notNullable(new ArrowType.Int(32, true)), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + ListVector nullEntriesVector = + new ListVector("nullEntriesVector", allocator, nonNullListType, null); + ListVector nullListVector = new ListVector("nullListVector", allocator, nullListType, null); + ListVector nullBothVector = new ListVector("nullBothVector", allocator, nullListType, null); + + nullEntriesVector.initializeChildrenFromFields(Arrays.asList(nullFieldType)); + nullListVector.initializeChildrenFromFields(Arrays.asList(nonNullFieldType)); + nullBothVector.initializeChildrenFromFields(Arrays.asList(nullFieldType)); + + // Set up VSR + List vectors = Arrays.asList(nullEntriesVector, nullListVector, nullBothVector); + int rowCount = 4; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data for nullEntriesVector + FieldWriter nullEntriesWriter = nullEntriesVector.getWriter(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeNull(); + nullEntriesWriter.integer().writeNull(); + nullEntriesWriter.endList(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeInt(0); + nullEntriesWriter.integer().writeInt(0); + nullEntriesWriter.endList(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeInt(123); + nullEntriesWriter.integer().writeInt(456); + nullEntriesWriter.endList(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeInt(789); + nullEntriesWriter.integer().writeInt(789); + nullEntriesWriter.endList(); + + // Set test data for nullListVector + FieldWriter nullListWriter = nullListVector.getWriter(); + nullListWriter.writeNull(); + nullListWriter.setPosition(1); // writeNull() does not inc. idx() on list vector + nullListWriter.startList(); + nullListWriter.integer().writeInt(0); + nullListWriter.integer().writeInt(0); + nullListWriter.endList(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeInt(123); + nullEntriesWriter.integer().writeInt(456); + nullEntriesWriter.endList(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeInt(789); + nullEntriesWriter.integer().writeInt(789); + nullEntriesWriter.endList(); + + // Set test data for nullBothVector + FieldWriter nullBothWriter = nullBothVector.getWriter(); + nullBothWriter.writeNull(); + nullBothWriter.setPosition(1); + nullBothWriter.startList(); + nullBothWriter.integer().writeNull(); + nullBothWriter.integer().writeNull(); + nullBothWriter.endList(); + nullListWriter.startList(); + nullListWriter.integer().writeInt(0); + nullListWriter.integer().writeInt(0); + nullListWriter.endList(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeInt(123); + nullEntriesWriter.integer().writeInt(456); + nullEntriesWriter.endList(); + + // Update count for the vectors + nullListVector.setValueCount(4); + nullEntriesVector.setValueCount(4); + nullBothVector.setValueCount(4); + + File dataFile = new File(TMP, "testRoundTripNullableLists.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripMap() throws Exception { + + // Field definitions + FieldType intMapField = new FieldType(false, new ArrowType.Map(false), null); + FieldType stringMapField = new FieldType(false, new ArrowType.Map(false), null); + FieldType dateMapField = new FieldType(false, new ArrowType.Map(false), null); + + Field keyField = new Field("key", FieldType.notNullable(new ArrowType.Utf8()), null); + Field intField = new Field("value", FieldType.notNullable(new ArrowType.Int(32, true)), null); + Field stringField = new Field("value", FieldType.notNullable(new ArrowType.Utf8()), null); + Field dateField = + new Field("value", FieldType.notNullable(new ArrowType.Date(DateUnit.DAY)), null); + + Field intEntryField = + new Field( + "entries", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList(keyField, intField)); + Field stringEntryField = + new Field( + "entries", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList(keyField, stringField)); + Field dateEntryField = + new Field( + "entries", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList(keyField, dateField)); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + MapVector intMapVector = new MapVector("intMap", allocator, intMapField, null); + MapVector stringMapVector = new MapVector("stringMap", allocator, stringMapField, null); + MapVector dateMapVector = new MapVector("dateMap", allocator, dateMapField, null); + + intMapVector.initializeChildrenFromFields(Arrays.asList(intEntryField)); + stringMapVector.initializeChildrenFromFields(Arrays.asList(stringEntryField)); + dateMapVector.initializeChildrenFromFields(Arrays.asList(dateEntryField)); + + // Set up VSR + List vectors = Arrays.asList(intMapVector, stringMapVector, dateMapVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Total number of entries that will be writen to each vector + int entryCount = 5 + 4 + 3; + + // Set test data for intList + BaseWriter.MapWriter writer = intMapVector.getWriter(); + for (int i = 0; i < rowCount; i++) { + writer.startMap(); + for (int j = 0; j < 5 - i; j++) { + writer.startEntry(); + writer.key().varChar().writeVarChar("key" + j); + writer.value().integer().writeInt(j); + writer.endEntry(); + } + writer.endMap(); + } + + // Update count for data vector (map writer does not do this) + intMapVector.getDataVector().setValueCount(entryCount); + + // Set test data for stringList + BaseWriter.MapWriter stringWriter = stringMapVector.getWriter(); + for (int i = 0; i < rowCount; i++) { + stringWriter.startMap(); + for (int j = 0; j < 5 - i; j++) { + stringWriter.startEntry(); + stringWriter.key().varChar().writeVarChar("key" + j); + stringWriter.value().varChar().writeVarChar("string" + j); + stringWriter.endEntry(); + } + stringWriter.endMap(); + } + + // Update count for the vectors + intMapVector.setValueCount(rowCount); + stringMapVector.setValueCount(rowCount); + dateMapVector.setValueCount(rowCount); + + // Update count for data vector (map writer does not do this) + stringMapVector.getDataVector().setValueCount(entryCount); + + // Set test data for dateList + BaseWriter.MapWriter dateWriter = dateMapVector.getWriter(); + for (int i = 0; i < rowCount; i++) { + dateWriter.startMap(); + for (int j = 0; j < 5 - i; j++) { + dateWriter.startEntry(); + dateWriter.key().varChar().writeVarChar("key" + j); + dateWriter.value().dateDay().writeDateDay((int) LocalDate.now().plusDays(j).toEpochDay()); + dateWriter.endEntry(); + } + dateWriter.endMap(); + } + + // Update count for data vector (map writer does not do this) + dateMapVector.getDataVector().setValueCount(entryCount); + + File dataFile = new File(TMP, "testRoundTripMap.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripNullableMap() throws Exception { + + // Field definitions + FieldType nullMapType = new FieldType(true, new ArrowType.Map(false), null); + FieldType nonNullMapType = new FieldType(false, new ArrowType.Map(false), null); + + Field keyField = new Field("key", FieldType.notNullable(new ArrowType.Utf8()), null); + Field nullFieldType = new Field("value", FieldType.nullable(new ArrowType.Int(32, true)), null); + Field nonNullFieldType = + new Field("value", FieldType.notNullable(new ArrowType.Int(32, true)), null); + Field nullEntryField = + new Field( + "entries", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList(keyField, nullFieldType)); + Field nonNullEntryField = + new Field( + "entries", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList(keyField, nonNullFieldType)); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + MapVector nullEntriesVector = + new MapVector("nullEntriesVector", allocator, nonNullMapType, null); + MapVector nullMapVector = new MapVector("nullMapVector", allocator, nullMapType, null); + MapVector nullBothVector = new MapVector("nullBothVector", allocator, nullMapType, null); + + nullEntriesVector.initializeChildrenFromFields(Arrays.asList(nullEntryField)); + nullMapVector.initializeChildrenFromFields(Arrays.asList(nonNullEntryField)); + nullBothVector.initializeChildrenFromFields(Arrays.asList(nullEntryField)); + + // Set up VSR + List vectors = Arrays.asList(nullEntriesVector, nullMapVector, nullBothVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data for intList + BaseWriter.MapWriter writer = nullEntriesVector.getWriter(); + writer.startMap(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key0"); + writer.value().integer().writeNull(); + writer.endEntry(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key1"); + writer.value().integer().writeNull(); + writer.endEntry(); + writer.endMap(); + writer.startMap(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key2"); + writer.value().integer().writeInt(0); + writer.endEntry(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key3"); + writer.value().integer().writeInt(0); + writer.endEntry(); + writer.endMap(); + writer.startMap(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key4"); + writer.value().integer().writeInt(123); + writer.endEntry(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key5"); + writer.value().integer().writeInt(456); + writer.endEntry(); + writer.endMap(); + + // Set test data for stringList + BaseWriter.MapWriter nullMapWriter = nullMapVector.getWriter(); + nullMapWriter.writeNull(); + nullMapWriter.setPosition(1); // writeNull() does not inc. idx() on map (list) vector + nullMapWriter.startMap(); + nullMapWriter.startEntry(); + nullMapWriter.key().varChar().writeVarChar("key2"); + nullMapWriter.value().integer().writeInt(0); + nullMapWriter.endEntry(); + writer.startMap(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key3"); + writer.value().integer().writeInt(0); + writer.endEntry(); + nullMapWriter.endMap(); + nullMapWriter.startMap(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key4"); + writer.value().integer().writeInt(123); + writer.endEntry(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key5"); + writer.value().integer().writeInt(456); + writer.endEntry(); + nullMapWriter.endMap(); + + // Set test data for dateList + BaseWriter.MapWriter nullBothWriter = nullBothVector.getWriter(); + nullBothWriter.writeNull(); + nullBothWriter.setPosition(1); + nullBothWriter.startMap(); + nullBothWriter.startEntry(); + nullBothWriter.key().varChar().writeVarChar("key2"); + nullBothWriter.value().integer().writeNull(); + nullBothWriter.endEntry(); + nullBothWriter.startEntry(); + nullBothWriter.key().varChar().writeVarChar("key3"); + nullBothWriter.value().integer().writeNull(); + nullBothWriter.endEntry(); + nullBothWriter.endMap(); + nullBothWriter.startMap(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key4"); + writer.value().integer().writeInt(123); + writer.endEntry(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key5"); + writer.value().integer().writeInt(456); + writer.endEntry(); + nullBothWriter.endMap(); + + // Update count for the vectors + nullEntriesVector.setValueCount(3); + nullMapVector.setValueCount(3); + nullBothVector.setValueCount(3); + + File dataFile = new File(TMP, "testRoundTripNullableMap.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripStruct() throws Exception { + + // Field definitions + FieldType structFieldType = new FieldType(false, new ArrowType.Struct(), null); + Field intField = + new Field("intField", FieldType.notNullable(new ArrowType.Int(32, true)), null); + Field stringField = new Field("stringField", FieldType.notNullable(new ArrowType.Utf8()), null); + Field dateField = + new Field("dateField", FieldType.notNullable(new ArrowType.Date(DateUnit.DAY)), null); + + // Create empty vector + BufferAllocator allocator = new RootAllocator(); + StructVector structVector = new StructVector("struct", allocator, structFieldType, null); + structVector.initializeChildrenFromFields(Arrays.asList(intField, stringField, dateField)); + + // Set up VSR + List vectors = Arrays.asList(structVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + BaseWriter.StructWriter structWriter = structVector.getWriter(); + + for (int i = 0; i < rowCount; i++) { + structWriter.start(); + structWriter.integer("intField").writeInt(i); + structWriter.varChar("stringField").writeVarChar("string" + i); + structWriter.dateDay("dateField").writeDateDay((int) LocalDate.now().toEpochDay() + i); + structWriter.end(); + } + + File dataFile = new File(TMP, "testRoundTripStruct.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripNullableStructs() throws Exception { + + // Field definitions + FieldType structFieldType = new FieldType(false, new ArrowType.Struct(), null); + FieldType nullableStructFieldType = new FieldType(true, new ArrowType.Struct(), null); + Field intField = + new Field("intField", FieldType.notNullable(new ArrowType.Int(32, true)), null); + Field nullableIntField = + new Field("nullableIntField", FieldType.nullable(new ArrowType.Int(32, true)), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + StructVector structVector = new StructVector("struct", allocator, structFieldType, null); + StructVector nullableStructVector = + new StructVector("nullableStruct", allocator, nullableStructFieldType, null); + structVector.initializeChildrenFromFields(Arrays.asList(intField, nullableIntField)); + nullableStructVector.initializeChildrenFromFields(Arrays.asList(intField, nullableIntField)); + + // Set up VSR + List vectors = Arrays.asList(structVector, nullableStructVector); + int rowCount = 4; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data for structVector + BaseWriter.StructWriter structWriter = structVector.getWriter(); + for (int i = 0; i < rowCount; i++) { + structWriter.setPosition(i); + structWriter.start(); + structWriter.integer("intField").writeInt(i); + if (i % 2 == 0) { + structWriter.integer("nullableIntField").writeInt(i * 10); + } else { + structWriter.integer("nullableIntField").writeNull(); + } + structWriter.end(); + } + + // Set test data for nullableStructVector + BaseWriter.StructWriter nullableStructWriter = nullableStructVector.getWriter(); + for (int i = 0; i < rowCount; i++) { + nullableStructWriter.setPosition(i); + if (i >= 2) { + nullableStructWriter.start(); + nullableStructWriter.integer("intField").writeInt(i); + if (i % 2 == 0) { + nullableStructWriter.integer("nullableIntField").writeInt(i * 10); + } else { + nullableStructWriter.integer("nullableIntField").writeNull(); + } + nullableStructWriter.end(); + } else { + nullableStructWriter.writeNull(); + } + } + + // Update count for the vector + structVector.setValueCount(rowCount); + nullableStructVector.setValueCount(rowCount); + + File dataFile = new File(TMP, "testRoundTripNullableStructs.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } +} diff --git a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripSchemaTest.java b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripSchemaTest.java new file mode 100644 index 0000000000..864e2c8b59 --- /dev/null +++ b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripSchemaTest.java @@ -0,0 +1,443 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.apache.arrow.vector.types.DateUnit; +import org.apache.arrow.vector.types.FloatingPointPrecision; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.avro.Schema; +import org.junit.jupiter.api.Test; + +public class RoundTripSchemaTest { + + private void doRoundTripTest(List fields) { + + AvroToArrowConfig config = new AvroToArrowConfig(null, 1, null, Collections.emptySet(), false); + + Schema avroSchema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord"); + org.apache.arrow.vector.types.pojo.Schema arrowSchema = + AvroToArrowUtils.createArrowSchema(avroSchema, config); + + // Compare string representations - equality not defined for logical types + assertEquals(fields, arrowSchema.getFields()); + } + + // Schema round trip for primitive types, nullable and non-nullable + + @Test + public void testRoundTripNullType() { + + List fields = + Arrays.asList(new Field("nullType", FieldType.notNullable(new ArrowType.Null()), null)); + + doRoundTripTest(fields); + } + + @Test + public void testRoundTripBooleanType() { + + List fields = + Arrays.asList( + new Field("nullableBool", FieldType.nullable(new ArrowType.Bool()), null), + new Field("nonNullableBool", FieldType.notNullable(new ArrowType.Bool()), null)); + + doRoundTripTest(fields); + } + + @Test + public void testRoundTripIntegerTypes() { + + AvroToArrowConfig config = new AvroToArrowConfig(null, 1, null, Collections.emptySet(), false); + + // Only round trip types with direct equivalent in Avro + + List fields = + Arrays.asList( + new Field("nullableInt32", FieldType.nullable(new ArrowType.Int(32, true)), null), + new Field("nonNullableInt32", FieldType.notNullable(new ArrowType.Int(32, true)), null), + new Field("nullableInt64", FieldType.nullable(new ArrowType.Int(64, true)), null), + new Field( + "nonNullableInt64", FieldType.notNullable(new ArrowType.Int(64, true)), null)); + + Schema avroSchema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord"); + org.apache.arrow.vector.types.pojo.Schema arrowSchema = + AvroToArrowUtils.createArrowSchema(avroSchema, config); + + // Exact match on fields after round trip + assertEquals(fields, arrowSchema.getFields()); + } + + @Test + public void testRoundTripFloatingPointTypes() { + + // Only round trip types with direct equivalent in Avro + + List fields = + Arrays.asList( + new Field( + "nullableFloat32", + FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)), + null), + new Field( + "nonNullableFloat32", + FieldType.notNullable(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)), + null), + new Field( + "nullableFloat64", + FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + null), + new Field( + "nonNullableFloat64", + FieldType.notNullable(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + null)); + + doRoundTripTest(fields); + } + + @Test + public void testRoundTripStringTypes() { + + List fields = + Arrays.asList( + new Field("nullableUtf8", FieldType.nullable(new ArrowType.Utf8()), null), + new Field("nonNullableUtf8", FieldType.notNullable(new ArrowType.Utf8()), null)); + + doRoundTripTest(fields); + } + + @Test + public void testRoundTripBinaryTypes() { + + List fields = + Arrays.asList( + new Field("nullableBinary", FieldType.nullable(new ArrowType.Binary()), null), + new Field("nonNullableBinary", FieldType.notNullable(new ArrowType.Binary()), null)); + + doRoundTripTest(fields); + } + + @Test + public void testRoundTripFixedSizeBinaryTypes() { + + List fields = + Arrays.asList( + new Field( + "nullableFixedSizeBinary", + FieldType.nullable(new ArrowType.FixedSizeBinary(10)), + null), + new Field( + "nonNullableFixedSizeBinary", + FieldType.notNullable(new ArrowType.FixedSizeBinary(10)), + null)); + + doRoundTripTest(fields); + } + + // Schema round trip for logical types, nullable and non-nullable + + @Test + public void testRoundTripDecimalTypes() { + + List fields = + Arrays.asList( + new Field( + "nullableDecimal128", FieldType.nullable(new ArrowType.Decimal(10, 2, 128)), null), + new Field( + "nonNullableDecimal1281", + FieldType.notNullable(new ArrowType.Decimal(10, 2, 128)), + null), + new Field( + "nonNullableDecimal1282", + FieldType.notNullable(new ArrowType.Decimal(15, 5, 128)), + null), + new Field( + "nonNullableDecimal1283", + FieldType.notNullable(new ArrowType.Decimal(20, 10, 128)), + null), + new Field( + "nullableDecimal256", FieldType.nullable(new ArrowType.Decimal(55, 15, 256)), null), + new Field( + "nonNullableDecimal2561", + FieldType.notNullable(new ArrowType.Decimal(55, 25, 256)), + null), + new Field( + "nonNullableDecimal2562", + FieldType.notNullable(new ArrowType.Decimal(25, 8, 256)), + null), + new Field( + "nonNullableDecimal2563", + FieldType.notNullable(new ArrowType.Decimal(60, 50, 256)), + null)); + + doRoundTripTest(fields); + } + + @Test + public void testRoundTripDateTypes() { + + List fields = + Arrays.asList( + new Field( + "nullableDateDay", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null), + new Field( + "nonNullableDateDay", + FieldType.notNullable(new ArrowType.Date(DateUnit.DAY)), + null)); + + doRoundTripTest(fields); + } + + @Test + public void testRoundTripTimeTypes() { + + List fields = + Arrays.asList( + new Field( + "nullableTimeMillis", + FieldType.nullable(new ArrowType.Time(TimeUnit.MILLISECOND, 32)), + null), + new Field( + "nonNullableTimeMillis", + FieldType.notNullable(new ArrowType.Time(TimeUnit.MILLISECOND, 32)), + null), + new Field( + "nullableTimeMicros", + FieldType.nullable(new ArrowType.Time(TimeUnit.MICROSECOND, 64)), + null), + new Field( + "nonNullableTimeMicros", + FieldType.notNullable(new ArrowType.Time(TimeUnit.MICROSECOND, 64)), + null)); + + doRoundTripTest(fields); + } + + @Test + public void testRoundTripZoneAwareTimestampTypes() { + + List fields = + Arrays.asList( + new Field( + "nullableTimestampMillisTz", + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC")), + null), + new Field( + "nonNullableTimestampMillisTz", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC")), + null), + new Field( + "nullableTimestampMicrosTz", + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC")), + null), + new Field( + "nonNullableTimestampMicrosTz", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC")), + null), + new Field( + "nullableTimestampNanosTz", + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.NANOSECOND, "UTC")), + null), + new Field( + "nonNullableTimestampNanosTz", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.NANOSECOND, "UTC")), + null)); + + doRoundTripTest(fields); + } + + @Test + public void testRoundTripLocalTimestampTypes() { + + List fields = + Arrays.asList( + new Field( + "nullableTimestampMillis", + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, null)), + null), + new Field( + "nonNullableTimestampMillis", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, null)), + null), + new Field( + "nullableTimestampMicros", + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MICROSECOND, null)), + null), + new Field( + "nonNullableTimestampMicros", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.MICROSECOND, null)), + null), + new Field( + "nullableTimestampNanos", + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.NANOSECOND, null)), + null), + new Field( + "nonNullableTimestampNanos", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.NANOSECOND, null)), + null)); + + doRoundTripTest(fields); + } + + // Schema round trip for complex types, where the contents are primitive and logical types + + @Test + public void testRoundTripListType() { + + List fields = + Arrays.asList( + new Field( + "nullableIntList", + FieldType.nullable(new ArrowType.List()), + Arrays.asList( + new Field("$data$", FieldType.nullable(new ArrowType.Int(32, true)), null))), + new Field( + "nullableDoubleList", + FieldType.nullable(new ArrowType.List()), + Arrays.asList( + new Field( + "$data$", + FieldType.notNullable( + new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + null))), + new Field( + "nonNullableDecimalList", + FieldType.notNullable(new ArrowType.List()), + Arrays.asList( + new Field( + "$data$", FieldType.nullable(new ArrowType.Decimal(10, 2, 128)), null))), + new Field( + "nonNullableTimestampList", + FieldType.notNullable(new ArrowType.List()), + Arrays.asList( + new Field( + "$data$", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC")), + null)))); + + doRoundTripTest(fields); + } + + @Test + public void testRoundTripMapType() { + + List fields = + Arrays.asList( + new Field( + "nullableMapWithNullableInt", + FieldType.nullable(new ArrowType.Map(false)), + Arrays.asList( + new Field( + "entries", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList( + new Field("key", FieldType.notNullable(new ArrowType.Utf8()), null), + new Field( + "value", FieldType.nullable(new ArrowType.Int(32, true)), null))))), + new Field( + "nullableMapWithNonNullableDouble", + FieldType.nullable(new ArrowType.Map(false)), + Arrays.asList( + new Field( + "entries", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList( + new Field("key", FieldType.notNullable(new ArrowType.Utf8()), null), + new Field( + "value", + FieldType.notNullable( + new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + null))))), + new Field( + "nonNullableMapWithNullableDecimal", + FieldType.notNullable(new ArrowType.Map(false)), + Arrays.asList( + new Field( + "entries", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList( + new Field("key", FieldType.notNullable(new ArrowType.Utf8()), null), + new Field( + "value", + FieldType.nullable(new ArrowType.Decimal(10, 2, 128)), + null))))), + new Field( + "nonNullableMapWithNonNullableTimestamp", + FieldType.notNullable(new ArrowType.Map(false)), + Arrays.asList( + new Field( + "entries", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList( + new Field("key", FieldType.notNullable(new ArrowType.Utf8()), null), + new Field( + "value", + FieldType.notNullable( + new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC")), + null)))))); + + doRoundTripTest(fields); + } + + @Test + public void testRoundTripStructType() { + + List fields = + Arrays.asList( + new Field( + "nullableRecord", + FieldType.nullable(new ArrowType.Struct()), + Arrays.asList( + new Field("field1", FieldType.nullable(new ArrowType.Int(32, true)), null), + new Field( + "field2", + FieldType.notNullable( + new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + null), + new Field( + "field3", FieldType.nullable(new ArrowType.Decimal(10, 2, 128)), null), + new Field( + "field4", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC")), + null))), + new Field( + "nonNullableRecord", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList( + new Field("field1", FieldType.nullable(new ArrowType.Int(32, true)), null), + new Field( + "field2", + FieldType.notNullable( + new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + null), + new Field( + "field3", FieldType.nullable(new ArrowType.Decimal(10, 2, 128)), null), + new Field( + "field4", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC")), + null)))); + + doRoundTripTest(fields); + } +} diff --git a/adapter/avro/src/test/resources/schema/logical/test_decimal_invalid1.avsc b/adapter/avro/src/test/resources/schema/logical/test_decimal_invalid1.avsc index 18d7d63fc7..c1867811c7 100644 --- a/adapter/avro/src/test/resources/schema/logical/test_decimal_invalid1.avsc +++ b/adapter/avro/src/test/resources/schema/logical/test_decimal_invalid1.avsc @@ -20,6 +20,6 @@ "name": "test", "type": "bytes", "logicalType" : "decimal", - "precision": 39, + "precision": 77, "scale": 2 } diff --git a/adapter/avro/src/test/resources/schema/logical/test_local_timestamp_micros.avsc b/adapter/avro/src/test/resources/schema/logical/test_local_timestamp_micros.avsc new file mode 100644 index 0000000000..db456e8a84 --- /dev/null +++ b/adapter/avro/src/test/resources/schema/logical/test_local_timestamp_micros.avsc @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "name": "test", + "type": "long", + "logicalType" : "local-timestamp-micros" +} diff --git a/adapter/avro/src/test/resources/schema/logical/test_local_timestamp_millis.avsc b/adapter/avro/src/test/resources/schema/logical/test_local_timestamp_millis.avsc new file mode 100644 index 0000000000..6a3cf9bccb --- /dev/null +++ b/adapter/avro/src/test/resources/schema/logical/test_local_timestamp_millis.avsc @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "name": "test", + "type": "long", + "logicalType" : "local-timestamp-millis" +} diff --git a/adapter/avro/src/test/resources/schema/logical/test_local_timestamp_nanos.avsc b/adapter/avro/src/test/resources/schema/logical/test_local_timestamp_nanos.avsc new file mode 100644 index 0000000000..96ca8bbfa4 --- /dev/null +++ b/adapter/avro/src/test/resources/schema/logical/test_local_timestamp_nanos.avsc @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "name": "test", + "type": "long", + "logicalType" : "local-timestamp-nanos" +} diff --git a/adapter/avro/src/test/resources/schema/logical/test_timestamp_nanos.avsc b/adapter/avro/src/test/resources/schema/logical/test_timestamp_nanos.avsc new file mode 100644 index 0000000000..9e05eab408 --- /dev/null +++ b/adapter/avro/src/test/resources/schema/logical/test_timestamp_nanos.avsc @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "name": "test", + "type": "long", + "logicalType" : "timestamp-nanos" +} From 0d296dffd39e14dbcf288c2da873777e08091cbd Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 23 Apr 2025 10:29:08 +0900 Subject: [PATCH 151/408] GH-587: [Release] Add .env description to dev/release/README.md (#724) ## What's Changed Add missing `dev/release/.env` description to `dev/release/README.md`. This also moves `GH_TOKEN` to `dev/release/.env`. Closes #587. --- dev/release/.env.example | 5 ++ dev/release/README.md | 99 +++++++++++++++++++++++-------------- dev/release/bump_version.sh | 7 +++ dev/release/release.sh | 7 +++ dev/release/release_rc.sh | 14 +++--- 5 files changed, 89 insertions(+), 43 deletions(-) diff --git a/dev/release/.env.example b/dev/release/.env.example index cc7fd58bfc..4a57e34ed3 100644 --- a/dev/release/.env.example +++ b/dev/release/.env.example @@ -15,6 +15,11 @@ # specific language governing permissions and limitations # under the License. +# The GitHub token to upload artifacts to GitHub Release. +# +# You must set this. +#GH_TOKEN=secret + # The GPG key ID to sign artifacts. The GPG key ID must be registered # to both of the followings: # diff --git a/dev/release/README.md b/dev/release/README.md index 9069a4fdaf..8aee0fd106 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -27,45 +27,38 @@ 4. Publish (detailed later) 5. Bump version for new development (detailed later) -### Bump version for new release +### Prepare release environment -Run `dev/release/bump_version.sh` on a working copy of your fork not -`git@github.com:apache/arrow-java`: +This step is needed only when you act as a release manager first time. -```console -$ git clone git@github.com:${YOUR_GITHUB_ACCOUNT}/arrow-java.git arrow-java.${YOUR_GITHUB_ACCOUNT} -$ cd arrow-java.${YOUR_GITHUB_ACCOUNT} -$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/bump_version.sh ${NEW_VERSION} -``` +We use the following variables in multiple steps: -Here is an example to bump version to 19.0.0: +* `GH_TOKEN`: GitHub personal access token to automate GitHub related + operations +* `GPG_KEY_ID`: PGP key ID that is used for signing official artifacts + by GnuPG -``` -$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/bump_version.sh 19.0.0 -``` +We use `dev/release/.env` to share these variables in multiple +steps. You can use `dev/release/.env.example` as a template: -It creates a feature branch and adds a commit that bumps version. This -opens a pull request from the feature branch by `gh pr create`. So you -need `gh` command and GitHub personal access token. +```console +$ cp dev/release/.env{.example,} +$ chmod go-r dev/release/.env +$ editor dev/release/.env +``` -See also: +See https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens +how to prepare GitHub personal access token for `GH_TOKEN`. -We need to merge the pull request before we cut a RC. If we try cut a -RC without merging the pull request, the script to cut a RC is failed. +Note that you also need to install `gh` command because our scripts +use `gh` command to use GitHub API. See +https://github.com/cli/cli#installation how to install `gh` +command. -### Prepare RC and vote - -You can use `dev/release/release_rc.sh`. - -Requirements to run `release_rc.sh`: - - * You must be an Apache Arrow committer or PMC member - * You must prepare your PGP key for signing - * You must configure Maven - -If you don't have a PGP key, -https://infra.apache.org/release-signing.html#generate may be helpful. +If you don't have a PGP key for `GPG_KEY_ID`, see +https://infra.apache.org/release-signing.html#genegrate how to +generate your PGP key. Your PGP key must be registered to the followings: @@ -85,6 +78,40 @@ $ head KEYS $ svn ci KEYS ``` +### Bump version for new release + +Run `dev/release/bump_version.sh` on a working copy of your fork not +`git@github.com:apache/arrow-java`: + +```console +$ git clone git@github.com:${YOUR_GITHUB_ACCOUNT}/arrow-java.git arrow-java.${YOUR_GITHUB_ACCOUNT} +$ cd arrow-java.${YOUR_GITHUB_ACCOUNT} +$ dev/release/bump_version.sh ${NEW_VERSION} +``` + +Here is an example to bump version to 19.0.0: + +``` +$ dev/release/bump_version.sh 19.0.0 +``` + +It creates a feature branch and adds a commit that bumps version. This +opens a pull request from the feature branch. + +We need to merge the pull request before we cut a RC. If we try +cutting a RC without merging the pull request, the script to cut a RC +is failed. + +### Prepare RC and vote + +You can use `dev/release/release_rc.sh`. + +Requirements to run `release_rc.sh`: + + * You must be an Apache Arrow committer or PMC member + * You must prepare your PGP key for signing + * You must configure Maven + Configure Maven to publish artifacts to Apache repositories. You will need to setup a master password at `~/.m2/settings-security.xml` and `~/.m2/settings.xml` as specified on [the Apache @@ -102,7 +129,7 @@ Run `dev/release/release_rc.sh` on a working copy of ```console $ git clone git@github.com:apache/arrow-java.git $ cd arrow-java -$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release_rc.sh ${RC} +$ dev/release/release_rc.sh ${RC} (Send a vote email to dev@arrow.apache.org. You can use a draft shown by release_rc.sh for the email.) ``` @@ -110,7 +137,7 @@ $ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release_rc.sh ${RC} Here is an example to release RC1: ```console -$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release_rc.sh 1 +$ dev/release/release_rc.sh 1 ``` The argument of `release_rc.sh` is the RC number. If RC1 has a @@ -128,13 +155,13 @@ Run `dev/release/release.sh` on a working copy of archive to apache.org: ```console -$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release.sh ${VERSION} ${RC} +$ dev/release/release.sh ${VERSION} ${RC} ``` Here is an example to release 19.0.0 RC1: ```console -$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release.sh 19.0.0 1 +$ dev/release/release.sh 19.0.0 1 ``` Add the release to ASF's report database via [Apache Committee Report @@ -160,13 +187,13 @@ Run `dev/release/bump_version.sh` on a working copy of your fork not ```console $ git clone git@github.com:${YOUR_GITHUB_ACCOUNT}/arrow-java.git arrow-java.${YOUR_GITHUB_ACCOUNT} $ cd arrow-java.${YOUR_GITHUB_ACCOUNT} -$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/bump_version.sh ${NEW_VERSION}-SNAPSHOT +$ dev/release/bump_version.sh ${NEW_VERSION}-SNAPSHOT ``` Here is an example to bump version to 19.0.1-SNAPSHOT: ``` -$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/bump_version.sh 19.0.0-SNAPSHOT +$ dev/release/bump_version.sh 19.0.0-SNAPSHOT ``` It creates a feature branch and adds a commit that bumps version. This diff --git a/dev/release/bump_version.sh b/dev/release/bump_version.sh index d1c127de35..458e930f98 100755 --- a/dev/release/bump_version.sh +++ b/dev/release/bump_version.sh @@ -31,6 +31,13 @@ fi version=$1 +if [ ! -f "${SOURCE_DIR}/.env" ]; then + echo "You must create ${SOURCE_DIR}/.env" + echo "You can use ${SOURCE_DIR}/.env.example as template" + exit 1 +fi +. "${SOURCE_DIR}/.env" + cd "${SOURCE_TOP_DIR}" git_origin_url="$(git remote get-url origin)" diff --git a/dev/release/release.sh b/dev/release/release.sh index c5e67907e2..70e1f96454 100755 --- a/dev/release/release.sh +++ b/dev/release/release.sh @@ -28,6 +28,13 @@ fi version=$1 rc=$2 +if [ ! -f "${SOURCE_DIR}/.env" ]; then + echo "You must create ${SOURCE_DIR}/.env" + echo "You can use ${SOURCE_DIR}/.env.example as template" + exit 1 +fi +. "${SOURCE_DIR}/.env" + git_origin_url="$(git remote get-url origin)" repository="${git_origin_url#*github.com?}" repository="${repository%.git}" diff --git a/dev/release/release_rc.sh b/dev/release/release_rc.sh index b0107dce85..ff77718b8d 100755 --- a/dev/release/release_rc.sh +++ b/dev/release/release_rc.sh @@ -36,6 +36,13 @@ rc=$1 : "${RELEASE_SIGN:=${RELEASE_DEFAULT}}" : "${RELEASE_UPLOAD:=${RELEASE_DEFAULT}}" +if [ ! -f "${SOURCE_DIR}/.env" ]; then + echo "You must create ${SOURCE_DIR}/.env" + echo "You can use ${SOURCE_DIR}/.env.example as template" + exit 1 +fi +. "${SOURCE_DIR}/.env" + cd "${SOURCE_TOP_DIR}" if [ "${RELEASE_PULL}" -gt 0 ] || [ "${RELEASE_PUSH_TAG}" -gt 0 ]; then @@ -84,13 +91,6 @@ artifacts_dir="apache-arrow-java-${version}-rc${rc}" signed_artifacts_dir="${artifacts_dir}-signed" if [ "${RELEASE_SIGN}" -gt 0 ]; then - if [ ! -f "${SOURCE_DIR}/.env" ]; then - echo "You must create ${SOURCE_DIR}/.env" - echo "You can use ${SOURCE_DIR}/.env.example as template" - exit 1 - fi - . "${SOURCE_DIR}/.env" - git_origin_url="$(git remote get-url origin)" repository="${git_origin_url#*github.com?}" repository="${repository%.git}" From 26630c38f563821aa40d1c5a6df30101dde8c171 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Giraldo?= Date: Sun, 27 Apr 2025 23:28:37 -0600 Subject: [PATCH 152/408] GH-463: Improve TZ support for JDBC driver (#464) This PR adds support for natively fetching `java.time.*` objects through the JDBC driver. DateVector - getObject(LocalDate.class) DateTimeVector - getObject(OffsetDateTime.class) - getObject(LocalDateTime.class) - getObject(ZonedDateTime.class) - getObject(Instant.class) TimeVector - getObject(LocalTime.class) This PR also changes the behavior for vectors that include TZ info. These will now return as `TIMESTAMP_WITH_TIMEZONE`. The behavior for different ways to access a TimeStampVector are as follows: | | Vector with TZ | Vector W/O TZ | |---------------------------------|------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------| | getTimestamp() | Get timestamp in the vector TZ | Get timestamp in UTC | | getTimestamp(calendar) | Get timestamp by adjusting from the vector TZ to the desired calendar TZ | Get timestamp by adjusting from UTC to the desired calendar TZ (a bug, IMO) | | getObject(LocalDateTime.class) | Get LocalDateTime by taking the timestamp at the vector TZ and taking the "wall-clock" time at that moment | Treat the epoch value in the vector as the "wall-clock" time | | getObject(Instant.class) | Get Instant represented by the value in the vector TZ | Get Instant represented by the value in UTC | | getObject(OffsetDateTime.class) | Get OffsetDateTime represented by the value in the vector TZ (will account for daylight adjustment) | Get OffsetDateTime represented by the value in UTC (will account for daylight adjustment) | | getObject(ZonedDateTime.class) | Get ZonedDateTime represented by the value in the vector at in its TZ | Get ZonedDateTime represented by the value in the vector at in UTC | Closes #463 --- .../ArrowFlightJdbcDateVectorAccessor.java | 19 +++ ...rrowFlightJdbcTimeStampVectorAccessor.java | 101 +++++++++++++-- .../ArrowFlightJdbcTimeVectorAccessor.java | 19 +++ .../arrow/driver/jdbc/utils/SqlTypes.java | 8 +- .../jdbc/ArrowDatabaseMetadataTest.java | 5 +- ...FlightJdbcTimeStampVectorAccessorTest.java | 118 +++++++++++++++++- .../arrow/driver/jdbc/utils/SqlTypesTest.java | 12 ++ 7 files changed, 269 insertions(+), 13 deletions(-) diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcDateVectorAccessor.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcDateVectorAccessor.java index ebe4016209..cdafeffc32 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcDateVectorAccessor.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcDateVectorAccessor.java @@ -24,7 +24,9 @@ import static org.apache.calcite.avatica.util.DateTimeUtils.unixDateToString; import java.sql.Date; +import java.sql.SQLException; import java.sql.Timestamp; +import java.time.LocalDate; import java.util.Calendar; import java.util.concurrent.TimeUnit; import java.util.function.IntSupplier; @@ -85,6 +87,19 @@ public Object getObject() { return this.getDate(null); } + @Override + public T getObject(final Class type) throws SQLException { + final Object value; + if (type == LocalDate.class) { + value = getLocalDate(); + } else if (type == Date.class) { + value = getObject(); + } else { + throw new SQLException("Object type not supported for Date Vector"); + } + return !type.isPrimitive() && wasNull ? null : type.cast(value); + } + @Override public Date getDate(Calendar calendar) { fillHolder(); @@ -134,4 +149,8 @@ protected static TimeUnit getTimeUnitForVector(ValueVector vector) { throw new IllegalArgumentException("Invalid Arrow vector"); } + + private LocalDate getLocalDate() { + return getDate(null).toLocalDate(); + } } diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeStampVectorAccessor.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeStampVectorAccessor.java index debdd0fcb4..813fbc7cfd 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeStampVectorAccessor.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeStampVectorAccessor.java @@ -21,11 +21,18 @@ import static org.apache.arrow.driver.jdbc.accessor.impl.calendar.ArrowFlightJdbcTimeStampVectorGetter.createGetter; import java.sql.Date; +import java.sql.SQLException; import java.sql.Time; import java.sql.Timestamp; +import java.time.Instant; import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.time.temporal.ChronoUnit; import java.util.Calendar; +import java.util.Objects; +import java.util.Set; import java.util.TimeZone; import java.util.concurrent.TimeUnit; import java.util.function.IntSupplier; @@ -43,6 +50,7 @@ public class ArrowFlightJdbcTimeStampVectorAccessor extends ArrowFlightJdbcAcces private final TimeUnit timeUnit; private final LongToLocalDateTime longToLocalDateTime; private final Holder holder; + private final boolean isZoned; /** Functional interface used to convert a number (in any time resolution) to LocalDateTime. */ interface LongToLocalDateTime { @@ -58,6 +66,9 @@ public ArrowFlightJdbcTimeStampVectorAccessor( this.holder = new Holder(); this.getter = createGetter(vector); + // whether the vector included TZ info + this.isZoned = getVectorIsZoned(vector); + // non-null, either the vector TZ or default to UTC this.timeZone = getTimeZoneForVector(vector); this.timeUnit = getTimeUnitForVector(vector); this.longToLocalDateTime = getLongToLocalDateTimeForVector(vector, this.timeZone); @@ -68,11 +79,62 @@ public Class getObjectClass() { return Timestamp.class; } + @Override + public T getObject(final Class type) throws SQLException { + final Object value; + if (!this.isZoned + & Set.of(OffsetDateTime.class, ZonedDateTime.class, Instant.class).contains(type)) { + throw new SQLException( + "Vectors without timezones can't be converted to objects with offset/tz info."); + } else if (type == OffsetDateTime.class) { + value = getOffsetDateTime(); + } else if (type == LocalDateTime.class) { + value = getLocalDateTime(null); + } else if (type == ZonedDateTime.class) { + value = getZonedDateTime(); + } else if (type == Instant.class) { + value = getInstant(); + } else if (type == Timestamp.class) { + value = getObject(); + } else { + throw new SQLException("Object type not supported for TimeStamp Vector"); + } + + return !type.isPrimitive() && wasNull ? null : type.cast(value); + } + @Override public Object getObject() { return this.getTimestamp(null); } + private ZonedDateTime getZonedDateTime() { + LocalDateTime localDateTime = getLocalDateTime(null); + if (localDateTime == null) { + return null; + } + + return localDateTime.atZone(this.timeZone.toZoneId()); + } + + private OffsetDateTime getOffsetDateTime() { + LocalDateTime localDateTime = getLocalDateTime(null); + if (localDateTime == null) { + return null; + } + ZoneOffset offset = this.timeZone.toZoneId().getRules().getOffset(localDateTime); + return localDateTime.atOffset(offset); + } + + private Instant getInstant() { + LocalDateTime localDateTime = getLocalDateTime(null); + if (localDateTime == null) { + return null; + } + ZoneOffset offset = this.timeZone.toZoneId().getRules().getOffset(localDateTime); + return localDateTime.toInstant(offset); + } + private LocalDateTime getLocalDateTime(Calendar calendar) { getter.get(getCurrentRow(), holder); this.wasNull = holder.isSet == 0; @@ -85,7 +147,9 @@ private LocalDateTime getLocalDateTime(Calendar calendar) { LocalDateTime localDateTime = this.longToLocalDateTime.fromLong(value); - if (calendar != null) { + // Adjust timestamp to desired calendar (if provided) only if the column includes TZ info, + // otherwise treat as wall-clock time + if (calendar != null && this.isZoned) { TimeZone timeZone = calendar.getTimeZone(); long millis = this.timeUnit.toMillis(value); localDateTime = @@ -102,7 +166,7 @@ public Date getDate(Calendar calendar) { return null; } - return new Date(Timestamp.valueOf(localDateTime).getTime()); + return new Date(getTimestampWithOffset(calendar, localDateTime).getTime()); } @Override @@ -112,7 +176,7 @@ public Time getTime(Calendar calendar) { return null; } - return new Time(Timestamp.valueOf(localDateTime).getTime()); + return new Time(getTimestampWithOffset(calendar, localDateTime).getTime()); } @Override @@ -122,6 +186,24 @@ public Timestamp getTimestamp(Calendar calendar) { return null; } + return getTimestampWithOffset(calendar, localDateTime); + } + + /** + * Apply offset to LocalDateTime to get a Timestamp with legacy behavior. Previously we applied + * the offset to the LocalDateTime even if the underlying Vector did not have a TZ. In order to + * support java.time.* accessors, we fixed this so we only apply the offset if the underlying + * vector includes TZ info. In order to maintain backward compatibility, we apply the offset if + * needed for getDate, getTime, and getTimestamp. + */ + private Timestamp getTimestampWithOffset(Calendar calendar, LocalDateTime localDateTime) { + if (calendar != null && !isZoned) { + TimeZone timeZone = calendar.getTimeZone(); + long millis = Timestamp.valueOf(localDateTime).getTime(); + localDateTime = + localDateTime.minus( + timeZone.getOffset(millis) - this.timeZone.getOffset(millis), ChronoUnit.MILLIS); + } return Timestamp.valueOf(localDateTime); } @@ -170,11 +252,14 @@ protected static TimeZone getTimeZoneForVector(TimeStampVector vector) { ArrowType.Timestamp arrowType = (ArrowType.Timestamp) vector.getField().getFieldType().getType(); - String timezoneName = arrowType.getTimezone(); - if (timezoneName == null) { - return TimeZone.getTimeZone("UTC"); - } - + String timezoneName = Objects.requireNonNullElse(arrowType.getTimezone(), "UTC"); return TimeZone.getTimeZone(timezoneName); } + + protected static boolean getVectorIsZoned(TimeStampVector vector) { + ArrowType.Timestamp arrowType = + (ArrowType.Timestamp) vector.getField().getFieldType().getType(); + + return arrowType.getTimezone() != null; + } } diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeVectorAccessor.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeVectorAccessor.java index 2c03ee631e..d525c2fdd2 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeVectorAccessor.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeVectorAccessor.java @@ -20,8 +20,10 @@ import static org.apache.arrow.driver.jdbc.accessor.impl.calendar.ArrowFlightJdbcTimeVectorGetter.Holder; import static org.apache.arrow.driver.jdbc.accessor.impl.calendar.ArrowFlightJdbcTimeVectorGetter.createGetter; +import java.sql.SQLException; import java.sql.Time; import java.sql.Timestamp; +import java.time.LocalTime; import java.util.Calendar; import java.util.concurrent.TimeUnit; import java.util.function.IntSupplier; @@ -121,6 +123,19 @@ public Object getObject() { return this.getTime(null); } + @Override + public T getObject(final Class type) throws SQLException { + final Object value; + if (type == LocalTime.class) { + value = getLocalTime(); + } else if (type == Time.class) { + value = getObject(); + } else { + throw new SQLException("Object type not supported for Time Vector"); + } + return !type.isPrimitive() && wasNull ? null : type.cast(value); + } + @Override public Time getTime(Calendar calendar) { fillHolder(); @@ -134,6 +149,10 @@ public Time getTime(Calendar calendar) { return new ArrowFlightJdbcTime(DateTimeUtils.applyCalendarOffset(milliseconds, calendar)); } + private LocalTime getLocalTime() { + return getTime(null).toLocalTime(); + } + private void fillHolder() { getter.get(getCurrentRow(), holder); this.wasNull = holder.isSet == 0; diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/SqlTypes.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/SqlTypes.java index 96cb056db2..1b76ca0c95 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/SqlTypes.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/SqlTypes.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.driver.jdbc.utils; +import com.google.common.base.Strings; import java.sql.Types; import java.util.HashMap; import java.util.Map; @@ -120,7 +121,12 @@ public static int getSqlTypeIdFromArrowType(ArrowType arrowType) { case Time: return Types.TIME; case Timestamp: - return Types.TIMESTAMP; + String tz = ((ArrowType.Timestamp) arrowType).getTimezone(); + if (Strings.isNullOrEmpty(tz)) { + return Types.TIMESTAMP; + } else { + return Types.TIMESTAMP_WITH_TIMEZONE; + } case Bool: return Types.BOOLEAN; case Decimal: diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java index 88a172e4f2..70d3bcbd33 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java @@ -299,8 +299,9 @@ public class ArrowDatabaseMetadataTest { private static Connection connection; static { - List expectedGetColumnsDataTypes = Arrays.asList(3, 93, 4); - List expectedGetColumnsTypeName = Arrays.asList("DECIMAL", "TIMESTAMP", "INTEGER"); + List expectedGetColumnsDataTypes = Arrays.asList(3, 2014, 4); + List expectedGetColumnsTypeName = + Arrays.asList("DECIMAL", "TIMESTAMP_WITH_TIMEZONE", "INTEGER"); List expectedGetColumnsRadix = Arrays.asList(10, null, 10); List expectedGetColumnsColumnSize = Arrays.asList(5, 29, 10); List expectedGetColumnsDecimalDigits = Arrays.asList(2, 9, 0); diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeStampVectorAccessorTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeStampVectorAccessorTest.java index 2e329f148e..e4863bd80e 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeStampVectorAccessorTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeStampVectorAccessorTest.java @@ -16,17 +16,23 @@ */ package org.apache.arrow.driver.jdbc.accessor.impl.calendar; -import static org.apache.arrow.driver.jdbc.accessor.impl.calendar.ArrowFlightJdbcTimeStampVectorAccessor.getTimeUnitForVector; -import static org.apache.arrow.driver.jdbc.accessor.impl.calendar.ArrowFlightJdbcTimeStampVectorAccessor.getTimeZoneForVector; +import static org.apache.arrow.driver.jdbc.accessor.impl.calendar.ArrowFlightJdbcTimeStampVectorAccessor.*; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.sql.Date; +import java.sql.SQLException; import java.sql.Time; import java.sql.Timestamp; +import java.time.Instant; import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.Calendar; +import java.util.Objects; import java.util.TimeZone; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; @@ -199,6 +205,99 @@ public void testShouldGetTimestampReturnValidTimestampWithCalendar( }); } + @ParameterizedTest + @MethodSource("data") + public void testShouldGetObjectReturnValidLocalDateTime( + Supplier vectorSupplier, String vectorType, String timeZone) + throws Exception { + setup(vectorSupplier); + final String expectedTimeZone = Objects.requireNonNullElse(timeZone, "UTC"); + + accessorIterator.iterate( + vector, + (accessor, currentRow) -> { + final LocalDateTime value = accessor.getObject(LocalDateTime.class); + final LocalDateTime expectedValue = + getZonedDateTime(currentRow, expectedTimeZone).toLocalDateTime(); + + assertThat(value, equalTo(expectedValue)); + assertThat(accessor.wasNull(), is(false)); + }); + } + + @ParameterizedTest + @MethodSource("data") + public void testShouldGetObjectReturnValidInstant( + Supplier vectorSupplier, String vectorType, String timeZone) + throws Exception { + setup(vectorSupplier); + final String expectedTimeZone = Objects.requireNonNullElse(timeZone, "UTC"); + final boolean vectorHasTz = timeZone != null; + accessorIterator.iterate( + vector, + (accessor, currentRow) -> { + if (vectorHasTz) { + final Instant value = accessor.getObject(Instant.class); + final Instant expectedValue = + getZonedDateTime(currentRow, expectedTimeZone).toInstant(); + + assertThat(value, equalTo(expectedValue)); + assertThat(accessor.wasNull(), is(false)); + } else { + assertThrows(SQLException.class, () -> accessor.getObject(Instant.class)); + } + }); + } + + @ParameterizedTest + @MethodSource("data") + public void testShouldGetObjectReturnValidOffsetDateTime( + Supplier vectorSupplier, String vectorType, String timeZone) + throws Exception { + setup(vectorSupplier); + final String expectedTimeZone = Objects.requireNonNullElse(timeZone, "UTC"); + final boolean vectorHasTz = timeZone != null; + accessorIterator.iterate( + vector, + (accessor, currentRow) -> { + if (vectorHasTz) { + final OffsetDateTime value = accessor.getObject(OffsetDateTime.class); + final OffsetDateTime expectedValue = + getZonedDateTime(currentRow, expectedTimeZone).toOffsetDateTime(); + + assertThat(value, equalTo(expectedValue)); + assertThat(value.getOffset(), equalTo(expectedValue.getOffset())); + assertThat(accessor.wasNull(), is(false)); + } else { + assertThrows(SQLException.class, () -> accessor.getObject(OffsetDateTime.class)); + } + }); + } + + @ParameterizedTest + @MethodSource("data") + public void testShouldGetObjectReturnValidZonedDateTime( + Supplier vectorSupplier, String vectorType, String timeZone) + throws Exception { + setup(vectorSupplier); + final String expectedTimeZone = Objects.requireNonNullElse(timeZone, "UTC"); + final boolean vectorHasTz = timeZone != null; + accessorIterator.iterate( + vector, + (accessor, currentRow) -> { + if (vectorHasTz) { + final ZonedDateTime value = accessor.getObject(ZonedDateTime.class); + final ZonedDateTime expectedValue = getZonedDateTime(currentRow, expectedTimeZone); + + assertThat(value, equalTo(expectedValue)); + assertThat(value.getZone(), equalTo(ZoneId.of(expectedTimeZone))); + assertThat(accessor.wasNull(), is(false)); + } else { + assertThrows(SQLException.class, () -> accessor.getObject(ZonedDateTime.class)); + } + }); + } + @ParameterizedTest @MethodSource("data") public void testShouldGetTimestampReturnNull(Supplier vectorSupplier) { @@ -320,6 +419,21 @@ private Timestamp getTimestampForVector(int currentRow, String timeZone) { return expectedTimestamp; } + /** ZonedDateTime contains all necessary information to generate any java.time object. */ + private ZonedDateTime getZonedDateTime(int currentRow, String timeZone) { + Object object = vector.getObject(currentRow); + TimeZone tz = TimeZone.getTimeZone(timeZone); + ZonedDateTime expectedTimestamp = null; + if (object instanceof LocalDateTime) { + expectedTimestamp = ((LocalDateTime) object).atZone(tz.toZoneId()); + } else if (object instanceof Long) { + TimeUnit timeUnit = getTimeUnitForVector(vector); + Instant instant = Instant.ofEpochMilli(timeUnit.toMillis((Long) object)); + expectedTimestamp = ZonedDateTime.ofInstant(instant, tz.toZoneId()); + } + return expectedTimestamp; + } + @ParameterizedTest @MethodSource("data") public void testShouldGetObjectClass(Supplier vectorSupplier) throws Exception { diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/SqlTypesTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/SqlTypesTest.java index 00af3c96ba..a6dd6b3275 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/SqlTypesTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/SqlTypesTest.java @@ -48,9 +48,15 @@ public void testGetSqlTypeIdFromArrowType() { assertEquals(Types.DATE, getSqlTypeIdFromArrowType(new ArrowType.Date(DateUnit.MILLISECOND))); assertEquals( Types.TIME, getSqlTypeIdFromArrowType(new ArrowType.Time(TimeUnit.MILLISECOND, 32))); + assertEquals( + Types.TIMESTAMP, + getSqlTypeIdFromArrowType(new ArrowType.Timestamp(TimeUnit.MILLISECOND, null))); assertEquals( Types.TIMESTAMP, getSqlTypeIdFromArrowType(new ArrowType.Timestamp(TimeUnit.MILLISECOND, ""))); + assertEquals( + Types.TIMESTAMP_WITH_TIMEZONE, + getSqlTypeIdFromArrowType(new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC"))); assertEquals(Types.BOOLEAN, getSqlTypeIdFromArrowType(new ArrowType.Bool())); @@ -95,9 +101,15 @@ public void testGetSqlTypeNameFromArrowType() { assertEquals("DATE", getSqlTypeNameFromArrowType(new ArrowType.Date(DateUnit.MILLISECOND))); assertEquals("TIME", getSqlTypeNameFromArrowType(new ArrowType.Time(TimeUnit.MILLISECOND, 32))); + assertEquals( + "TIMESTAMP", + getSqlTypeNameFromArrowType(new ArrowType.Timestamp(TimeUnit.MILLISECOND, null))); assertEquals( "TIMESTAMP", getSqlTypeNameFromArrowType(new ArrowType.Timestamp(TimeUnit.MILLISECOND, ""))); + assertEquals( + "TIMESTAMP_WITH_TIMEZONE", + getSqlTypeNameFromArrowType(new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC"))); assertEquals("BOOLEAN", getSqlTypeNameFromArrowType(new ArrowType.Bool())); From c2234549827488d32b6698f91e26053a293d808e Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Mon, 28 Apr 2025 14:23:46 +0800 Subject: [PATCH 153/408] GH-729: [JDBC] Fix BinaryConsumer consuming null value (#730) ## What's Changed Set `startOffset` of the next item when `BinaryConsumer` consuming `null` value. Closes #729 . --- .../adapter/jdbc/consumer/BinaryConsumer.java | 25 ++++++++++--------- .../jdbc/consumer/BinaryConsumerTest.java | 11 +++++++- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumer.java b/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumer.java index edbc6360df..73ec04b8a0 100644 --- a/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumer.java +++ b/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumer.java @@ -51,13 +51,15 @@ public BinaryConsumer(VarBinaryVector vector, int index) { /** consume a InputStream. */ public void consume(InputStream is) throws IOException { + while (currentIndex >= vector.getValueCapacity()) { + vector.reallocValidityAndOffsetBuffers(); + } + + final int startOffset = vector.getStartOffset(currentIndex); + final ArrowBuf offsetBuffer = vector.getOffsetBuffer(); + int dataLength = 0; + if (is != null) { - while (currentIndex >= vector.getValueCapacity()) { - vector.reallocValidityAndOffsetBuffers(); - } - final int startOffset = vector.getStartOffset(currentIndex); - final ArrowBuf offsetBuffer = vector.getOffsetBuffer(); - int dataLength = 0; int read; while ((read = is.read(reuseBytes)) != -1) { while (vector.getDataBuffer().capacity() < (startOffset + dataLength + read)) { @@ -66,11 +68,12 @@ public void consume(InputStream is) throws IOException { vector.getDataBuffer().setBytes(startOffset + dataLength, reuseBytes, 0, read); dataLength += read; } - offsetBuffer.setInt( - (currentIndex + 1) * ((long) VarBinaryVector.OFFSET_WIDTH), startOffset + dataLength); + BitVectorHelper.setBit(vector.getValidityBuffer(), currentIndex); - vector.setLastSet(currentIndex); } + offsetBuffer.setInt( + (currentIndex + 1) * ((long) VarBinaryVector.OFFSET_WIDTH), startOffset + dataLength); + vector.setLastSet(currentIndex); } public void moveWriterPosition() { @@ -95,9 +98,7 @@ public NullableBinaryConsumer(VarBinaryVector vector, int index) { @Override public void consume(ResultSet resultSet) throws SQLException, IOException { InputStream is = resultSet.getBinaryStream(columnIndexInResultSet); - if (!resultSet.wasNull()) { - consume(is); - } + consume(is); moveWriterPosition(); } } diff --git a/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumerTest.java b/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumerTest.java index b1e253794d..bb836578e2 100644 --- a/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumerTest.java +++ b/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumerTest.java @@ -22,6 +22,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import org.apache.arrow.vector.BaseValueVector; import org.apache.arrow.vector.VarBinaryVector; import org.junit.jupiter.api.Test; @@ -65,7 +66,11 @@ public void testConsumeInputStream(byte[][] values, boolean nullable) throws IOE nullable, binaryConsumer -> { for (byte[] value : values) { - binaryConsumer.consume(new ByteArrayInputStream(value)); + if (value != null) { + binaryConsumer.consume(new ByteArrayInputStream(value)); + } else { + binaryConsumer.consume((InputStream) null); + } binaryConsumer.moveWriterPosition(); } }, @@ -119,5 +124,9 @@ public void testConsumeInputStream() throws IOException { testRecords[i] = createBytes(DEFAULT_RECORD_BYTE_COUNT); } testConsumeInputStream(testRecords, false); + + byte[] bytes1 = new byte[] {1, 2, 3}; + byte[] bytes2 = new byte[] {4, 5, 6}; + testConsumeInputStream(new byte[][] {bytes1, null, bytes2}, true); } } From b9e37f0ccecc2651fec3487472c203bd223290e8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Apr 2025 17:32:40 +0900 Subject: [PATCH 154/408] MINOR: [CI] Bump actions/download-artifact from 4.2.1 to 4.3.0 (#733) Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 4.2.1 to 4.3.0. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 5b78cc9395..7e3cf5f6f2 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -101,7 +101,7 @@ jobs: packages: write steps: - name: Download source archive - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: release-source - name: Extract source archive @@ -174,7 +174,7 @@ jobs: MACOSX_DEPLOYMENT_TARGET: "14.0" steps: - name: Download source archive - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: release-source - name: Extract source archive @@ -298,7 +298,7 @@ jobs: arch: "x86_64" steps: - name: Download source archive - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: release-source - name: Extract source archive @@ -371,7 +371,7 @@ jobs: - jni-windows steps: - name: Download artifacts - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: path: artifacts - name: Decompress artifacts @@ -452,11 +452,11 @@ jobs: with: cache: 'pip' - name: Download source archive - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: release-source - name: Download Javadocs - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: reference - name: Extract source archive @@ -521,7 +521,7 @@ jobs: cp ../.asf.yaml ./ git add .nojekyll .asf.yaml - name: Download - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: release-html - name: Extract @@ -557,7 +557,7 @@ jobs: - ubuntu-latest steps: - name: Download release artifacts - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: pattern: release-* - name: Verify @@ -591,7 +591,7 @@ jobs: contents: write steps: - name: Download release artifacts - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: pattern: release-* path: artifacts From 8314c1912ed501171a2085feb1909b07892a6a25 Mon Sep 17 00:00:00 2001 From: Mateusz Rzeszutek Date: Sat, 3 May 2025 11:01:23 +0200 Subject: [PATCH 155/408] GH-737: [FlightSQL] Allow returning column remarks in FlightSQL's CommandGetTables (#727) Resolves #737 ## What's Changed This is an implementation of https://github.com/apache/arrow/pull/46110 for Java --- arrow-format/FlightSql.proto | 4 ++ .../tests/FlightSqlScenarioProducer.java | 2 + .../driver/jdbc/ArrowDatabaseMetadata.java | 6 +++ .../arrow/driver/jdbc/utils/ConvertUtils.java | 4 ++ .../jdbc/ArrowDatabaseMetadataTest.java | 53 ++++++++++++++----- .../driver/jdbc/utils/ConvertUtilsTest.java | 2 + .../flight/sql/FlightSqlColumnMetadata.java | 21 ++++++++ 7 files changed, 80 insertions(+), 12 deletions(-) diff --git a/arrow-format/FlightSql.proto b/arrow-format/FlightSql.proto index 3568d851cb..566230c2a6 100644 --- a/arrow-format/FlightSql.proto +++ b/arrow-format/FlightSql.proto @@ -1212,6 +1212,7 @@ message CommandGetDbSchemas { * - ARROW:FLIGHT:SQL:IS_CASE_SENSITIVE - "1" indicates if the column is case-sensitive, "0" otherwise. * - ARROW:FLIGHT:SQL:IS_READ_ONLY - "1" indicates if the column is read only, "0" otherwise. * - ARROW:FLIGHT:SQL:IS_SEARCHABLE - "1" indicates if the column is searchable via WHERE clause, "0" otherwise. + * - ARROW:FLIGHT:SQL:REMARKS - A comment describing the column. * The returned data should be ordered by catalog_name, db_schema_name, table_name, then table_type, followed by table_schema if requested. */ message CommandGetTables { @@ -1678,6 +1679,7 @@ message ActionEndSavepointRequest { * - ARROW:FLIGHT:SQL:IS_CASE_SENSITIVE - "1" indicates if the column is case-sensitive, "0" otherwise. * - ARROW:FLIGHT:SQL:IS_READ_ONLY - "1" indicates if the column is read only, "0" otherwise. * - ARROW:FLIGHT:SQL:IS_SEARCHABLE - "1" indicates if the column is searchable via WHERE clause, "0" otherwise. + * - ARROW:FLIGHT:SQL:REMARKS - A comment describing the column. * - GetFlightInfo: execute the query. */ message CommandStatementQuery { @@ -1703,6 +1705,7 @@ message CommandStatementQuery { * - ARROW:FLIGHT:SQL:IS_CASE_SENSITIVE - "1" indicates if the column is case-sensitive, "0" otherwise. * - ARROW:FLIGHT:SQL:IS_READ_ONLY - "1" indicates if the column is read only, "0" otherwise. * - ARROW:FLIGHT:SQL:IS_SEARCHABLE - "1" indicates if the column is searchable via WHERE clause, "0" otherwise. + * - ARROW:FLIGHT:SQL:REMARKS - A comment describing the column. * - GetFlightInfo: execute the query. * - DoPut: execute the query. */ @@ -1739,6 +1742,7 @@ message TicketStatementQuery { * - ARROW:FLIGHT:SQL:IS_CASE_SENSITIVE - "1" indicates if the column is case-sensitive, "0" otherwise. * - ARROW:FLIGHT:SQL:IS_READ_ONLY - "1" indicates if the column is read only, "0" otherwise. * - ARROW:FLIGHT:SQL:IS_SEARCHABLE - "1" indicates if the column is searchable via WHERE clause, "0" otherwise. + * - ARROW:FLIGHT:SQL:REMARKS - A comment describing the column. * * If the schema is retrieved after parameter values have been bound with DoPut, then the server should account * for the parameters when determining the schema. diff --git a/flight/flight-integration-tests/src/main/java/org/apache/arrow/flight/integration/tests/FlightSqlScenarioProducer.java b/flight/flight-integration-tests/src/main/java/org/apache/arrow/flight/integration/tests/FlightSqlScenarioProducer.java index be746b5757..e400c031c2 100644 --- a/flight/flight-integration-tests/src/main/java/org/apache/arrow/flight/integration/tests/FlightSqlScenarioProducer.java +++ b/flight/flight-integration-tests/src/main/java/org/apache/arrow/flight/integration/tests/FlightSqlScenarioProducer.java @@ -98,6 +98,7 @@ static Schema getQuerySchema() { .isSearchable(true) .catalogName("catalog_test") .precision(100) + .remarks("test column") .build() .getMetadataMap()), null))); @@ -126,6 +127,7 @@ static Schema getQueryWithTransactionSchema() { .isSearchable(true) .catalogName("catalog_test") .precision(100) + .remarks("test column") .build() .getMetadataMap()), null))); diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java index 3f072d071b..7185ddfe01 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java @@ -1066,6 +1066,7 @@ private int setGetColumnsVectorSchemaRootFromFields( (VarCharVector) currentRoot.getVector("IS_AUTOINCREMENT"); final VarCharVector isGeneratedColumnVector = (VarCharVector) currentRoot.getVector("IS_GENERATEDCOLUMN"); + final VarCharVector remarksVector = (VarCharVector) currentRoot.getVector("REMARKS"); for (int i = 0; i < tableColumnsSize; i++, ordinalIndex++) { final Field field = tableColumns.get(i); @@ -1139,6 +1140,11 @@ private int setGetColumnsVectorSchemaRootFromFields( isAutoincrementVector.setSafe(insertIndex, EMPTY_BYTE_ARRAY); } + String remarks = columnMetadata.getRemarks(); + if (remarks != null) { + remarksVector.setSafe(insertIndex, remarks.getBytes(CHARSET)); + } + // Fields also don't hold information about IS_AUTOINCREMENT and IS_GENERATEDCOLUMN, // so we're setting an empty string (as bytes), which means it couldn't be determined. isGeneratedColumnVector.setSafe(insertIndex, EMPTY_BYTE_ARRAY); diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java index 17b0f42dc7..5dd4c69c73 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java @@ -136,6 +136,10 @@ public static void setOnColumnMetaDataBuilder( if (searchable != null) { builder.setSearchable(searchable); } + final String remarks = columnMetadata.getRemarks(); + if (remarks != null) { + builder.setLabel(remarks); + } } /** diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java index 70d3bcbd33..81579cc387 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java @@ -26,7 +26,6 @@ import static java.util.stream.Collectors.toList; import static java.util.stream.IntStream.range; import static org.apache.arrow.driver.jdbc.utils.MockFlightSqlProducer.serializeSchema; -import static org.apache.arrow.flight.sql.impl.FlightSql.CommandGetCrossReference; import static org.apache.arrow.flight.sql.impl.FlightSql.SqlSupportsConvert.SQL_CONVERT_BIGINT_VALUE; import static org.apache.arrow.flight.sql.impl.FlightSql.SqlSupportsConvert.SQL_CONVERT_BIT_VALUE; import static org.apache.arrow.flight.sql.impl.FlightSql.SqlSupportsConvert.SQL_CONVERT_INTEGER_VALUE; @@ -55,9 +54,11 @@ import org.apache.arrow.driver.jdbc.utils.ResultSetTestUtils; import org.apache.arrow.driver.jdbc.utils.ThrowableAssertionUtils; import org.apache.arrow.flight.FlightProducer.ServerStreamListener; +import org.apache.arrow.flight.sql.FlightSqlColumnMetadata; import org.apache.arrow.flight.sql.FlightSqlProducer.Schemas; import org.apache.arrow.flight.sql.impl.FlightSql; import org.apache.arrow.flight.sql.impl.FlightSql.CommandGetCatalogs; +import org.apache.arrow.flight.sql.impl.FlightSql.CommandGetCrossReference; import org.apache.arrow.flight.sql.impl.FlightSql.CommandGetDbSchemas; import org.apache.arrow.flight.sql.impl.FlightSql.CommandGetExportedKeys; import org.apache.arrow.flight.sql.impl.FlightSql.CommandGetImportedKeys; @@ -79,6 +80,7 @@ import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.Text; import org.junit.jupiter.api.AfterAll; @@ -322,7 +324,7 @@ public class ArrowDatabaseMetadataTest { expectedGetColumnsDecimalDigits.get(i % 3), expectedGetColumnsRadix.get(i % 3), !Objects.equals(expectedGetColumnsIsNullable.get(i % 3), "NO") ? 1 : 0, - null, + format("column description #%d", (i % 3) + 1), null, null, null, @@ -419,17 +421,44 @@ public static void setUpBeforeClass() throws SQLException { try (final BufferAllocator allocator = new RootAllocator(); final VectorSchemaRoot root = VectorSchemaRoot.create(Schemas.GET_TABLES_SCHEMA, allocator)) { + final Field field1 = + new Field( + "column_1", + new FieldType( + true, + ArrowType.Decimal.createDecimal(5, 2, 128), + null, + new FlightSqlColumnMetadata.Builder() + .remarks("column description #1") + .build() + .getMetadataMap()), + null); + final Field field2 = + new Field( + "column_2", + new FieldType( + true, + new ArrowType.Timestamp(TimeUnit.NANOSECOND, "UTC"), + null, + new FlightSqlColumnMetadata.Builder() + .remarks("column description #2") + .build() + .getMetadataMap()), + null); + final Field field3 = + new Field( + "column_3", + new FieldType( + false, + Types.MinorType.INT.getType(), + null, + new FlightSqlColumnMetadata.Builder() + .remarks("column description #3") + .build() + .getMetadataMap()), + null); final byte[] filledTableSchemaBytes = - copyFrom( - serializeSchema( - new Schema( - Arrays.asList( - Field.nullable( - "column_1", ArrowType.Decimal.createDecimal(5, 2, 128)), - Field.nullable( - "column_2", - new ArrowType.Timestamp(TimeUnit.NANOSECOND, "UTC")), - Field.notNullable("column_3", Types.MinorType.INT.getType()))))) + copyFrom(serializeSchema(new Schema(Arrays.asList(field1, field2, field3)))) .toByteArray(); final VarCharVector catalogName = (VarCharVector) root.getVector("catalog_name"); final VarCharVector schemaName = (VarCharVector) root.getVector("db_schema_name"); diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ConvertUtilsTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ConvertUtilsTest.java index f6f549b5ed..b6fdc99694 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ConvertUtilsTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ConvertUtilsTest.java @@ -46,6 +46,7 @@ public void testShouldSetOnColumnMetaDataBuilder() { .isSearchable(true) .precision(20) .scale(10) + .remarks("test column") .build(); ConvertUtils.setOnColumnMetaDataBuilder(builder, expectedColumnMetaData.getMetadataMap()); assertBuilder(builder, expectedColumnMetaData); @@ -119,5 +120,6 @@ private void assertBuilder( assertThat(flightSqlColumnMetaData.isReadOnly(), equalTo(builder.getReadOnly())); assertThat(precision == null ? 0 : precision, equalTo(builder.getPrecision())); assertThat(scale == null ? 0 : scale, equalTo(builder.getScale())); + assertThat(flightSqlColumnMetaData.getRemarks(), equalTo(builder.getLabel())); } } diff --git a/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlColumnMetadata.java b/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlColumnMetadata.java index 1bcc55a660..3a969e10cf 100644 --- a/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlColumnMetadata.java +++ b/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlColumnMetadata.java @@ -53,6 +53,7 @@ public class FlightSqlColumnMetadata { private static final String IS_CASE_SENSITIVE = "ARROW:FLIGHT:SQL:IS_CASE_SENSITIVE"; private static final String IS_READ_ONLY = "ARROW:FLIGHT:SQL:IS_READ_ONLY"; private static final String IS_SEARCHABLE = "ARROW:FLIGHT:SQL:IS_SEARCHABLE"; + private static final String REMARKS = "ARROW:FLIGHT:SQL:REMARKS"; private static final String BOOLEAN_TRUE_STR = "1"; private static final String BOOLEAN_FALSE_STR = "0"; @@ -193,6 +194,15 @@ public Boolean isSearchable() { return stringToBoolean(value); } + /** + * Returns the comment describing the column. + * + * @return The comment describing the column. + */ + public String getRemarks() { + return metadataMap.get(REMARKS); + } + /** Builder of FlightSqlColumnMetadata, used on FlightSqlProducer implementations. */ public static class Builder { private final Map metadataMap; @@ -312,6 +322,17 @@ public Builder isSearchable(boolean isSearchable) { return this; } + /** + * Sets the comment describing the column. + * + * @param remarks The comment describing the column. + * @return This builder. + */ + public Builder remarks(String remarks) { + metadataMap.put(REMARKS, remarks); + return this; + } + /** * Builds a new instance of FlightSqlColumnMetadata. * From f3b50019bdbef5ecc1bfcb293129bbcf7204babd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 May 2025 09:23:07 +0200 Subject: [PATCH 156/408] MINOR: Bump com.google.guava:guava-bom from 33.4.5-jre to 33.4.8-jre (#720) Bumps [com.google.guava:guava-bom](https://github.com/google/guava) from 33.4.5-jre to 33.4.8-jre.

Release notes

Sourced from com.google.guava:guava-bom's releases.

33.4.8

Guava 33.4.8 fixes a problem that we introduced while starting to migrate guava-android off Unsafe in 33.4.7.

Even if you're not upgrading from Guava 33.4.0 or earlier, still read the release notes for Guava 33.4.1. Those release notes contain information about the effects of Guava 33.4.5 and higher on the module system.

Maven

<dependency>
  <groupId>com.google.guava</groupId>
  <artifactId>guava</artifactId>
  <version>33.4.8-jre</version>
  <!-- or, for Android: -->
  <version>33.4.8-android</version>
</dependency>

Jar files

Guava requires one runtime dependency, which you can download here:

Javadoc

JDiff

Changelog

  • util.concurrent: Removed our VarHandle code from guava-android. While the code was never used at runtime under Android, it was causing problems under the Android Gradle Plugin with a minSdkVersion below 26. To continue to avoid sun.misc.Unsafe under the JVM, guava-android will now always use AtomicReferenceFieldUpdater when run there. (75da92419a)

33.4.7

Prefer to upgrade straight to 33.4.8: 33.4.7 breaks the build of Android apps with a minSdkVersion below 26. We will publish a fixed version soon. This problem is fixed in 33.4.8.

Guava 33.4.7, like 33.4.6, fixes two problems that we introduced while modularizing Guava and migrating off Unsafe in 33.4.5.

Even if you're not upgrading from Guava 33.4.0 or earlier, still read the release notes for Guava 33.4.1. Those release notes contain information about the effects of Guava 33.4.5 and higher on the module system.

Maven

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.guava:guava-bom&package-manager=maven&previous-version=33.4.5-jre&new-version=33.4.8-jre)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ab988c7d05..fc274abcff 100644 --- a/pom.xml +++ b/pom.xml @@ -95,7 +95,7 @@ under the License. 1.9.0 5.12.1 2.0.17 - 33.4.5-jre + 33.4.8-jre 4.1.119.Final 1.71.0 4.30.1 From 75741c8ae41b74944ad84de0e2007b8a1dc3c3dd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 May 2025 09:23:50 +0200 Subject: [PATCH 157/408] MINOR: Bump com.google.protobuf:protobuf-bom from 4.30.1 to 4.30.2 (#696) Bumps [com.google.protobuf:protobuf-bom](https://github.com/protocolbuffers/protobuf) from 4.30.1 to 4.30.2.
Commits
  • 43e1626 Updating version.json and repo version numbers to: 30.2
  • 7a4c63b Fix lite classes in the protobuf-java Maven release to be JDK8 compatible. (#...
  • 7831669 Remove dllexport attribute on variable definition. (#20833)
  • da9cadc Restore JDK8 compatibility in Bazel for libraries with dependencies from Mave...
  • 09b5078 Add protobuf_maven artifacts to protobuf_maven_dev as well so they can still ...
  • b7f06f1 Add volatile to featuresResolved (#20766)
  • b69f653 Restore generator headers in cmake install until the next breaking C++ releas...
  • f4b0a79 Restore custom protobuf maven namespaces to avoid polluting main maven namesp...
  • 2dc9f35 Fix Java concurrency issue in feature resolution for old <=3.25.x gencode usi...
  • 221b2a0 Change how we decide which empty string implementation to use. (#20708)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.protobuf:protobuf-bom&package-manager=maven&previous-version=4.30.1&new-version=4.30.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fc274abcff..2e2f1659dd 100644 --- a/pom.xml +++ b/pom.xml @@ -98,7 +98,7 @@ under the License. 33.4.8-jre 4.1.119.Final 1.71.0 - 4.30.1 + 4.30.2 2.18.3 3.4.1 25.2.10 From fc75ff7e2bd7abe187320371a7470c9a7547fdae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 May 2025 18:14:21 +0200 Subject: [PATCH 158/408] MINOR: Bump dep.junit.jupiter.version from 5.12.1 to 5.12.2 (#713) Bumps `dep.junit.jupiter.version` from 5.12.1 to 5.12.2. Updates `org.junit.jupiter:junit-jupiter-engine` from 5.12.1 to 5.12.2
Release notes

Sourced from org.junit.jupiter:junit-jupiter-engine's releases.

JUnit 5.12.2 = Platform 1.12.2 + Jupiter 5.12.2 + Vintage 5.12.2

See Release Notes.

Full Changelog: https://github.com/junit-team/junit5/compare/r5.12.1...r5.12.2

Commits
  • 0a44659 Release 5.12.2
  • 4c7dfdc Finalize 5.12.2 release notes
  • 561613e Fix handling of CleanupMode.ON_SUCCESS
  • 19d07d2 Add 5.12.2 release notes from template
  • 803cbb6 Add build parameter for enabling dry-run mode for test execution
  • eb43e62 Back to snapshots for further development
  • See full diff in compare view

Updates `org.junit.jupiter:junit-jupiter-api` from 5.12.1 to 5.12.2
Release notes

Sourced from org.junit.jupiter:junit-jupiter-api's releases.

JUnit 5.12.2 = Platform 1.12.2 + Jupiter 5.12.2 + Vintage 5.12.2

See Release Notes.

Full Changelog: https://github.com/junit-team/junit5/compare/r5.12.1...r5.12.2

Commits
  • 0a44659 Release 5.12.2
  • 4c7dfdc Finalize 5.12.2 release notes
  • 561613e Fix handling of CleanupMode.ON_SUCCESS
  • 19d07d2 Add 5.12.2 release notes from template
  • 803cbb6 Add build parameter for enabling dry-run mode for test execution
  • eb43e62 Back to snapshots for further development
  • See full diff in compare view

Updates `org.junit.jupiter:junit-jupiter-params` from 5.12.1 to 5.12.2
Release notes

Sourced from org.junit.jupiter:junit-jupiter-params's releases.

JUnit 5.12.2 = Platform 1.12.2 + Jupiter 5.12.2 + Vintage 5.12.2

See Release Notes.

Full Changelog: https://github.com/junit-team/junit5/compare/r5.12.1...r5.12.2

Commits
  • 0a44659 Release 5.12.2
  • 4c7dfdc Finalize 5.12.2 release notes
  • 561613e Fix handling of CleanupMode.ON_SUCCESS
  • 19d07d2 Add 5.12.2 release notes from template
  • 803cbb6 Add build parameter for enabling dry-run mode for test execution
  • eb43e62 Back to snapshots for further development
  • See full diff in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2e2f1659dd..b8b24ca110 100644 --- a/pom.xml +++ b/pom.xml @@ -93,7 +93,7 @@ under the License. ${project.build.directory}/generated-sources 1.9.0 - 5.12.1 + 5.12.2 2.0.17 33.4.8-jre 4.1.119.Final From feb8e95fe2b283c23ec0b380d3522a9e3991bd0c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 May 2025 09:51:19 +0200 Subject: [PATCH 159/408] MINOR: Bump com.diffplug.spotless:spotless-maven-plugin from 2.44.3 to 2.44.4 (#717) Bumps [com.diffplug.spotless:spotless-maven-plugin](https://github.com/diffplug/spotless) from 2.44.3 to 2.44.4.
Release notes

Sourced from com.diffplug.spotless:spotless-maven-plugin's releases.

Maven Plugin v2.44.4

Changed

  • Use palantir-java-format 2.57.0 on Java 21. (#2447)
  • Re-try npm install with --prefer-online after ERESOLVE error. (#2448)
Commits
  • 0ca99e5 Published maven/2.44.4
  • 1b1a4fb Published gradle/7.0.3
  • 0fa3cab Published lib/3.1.1
  • 0fe8f9b Update README.md for android kotlin callouts (#2438)
  • d25f04d Minor tweak.
  • fa3fd1e Add the cool blockquote warning trick to the other Android spot.
  • 60993fd Fix the Android Kotlin warning and adjust its position.
  • 0426db1 Apply Gradle's strict plugin types validation to the Spotless plugin in prepa...
  • 48b4cde fix(deps): update dependency org.mockito:mockito-core to v5.17.0 (#2461)
  • 81f2f8b fix(deps): update dependency org.mockito:mockito-core to v5.17.0
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.diffplug.spotless:spotless-maven-plugin&package-manager=maven&previous-version=2.44.3&new-version=2.44.4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- bom/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index 171d0bc5e9..61b452b9c1 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -203,7 +203,7 @@ under the License. com.diffplug.spotless spotless-maven-plugin - 2.44.3 + 2.44.4 org.codehaus.mojo diff --git a/pom.xml b/pom.xml index b8b24ca110..fe716bad6e 100644 --- a/pom.xml +++ b/pom.xml @@ -487,7 +487,7 @@ under the License. com.diffplug.spotless spotless-maven-plugin - 2.44.3 + 2.44.4 org.codehaus.mojo From f32198fe28687c6e43df21149272764f2710947a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 May 2025 11:06:36 +0200 Subject: [PATCH 160/408] MINOR: Bump org.apache.commons:commons-text from 1.13.0 to 1.13.1 (#714) Bumps org.apache.commons:commons-text from 1.13.0 to 1.13.1. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.commons:commons-text&package-manager=maven&previous-version=1.13.0&new-version=1.13.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql/pom.xml b/flight/flight-sql/pom.xml index 5d8e720fd6..5f06a5e9eb 100644 --- a/flight/flight-sql/pom.xml +++ b/flight/flight-sql/pom.xml @@ -113,7 +113,7 @@ under the License. org.apache.commons commons-text - 1.13.0 + 1.13.1 test From b0cc03d4c62dabc44746f462f274b20003890e95 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 May 2025 11:07:00 +0200 Subject: [PATCH 161/408] MINOR: Bump org.jacoco:jacoco-maven-plugin from 0.8.12 to 0.8.13 (#699) Bumps [org.jacoco:jacoco-maven-plugin](https://github.com/jacoco/jacoco) from 0.8.12 to 0.8.13.
Release notes

Sourced from org.jacoco:jacoco-maven-plugin's releases.

0.8.13

New Features

  • JaCoCo now officially supports Java 23 and Java 24 (GitHub #1757, #1631, #1867).
  • Experimental support for Java 25 class files (GitHub #1807).
  • Calculation of line coverage for Kotlin inline functions (GitHub #1670).
  • Calculation of line coverage for Kotlin inline functions with reified type parameter (GitHub #1670, #1700).
  • Calculation of coverage for Kotlin JvmSynthetic functions (GitHub #1700).
  • Part of bytecode generated by the Kotlin Compose compiler plugin is filtered out during generation of report (GitHub #1616).
  • Part of bytecode generated by the Kotlin compiler for inline value classes is filtered out during generation of report (GitHub #1475).
  • Part of bytecode generated by the Kotlin compiler for suspending lambdas without suspension points is filtered out during generation of report (GitHub #1283).
  • Part of bytecode generated by the Kotlin compiler for when expressions and statements with nullable enum subject is filtered out during generation of report (GitHub #1774).
  • Part of bytecode generated by the Kotlin compiler for when expressions and statements with nullable String subject is filtered out during generation of report (GitHub #1769).
  • Part of bytecode generated by the Kotlin compiler for chains of safe call operators is filtered out during generation of report (GitHub #1810, #1818).
  • Method getEntries generated by the Kotlin compiler for enum classes is filtered out during generation of report (GitHub #1625).
  • Methods generated by the Kotlin compiler for constructors and functions with JvmOverloads annotation are filtered out (GitHub #1768).

Fixed bugs

  • Fixed interpretation of Kotlin SMAP (GitHub #1525).
  • File extensions are preserved in HTML report in case of clashes of normalized file names (GitHub #1660).

Non-functional Changes

  • JaCoCo build now uses Maven Wrapper and requires at least Maven 3.9.9 (GitHub #1708, #1707, #1681).
  • JaCoCo now depends on ASM 9.8 (GitHub #1862).
  • More context information when IllegalArgumentException occurs during reading of zip file (GitHub #1833).
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.jacoco:jacoco-maven-plugin&package-manager=maven&previous-version=0.8.12&new-version=0.8.13)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fe716bad6e..8360e1be2b 100644 --- a/pom.xml +++ b/pom.xml @@ -347,7 +347,7 @@ under the License. org.jacoco jacoco-maven-plugin - 0.8.12 + 0.8.13

... (truncated)

Commits
  • e303ead chore(main): release 2.56.0 (#3715)
  • 3714eee chore: update generation config (#3749)
  • 992e643 ci: Run all downstream protobuf CI checks with Java 8 JVM (#3747)
  • 64ac2c1 feat: Selective gapic generation phase II (#3730)
  • 29a78d3 fix(hermetic-build): use correct image name in templated graalvm jobs (#3743)
  • 243b4da build: introduce testing infra for GraalVM "C" (#3739)
  • 4383ea1 build(deps): update dependency com.google.cloud:google-cloud-shared-config to...
  • 07fb39d chore: introduce .kokoro templates for GraalVM jobs (#3737)
  • 087597e chore(sec): prefix hermetic-build images with infrastructure-public-image ...
  • b77a32b chore: prepare showcase for GraalVM for JDK 17 (#3734)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.api.grpc:proto-google-common-protos&package-manager=maven&previous-version=2.54.1&new-version=2.56.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml index 757de85769..92f58b84de 100644 --- a/flight/flight-core/pom.xml +++ b/flight/flight-core/pom.xml @@ -134,7 +134,7 @@ under the License. com.google.api.grpc proto-google-common-protos - 2.54.1 + 2.56.0 test From 93b7fdc2113c9c21e359edc910a6b44061b0ee50 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 May 2025 10:47:23 +0200 Subject: [PATCH 172/408] MINOR: Bump com.github.ben-manes.caffeine:caffeine from 3.1.8 to 3.2.0 (#747) Bumps [com.github.ben-manes.caffeine:caffeine](https://github.com/ben-manes/caffeine) from 3.1.8 to 3.2.0.
Release notes

Sourced from com.github.ben-manes.caffeine:caffeine's releases.

3.2.0

Cache

  • Added Sigstore signing of maven artifacts
  • Added Expiry static factory methods (#1499)
  • Migrated to JSpecify annotations (was checker framework)
  • Fixed variable expiration calculation when nearing overflow
  • Added logging when an async cache's removal listener fails
  • Added an expiration write optimization to more operations (#1320)
  • Fixed when a Weigher or Expiry fail on an async completion (#1687)
  • Fixed cases when the expiration ticker was also used for statistics (#1678)
  • Fixed refresh handling to skip if the async cache's entry is still loading (#1478)
  • Fixed containsKey for an async cache's synchronous view while in-flight (#1626)
  • Fixed premature expiration for an async cache when using nearly immediate expiration (#1623)
  • For a bulk async load returning extra mappings, wait to be added to the cache before returning (#1409)

Guava

  • Relaxed the OSGi version requirement (#1160)

JCache

  • Allow hibernate.javax.cache.uri to load the configuration from a jar (#1347)
Commits
  • 93d845e prepare for next release
  • 8022a16 expand expire write optimization to more operations (fixes #1320)
  • 9b65365 Avoid early expiration of an pending future due to delayed pinning (fixes #1623)
  • 25405d6 skip refreshing if the async cache entry is still loading (fixes #1478)
  • 2d93f2b fix variable expiration calculation when nearing overflow
  • 3ff2445 migrate to the maintained javapoet project
  • bbf54be minor polish and fixes #1820
  • b09770f upgrade to apache commons collections' junit5 test suite
  • 70f5cf8 use jakarta.inject for jcache guice integration test
  • ae21802 add missing asserts to openjdk tests that check only in their custom harness
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.github.ben-manes.caffeine:caffeine&package-manager=maven&previous-version=3.1.8&new-version=3.2.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index a95fbaca30..04c4691faf 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -151,7 +151,7 @@ under the License. com.github.ben-manes.caffeine caffeine - 3.1.8 + 3.2.0 From 8c74ced9c03e4aba1d1c5fea5f223a42be234dcd Mon Sep 17 00:00:00 2001 From: Gang Wu Date: Tue, 13 May 2025 23:43:14 +0800 Subject: [PATCH 173/408] MINOR: Bump version to 19.0.0-SNAPSHOT (#754) --- adapter/avro/pom.xml | 2 +- adapter/jdbc/pom.xml | 2 +- adapter/orc/pom.xml | 2 +- algorithm/pom.xml | 2 +- bom/pom.xml | 4 ++-- c/pom.xml | 2 +- compression/pom.xml | 2 +- dataset/pom.xml | 2 +- flight/flight-core/pom.xml | 2 +- flight/flight-integration-tests/pom.xml | 2 +- flight/flight-sql-jdbc-core/pom.xml | 2 +- flight/flight-sql-jdbc-driver/pom.xml | 2 +- flight/flight-sql/pom.xml | 2 +- flight/pom.xml | 2 +- format/pom.xml | 2 +- gandiva/pom.xml | 2 +- memory/memory-core/pom.xml | 2 +- memory/memory-netty-buffer-patch/pom.xml | 2 +- memory/memory-netty/pom.xml | 2 +- memory/memory-unsafe/pom.xml | 2 +- memory/pom.xml | 2 +- performance/pom.xml | 2 +- pom.xml | 4 ++-- tools/pom.xml | 2 +- vector/pom.xml | 2 +- 25 files changed, 27 insertions(+), 27 deletions(-) diff --git a/adapter/avro/pom.xml b/adapter/avro/pom.xml index cf9d353330..827d19f2a2 100644 --- a/adapter/avro/pom.xml +++ b/adapter/avro/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT ../../pom.xml diff --git a/adapter/jdbc/pom.xml b/adapter/jdbc/pom.xml index f92863fbd6..2f621d7a05 100644 --- a/adapter/jdbc/pom.xml +++ b/adapter/jdbc/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT ../../pom.xml diff --git a/adapter/orc/pom.xml b/adapter/orc/pom.xml index e60a7ceb3c..e3ae7d5163 100644 --- a/adapter/orc/pom.xml +++ b/adapter/orc/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT ../../pom.xml diff --git a/algorithm/pom.xml b/algorithm/pom.xml index e934eb7b22..898c2605b6 100644 --- a/algorithm/pom.xml +++ b/algorithm/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT arrow-algorithm Arrow Algorithms diff --git a/bom/pom.xml b/bom/pom.xml index 80f03d1205..61b452b9c1 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -29,7 +29,7 @@ under the License. org.apache.arrow arrow-bom - 18.3.0 + 19.0.0-SNAPSHOT pom Arrow Bill of Materials @@ -68,7 +68,7 @@ under the License. scm:git:https://github.com/apache/arrow-java.git scm:git:https://github.com/apache/arrow-java.git - v18.3.0 + main https://github.com/apache/arrow-java/tree/${project.scm.tag} diff --git a/c/pom.xml b/c/pom.xml index 290cb561c1..c90b6dc0ef 100644 --- a/c/pom.xml +++ b/c/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT arrow-c-data diff --git a/compression/pom.xml b/compression/pom.xml index 3443f11478..6f60eb7d0a 100644 --- a/compression/pom.xml +++ b/compression/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT arrow-compression Arrow Compression diff --git a/dataset/pom.xml b/dataset/pom.xml index efbe310ea2..6e56d555b7 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT arrow-dataset diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml index 92f58b84de..24beac391e 100644 --- a/flight/flight-core/pom.xml +++ b/flight/flight-core/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 18.3.0 + 19.0.0-SNAPSHOT flight-core diff --git a/flight/flight-integration-tests/pom.xml b/flight/flight-integration-tests/pom.xml index e7fb999149..78a2d08ee1 100644 --- a/flight/flight-integration-tests/pom.xml +++ b/flight/flight-integration-tests/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 18.3.0 + 19.0.0-SNAPSHOT flight-integration-tests diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 04c4691faf..d8e012101c 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 18.3.0 + 19.0.0-SNAPSHOT flight-sql-jdbc-core diff --git a/flight/flight-sql-jdbc-driver/pom.xml b/flight/flight-sql-jdbc-driver/pom.xml index 3776e97f3f..559c42597d 100644 --- a/flight/flight-sql-jdbc-driver/pom.xml +++ b/flight/flight-sql-jdbc-driver/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 18.3.0 + 19.0.0-SNAPSHOT flight-sql-jdbc-driver diff --git a/flight/flight-sql/pom.xml b/flight/flight-sql/pom.xml index 66ade30306..5f06a5e9eb 100644 --- a/flight/flight-sql/pom.xml +++ b/flight/flight-sql/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 18.3.0 + 19.0.0-SNAPSHOT flight-sql diff --git a/flight/pom.xml b/flight/pom.xml index 7b31e8ce91..2fc3e89ef8 100644 --- a/flight/pom.xml +++ b/flight/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT arrow-flight diff --git a/format/pom.xml b/format/pom.xml index 9b4eebfe3c..d3578b63d2 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT arrow-format diff --git a/gandiva/pom.xml b/gandiva/pom.xml index 167bf39cb9..5367bfdedf 100644 --- a/gandiva/pom.xml +++ b/gandiva/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT org.apache.arrow.gandiva diff --git a/memory/memory-core/pom.xml b/memory/memory-core/pom.xml index 840d3464ba..72ee69d60a 100644 --- a/memory/memory-core/pom.xml +++ b/memory/memory-core/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 18.3.0 + 19.0.0-SNAPSHOT arrow-memory-core diff --git a/memory/memory-netty-buffer-patch/pom.xml b/memory/memory-netty-buffer-patch/pom.xml index e9a63b2122..07dc7d2403 100644 --- a/memory/memory-netty-buffer-patch/pom.xml +++ b/memory/memory-netty-buffer-patch/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 18.3.0 + 19.0.0-SNAPSHOT arrow-memory-netty-buffer-patch diff --git a/memory/memory-netty/pom.xml b/memory/memory-netty/pom.xml index 42f35efb33..6d660da117 100644 --- a/memory/memory-netty/pom.xml +++ b/memory/memory-netty/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 18.3.0 + 19.0.0-SNAPSHOT arrow-memory-netty diff --git a/memory/memory-unsafe/pom.xml b/memory/memory-unsafe/pom.xml index 0af306cbfc..92dc0c9fe5 100644 --- a/memory/memory-unsafe/pom.xml +++ b/memory/memory-unsafe/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 18.3.0 + 19.0.0-SNAPSHOT arrow-memory-unsafe diff --git a/memory/pom.xml b/memory/pom.xml index 09a5bc2924..bc34c26050 100644 --- a/memory/pom.xml +++ b/memory/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT arrow-memory pom diff --git a/performance/pom.xml b/performance/pom.xml index 02bdf46a11..3f18188e3a 100644 --- a/performance/pom.xml +++ b/performance/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT arrow-performance jar diff --git a/pom.xml b/pom.xml index 9d89616356..d40f428b22 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT pom Apache Arrow Java Root POM @@ -81,7 +81,7 @@ under the License. scm:git:https://github.com/apache/arrow-java.git scm:git:https://github.com/apache/arrow-java.git - v18.3.0 + main https://github.com/apache/arrow-java/tree/${project.scm.tag} diff --git a/tools/pom.xml b/tools/pom.xml index d60281d80c..cb9a161308 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT arrow-tools Arrow Tools diff --git a/vector/pom.xml b/vector/pom.xml index 450e4ff71d..52ad5105ea 100644 --- a/vector/pom.xml +++ b/vector/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT arrow-vector Arrow Vectors From b459647910a6a373fb9f3d9cd0eb8cc717932bed Mon Sep 17 00:00:00 2001 From: Gang Wu Date: Wed, 14 May 2025 10:42:38 +0800 Subject: [PATCH 174/408] MINOR: add missing SOURCE_DIR in dev/release/release.sh (#755) ## What's Changed `dev/release/release.sh` requires `SOURCE_DIR` to locate `.env` but it is missing. --- dev/release/release.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/release/release.sh b/dev/release/release.sh index 70e1f96454..f08a618c4f 100755 --- a/dev/release/release.sh +++ b/dev/release/release.sh @@ -19,6 +19,8 @@ set -eu +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + if [ "$#" -ne 2 ]; then echo "Usage: $0 " echo " e.g.: $0 19.0.1 1" From 6be33bba2366f2335b962f25fe6ea0789665ae89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Po=C5=82omski?= Date: Thu, 15 May 2025 04:57:23 +0200 Subject: [PATCH 175/408] MINOR: Empty stream double check (#742) ## What's Changed In some cases InflaterInputStream can return a non zero on `available()` method call, while it is actually at EOS. A subsequent `read()` call would respond with -1 and eventually cause: 
Caused by: org.apache.arrow.flight.FlightRuntimeException: Failed to
read message.
at
org.apache.arrow.flight.CallStatus.toRuntimeException(CallStatus.java:121)
at
org.apache.arrow.flight.grpc.StatusUtils.fromGrpcRuntimeException(StatusUtils.java:161)
at
org.apache.arrow.flight.grpc.StatusUtils.fromThrowable(StatusUtils.java:182)
at
org.apache.arrow.flight.FlightStream$Observer.onError(FlightStream.java:489)
at org.apache.arrow.flight.FlightClient$1.onError(FlightClient.java:371)
at
io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onClose(ClientCalls.java:564)
at
io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
at
io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
at
io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
at
org.apache.arrow.flight.grpc.ClientInterceptorAdapter$FlightClientCallListener.onClose(ClientInterceptorAdapter.java:118)
at
io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:564)
	at io.grpc.internal.ClientCallImpl.access$100(ClientCallImpl.java:72)
at
io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInternal(ClientCallImpl.java:729)
at
io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:710)
	at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)
at
io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:133)
	... 3 common frames omitted
Caused by: java.lang.RuntimeException:
com.google.protobuf.InvalidProtocolBufferException: While parsing a
protocol message, the input ended unexpectedly in the middle of a field.
This could mean either that the input has been truncated or that an
embedded message misreported its own length.
	at org.apache.arrow.flight.ArrowMessage.frame(ArrowMessage.java:363)
at
org.apache.arrow.flight.ArrowMessage$ArrowMessageHolderMarshaller.parse(ArrowMessage.java:575)
at
org.apache.arrow.flight.ArrowMessage$ArrowMessageHolderMarshaller.parse(ArrowMessage.java:560)
	at io.grpc.MethodDescriptor.parseResponse(MethodDescriptor.java:284)
at
io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:657)
at
io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:644)
	... 5 common frames omitted
Caused by: com.google.protobuf.InvalidProtocolBufferException: While
parsing a protocol message, the input ended unexpectedly in the middle
of a field. This could mean either that the input has been truncated or
that an embedded message misreported its own length.
at
com.google.protobuf.InvalidProtocolBufferException.truncatedMessage(InvalidProtocolBufferException.java:92)
at
com.google.protobuf.CodedInputStream.readRawVarint32(CodedInputStream.java:568)
at
org.apache.arrow.flight.ArrowMessage.readRawVarint32(ArrowMessage.java:369)
	at org.apache.arrow.flight.ArrowMessage.frame(ArrowMessage.java:290)
	... 10 common frames omitted
--- .../java/org/apache/arrow/flight/ArrowMessage.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/flight/flight-core/src/main/java/org/apache/arrow/flight/ArrowMessage.java b/flight/flight-core/src/main/java/org/apache/arrow/flight/ArrowMessage.java index 9cefccb3fe..ab4eab3048 100644 --- a/flight/flight-core/src/main/java/org/apache/arrow/flight/ArrowMessage.java +++ b/flight/flight-core/src/main/java/org/apache/arrow/flight/ArrowMessage.java @@ -287,7 +287,11 @@ private static ArrowMessage frame(BufferAllocator allocator, final InputStream s ArrowBuf body = null; ArrowBuf appMetadata = null; while (stream.available() > 0) { - int tag = readRawVarint32(stream); + final int tagFirstByte = stream.read(); + if (tagFirstByte == -1) { + break; + } + int tag = readRawVarint32(tagFirstByte, stream); switch (tag) { case DESCRIPTOR_TAG: { @@ -366,6 +370,10 @@ private static ArrowMessage frame(BufferAllocator allocator, final InputStream s private static int readRawVarint32(InputStream is) throws IOException { int firstByte = is.read(); + return readRawVarint32(firstByte, is); + } + + private static int readRawVarint32(int firstByte, InputStream is) throws IOException { return CodedInputStream.readRawVarint32(firstByte, is); } From 7c25ce5d86490822600b49928d34a08b4dddad46 Mon Sep 17 00:00:00 2001 From: ViggoC Date: Thu, 22 May 2025 21:40:44 +0800 Subject: [PATCH 176/408] GH-52: Make RangeEqualsVisitor of RunEndEncodedVector more efficient (#761) ## What's Changed Avoid doing a binary search on every step to make the RangeEqualsVisitor of RunEndEncodedVector more efficient. Closes #52 . --- .../vector/compare/RangeEqualsVisitor.java | 44 ++++----- .../vector/complex/RunEndEncodedVector.java | 98 +++++++++++++++++++ .../arrow/vector/TestRunEndEncodedVector.java | 18 ++-- .../compare/TestRangeEqualsVisitor.java | 52 ++++++++++ 4 files changed, 181 insertions(+), 31 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java b/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java index abcf312c5e..bc2e3a6aab 100644 --- a/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java +++ b/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java @@ -43,6 +43,7 @@ import org.apache.arrow.vector.complex.ListViewVector; import org.apache.arrow.vector.complex.NonNullableStructVector; import org.apache.arrow.vector.complex.RunEndEncodedVector; +import org.apache.arrow.vector.complex.RunEndEncodedVector.RangeIterator; import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.complex.UnionVector; @@ -270,42 +271,35 @@ protected boolean compareRunEndEncodedVectors(Range range) { RunEndEncodedVector leftVector = (RunEndEncodedVector) left; RunEndEncodedVector rightVector = (RunEndEncodedVector) right; - final int leftRangeEnd = range.getLeftStart() + range.getLength(); - final int rightRangeEnd = range.getRightStart() + range.getLength(); + final RunEndEncodedVector.RangeIterator leftIterator = + new RunEndEncodedVector.RangeIterator(leftVector, range.getLeftStart(), range.getLength()); + final RunEndEncodedVector.RangeIterator rightIterator = + new RunEndEncodedVector.RangeIterator( + rightVector, range.getRightStart(), range.getLength()); FieldVector leftValuesVector = leftVector.getValuesVector(); FieldVector rightValuesVector = rightVector.getValuesVector(); RangeEqualsVisitor innerVisitor = createInnerVisitor(leftValuesVector, rightValuesVector, null); - int leftLogicalIndex = range.getLeftStart(); - int rightLogicalIndex = range.getRightStart(); + while (nextRun(leftIterator, rightIterator)) { + int leftPhysicalIndex = leftIterator.getRunIndex(); + int rightPhysicalIndex = rightIterator.getRunIndex(); - while (leftLogicalIndex < leftRangeEnd) { - // TODO: implement it more efficient - // https://github.com/apache/arrow/issues/44157 - int leftPhysicalIndex = leftVector.getPhysicalIndex(leftLogicalIndex); - int rightPhysicalIndex = rightVector.getPhysicalIndex(rightLogicalIndex); - if (leftValuesVector.accept( - innerVisitor, new Range(leftPhysicalIndex, rightPhysicalIndex, 1))) { - int leftRunEnd = leftVector.getRunEnd(leftLogicalIndex); - int rightRunEnd = rightVector.getRunEnd(rightLogicalIndex); - - int leftRunLength = Math.min(leftRunEnd, leftRangeEnd) - leftLogicalIndex; - int rightRunLength = Math.min(rightRunEnd, rightRangeEnd) - rightLogicalIndex; - - if (leftRunLength != rightRunLength) { - return false; - } else { - leftLogicalIndex = leftRunEnd; - rightLogicalIndex = rightRunEnd; - } - } else { + if (leftIterator.getRunLength() != rightIterator.getRunLength() + || !leftValuesVector.accept( + innerVisitor, new Range(leftPhysicalIndex, rightPhysicalIndex, 1))) { return false; } } - return true; + return leftIterator.isEnd() && rightIterator.isEnd(); + } + + private static boolean nextRun(RangeIterator leftIterator, RangeIterator rightIterator) { + boolean left = leftIterator.nextRun(); + boolean right = rightIterator.nextRun(); + return left && right; } protected RangeEqualsVisitor createInnerVisitor( diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/RunEndEncodedVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/RunEndEncodedVector.java index 1bb9a3d6c0..b83e13449a 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/RunEndEncodedVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/RunEndEncodedVector.java @@ -28,6 +28,7 @@ import org.apache.arrow.memory.OutOfMemoryException; import org.apache.arrow.memory.util.ByteFunctionHelpers; import org.apache.arrow.memory.util.hash.ArrowBufHasher; +import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.BaseIntVector; import org.apache.arrow.vector.BaseValueVector; import org.apache.arrow.vector.BigIntVector; @@ -820,4 +821,101 @@ static int getPhysicalIndex(FieldVector runEndVector, int logicalIndex) { return result; } + + public static class RangeIterator { + + private final RunEndEncodedVector runEndEncodedVector; + private final int rangeEnd; + private int runIndex; + private int runEnd; + private int logicalPos; + + /** + * Constructs a new RangeIterator for iterating over a range of values in a RunEndEncodedVector. + * + * @param runEndEncodedVector The vector to iterate over + * @param startIndex The logical start index of the range (inclusive) + * @param length The number of values to include in the range + * @throws IllegalArgumentException if startIndex is negative or (startIndex + length) exceeds + * vector bounds + */ + public RangeIterator(RunEndEncodedVector runEndEncodedVector, int startIndex, int length) { + int rangeEnd = startIndex + length; + Preconditions.checkArgument( + startIndex >= 0, "startIndex %s must be non negative.", startIndex); + Preconditions.checkArgument( + rangeEnd <= runEndEncodedVector.getValueCount(), + "(startIndex + length) %s out of range[0, %s].", + rangeEnd, + runEndEncodedVector.getValueCount()); + + this.rangeEnd = rangeEnd; + this.runEndEncodedVector = runEndEncodedVector; + this.runIndex = runEndEncodedVector.getPhysicalIndex(startIndex) - 1; + this.runEnd = startIndex; + this.logicalPos = -1; + } + + /** + * Advances to the next run in the range. + * + * @return true if there is another run available, false if iteration has completed + */ + public boolean nextRun() { + logicalPos = runEnd; + if (logicalPos >= rangeEnd) { + return false; + } + updateRun(); + return true; + } + + private void updateRun() { + runIndex++; + runEnd = (int) ((BaseIntVector) runEndEncodedVector.runEndsVector).getValueAsLong(runIndex); + } + + /** + * Advances to the next value in the range. + * + * @return true if there is another value available, false if iteration has completed + */ + public boolean nextValue() { + logicalPos++; + if (logicalPos >= rangeEnd) { + return false; + } + if (logicalPos == runEnd) { + updateRun(); + } + return true; + } + + /** + * Gets the current run index (physical position in the run-ends vector). + * + * @return the current run index + */ + public int getRunIndex() { + return runIndex; + } + + /** + * Gets the length of the current run within the iterator's range. + * + * @return the number of remaining values in current run within the iterator's range + */ + public int getRunLength() { + return Math.min(runEnd, rangeEnd) - logicalPos; + } + + /** + * Checks if iteration has completed. + * + * @return true if all values in the range have been processed, false otherwise + */ + public boolean isEnd() { + return logicalPos >= rangeEnd; + } + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestRunEndEncodedVector.java b/vector/src/test/java/org/apache/arrow/vector/TestRunEndEncodedVector.java index adf51c0730..9fa153e928 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestRunEndEncodedVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestRunEndEncodedVector.java @@ -148,12 +148,18 @@ public void testRangeCompare() { assertTrue( constantVector.accept( new RangeEqualsVisitor(constantVector, constantVector), new Range(1, 2, 13))); - assertFalse( - constantVector.accept( - new RangeEqualsVisitor(constantVector, constantVector), new Range(1, 10, 10))); - assertFalse( - constantVector.accept( - new RangeEqualsVisitor(constantVector, constantVector), new Range(10, 1, 10))); + + // throws exception if the range end is out the bound of the vector + assertThrows( + IllegalArgumentException.class, + () -> + constantVector.accept( + new RangeEqualsVisitor(constantVector, constantVector), new Range(1, 10, 10))); + assertThrows( + IllegalArgumentException.class, + () -> + constantVector.accept( + new RangeEqualsVisitor(constantVector, constantVector), new Range(10, 1, 10))); // Create REE vector representing: [1, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5]. RunEndEncodedVector reeVector = diff --git a/vector/src/test/java/org/apache/arrow/vector/compare/TestRangeEqualsVisitor.java b/vector/src/test/java/org/apache/arrow/vector/compare/TestRangeEqualsVisitor.java index 08da786eb2..9624734356 100644 --- a/vector/src/test/java/org/apache/arrow/vector/compare/TestRangeEqualsVisitor.java +++ b/vector/src/test/java/org/apache/arrow/vector/compare/TestRangeEqualsVisitor.java @@ -22,6 +22,7 @@ import java.nio.charset.Charset; import java.util.Arrays; +import java.util.List; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BigIntVector; @@ -39,6 +40,7 @@ import org.apache.arrow.vector.complex.LargeListViewVector; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.ListViewVector; +import org.apache.arrow.vector.complex.RunEndEncodedVector; import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.complex.UnionVector; import org.apache.arrow.vector.complex.impl.NullableStructWriter; @@ -53,7 +55,9 @@ import org.apache.arrow.vector.holders.NullableUInt4Holder; import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ArrowType.RunEndEncoded; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.junit.jupiter.api.AfterEach; @@ -1003,6 +1007,54 @@ public void testLargeListViewVectorApproxEquals() { } } + @Test + public void testRunEndEncodedFloat8ApproxEquals() { + try (final Float8Vector vector1 = new Float8Vector("float", allocator); + final Float8Vector vector2 = new Float8Vector("float", allocator); + final Float8Vector vector3 = new Float8Vector("float", allocator); + final IntVector reeVector = new IntVector("ree", allocator)) { + + final float epsilon = 1.0E-6f; + setVector(vector1, 1.1, 2.2); + setVector(vector2, 1.1 + epsilon / 2, 2.2 + epsilon / 2); + setVector(vector3, 1.1 + epsilon * 2, 2.2 + epsilon * 2); + setVector(reeVector, 1, 3); + + ArrowType type = MinorType.FLOAT8.getType(); + final FieldType valueType = FieldType.notNullable(type); + final FieldType runEndType = FieldType.notNullable(MinorType.INT.getType()); + + final Field valueField = new Field("value", valueType, null); + final Field runEndField = new Field("ree", runEndType, null); + + Field field = + new Field( + "ree_float", + FieldType.notNullable(RunEndEncoded.INSTANCE), + List.of(runEndField, valueField)); + + try (final RunEndEncodedVector encodedVector1 = + new RunEndEncodedVector(field, allocator, reeVector, vector1, null); + final RunEndEncodedVector encodedVector2 = + new RunEndEncodedVector(field, allocator, reeVector, vector2, null); + final RunEndEncodedVector encodedVector3 = + new RunEndEncodedVector(field, allocator, reeVector, vector3, null)) { + + encodedVector1.setValueCount(3); + encodedVector2.setValueCount(3); + encodedVector3.setValueCount(3); + + Range range = new Range(0, 0, encodedVector1.getValueCount()); + assertTrue( + new ApproxEqualsVisitor(encodedVector1, encodedVector2, epsilon, epsilon) + .rangeEquals(range)); + assertFalse( + new ApproxEqualsVisitor(encodedVector1, encodedVector3, epsilon, epsilon) + .rangeEquals(range)); + } + } + } + private void writeStructVector(NullableStructWriter writer, int value1, long value2) { writer.start(); writer.integer("f0").writeInt(value1); From 2aef66df8ce771b3e634e28c98e4442b8cb39995 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 24 May 2025 14:22:40 +0900 Subject: [PATCH 177/408] GH-768: Use apache/arrow-js for JS in integration test (#769) ## What's Changed `js/` in apache/arrow moved to apache/arrow-js. So let's use apache/arrow-js for JS. Closes #768. --- .github/workflows/test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5db5c988eb..f1028ce029 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -176,6 +176,11 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: java + - name: Checkout Arrow JavaScript + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: apache/arrow-js + path: js - name: Free up disk space run: | ci/scripts/util_free_space.sh @@ -199,6 +204,7 @@ jobs: -e ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS=java \ -e ARCHERY_INTEGRATION_WITH_GO=1 \ -e ARCHERY_INTEGRATION_WITH_JAVA=1 \ + -e ARCHERY_INTEGRATION_WITH_JS=1 \ -e ARCHERY_INTEGRATION_WITH_NANOARROW=1 \ -e ARCHERY_INTEGRATION_WITH_RUST=1 \ conda-integration From abef7af8490d706d52821e56afc6d1eb21e24faf Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 26 May 2025 08:55:13 +0900 Subject: [PATCH 178/408] GH-770: Ensure updating Homebrew Python on macos-13 (#771) ## What's Changed If we update older Python (e.g. Python 3.12) to newer Python (e.g. Python 3.13), depended packages may update newer Python when we update old Python. Let's use newer Python -> older Python order instead to avoid the situation. Closes #770. --- .github/workflows/rc.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 7e3cf5f6f2..72456fa556 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -221,6 +221,10 @@ jobs: # llvm@14 because llvm is newer than llvm@14. brew uninstall llvm || : + # We can remove this when we drop support for + # macos-13. because macos-14 or later uses /opt/homebrew/ + # not /usr/local/. + # # Ensure updating python@XXX with the "--overwrite" option. # If python@XXX is updated without "--overwrite", it causes # a conflict error. Because Python 3 installed not by @@ -229,10 +233,10 @@ jobs: # tries to replace /usr/local/bin/2to3 and so on and causes # a conflict error. brew update - for python_package in $(brew list | grep python@); do + for python_package in $(brew list | grep python@ | sort -r); do brew install --overwrite ${python_package} done - brew install --overwrite python + brew install --overwrite python3 if [ "$(uname -m)" = "arm64" ]; then # pkg-config formula is deprecated but it's still installed From c6f608e863d3cefdfc41a433ed854e43b20a0925 Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Wed, 28 May 2025 03:06:02 +0200 Subject: [PATCH 179/408] GH-765: Do not close/free imported BaseStruct objects (#766) ## What's Changed This PR removes the direct and indirect calls to `BaseStruct#close` from `org.apache.arrow.c.Data`. By not eagerly closing/freeing these objects callers can reuse instances multiple times. Closes #765. --- .../org/apache/arrow/c/ArrayImporter.java | 1 - .../arrow/c/ArrowArrayStreamReader.java | 1 - c/src/main/java/org/apache/arrow/c/Data.java | 219 ++++++++++++++++-- .../org/apache/arrow/c/RoundtripTest.java | 106 +++++++-- 4 files changed, 287 insertions(+), 40 deletions(-) diff --git a/c/src/main/java/org/apache/arrow/c/ArrayImporter.java b/c/src/main/java/org/apache/arrow/c/ArrayImporter.java index b74fb1b473..f31a8a1faa 100644 --- a/c/src/main/java/org/apache/arrow/c/ArrayImporter.java +++ b/c/src/main/java/org/apache/arrow/c/ArrayImporter.java @@ -58,7 +58,6 @@ void importArray(ArrowArray src) { ArrowArray ownedArray = ArrowArray.allocateNew(allocator); ownedArray.save(snapshot); src.markReleased(); - src.close(); recursionLevel = 0; diff --git a/c/src/main/java/org/apache/arrow/c/ArrowArrayStreamReader.java b/c/src/main/java/org/apache/arrow/c/ArrowArrayStreamReader.java index 07a88cd8d7..34a9c4ec03 100644 --- a/c/src/main/java/org/apache/arrow/c/ArrowArrayStreamReader.java +++ b/c/src/main/java/org/apache/arrow/c/ArrowArrayStreamReader.java @@ -44,7 +44,6 @@ final class ArrowArrayStreamReader extends ArrowReader { this.ownedStream = ArrowArrayStream.allocateNew(allocator); this.ownedStream.save(snapshot); stream.markReleased(); - stream.close(); } @Override diff --git a/c/src/main/java/org/apache/arrow/c/Data.java b/c/src/main/java/org/apache/arrow/c/Data.java index 0b4da33b4e..f9d2ee4542 100644 --- a/c/src/main/java/org/apache/arrow/c/Data.java +++ b/c/src/main/java/org/apache/arrow/c/Data.java @@ -231,6 +231,22 @@ public static void exportArrayStream( new ArrayStreamExporter(allocator).export(out, reader); } + /** + * Equivalent to calling {@link #importField(BufferAllocator, ArrowSchema, + * CDataDictionaryProvider, boolean) importField(allocator, schema, provider, true)}. + * + * @param allocator Buffer allocator for allocating dictionary vectors + * @param schema C data interface struct representing the field [inout] + * @param provider A dictionary provider will be initialized with empty dictionary vectors + * (optional) + * @return Imported field object + * @see #importField(BufferAllocator, ArrowSchema, CDataDictionaryProvider, boolean) + */ + public static Field importField( + BufferAllocator allocator, ArrowSchema schema, CDataDictionaryProvider provider) { + return importField(allocator, schema, provider, true); + } + /** * Import Java Field from the C data interface. * @@ -241,19 +257,42 @@ public static void exportArrayStream( * @param schema C data interface struct representing the field [inout] * @param provider A dictionary provider will be initialized with empty dictionary vectors * (optional) + * @param closeImportedStructs if true, the ArrowSchema struct will be closed when this method + * completes. * @return Imported field object */ public static Field importField( - BufferAllocator allocator, ArrowSchema schema, CDataDictionaryProvider provider) { + BufferAllocator allocator, + ArrowSchema schema, + CDataDictionaryProvider provider, + boolean closeImportedStructs) { try { SchemaImporter importer = new SchemaImporter(allocator); return importer.importField(schema, provider); } finally { schema.release(); - schema.close(); + if (closeImportedStructs) { + schema.close(); + } } } + /** + * Equivalent to calling {@link #importSchema(BufferAllocator, ArrowSchema, + * CDataDictionaryProvider, boolean) importSchema(allocator, schema, provider, true)}. + * + * @param allocator Buffer allocator for allocating dictionary vectors + * @param schema C data interface struct representing the field + * @param provider A dictionary provider will be initialized with empty dictionary vectors + * (optional) + * @return Imported schema object + * @see #importSchema(BufferAllocator, ArrowSchema, CDataDictionaryProvider, boolean) + */ + public static Schema importSchema( + BufferAllocator allocator, ArrowSchema schema, CDataDictionaryProvider provider) { + return importSchema(allocator, schema, provider, true); + } + /** * Import Java Schema from the C data interface. * @@ -264,11 +303,16 @@ public static Field importField( * @param schema C data interface struct representing the field * @param provider A dictionary provider will be initialized with empty dictionary vectors * (optional) + * @param closeImportedStructs if true, the ArrowSchema struct will be closed when this method + * completes. * @return Imported schema object */ public static Schema importSchema( - BufferAllocator allocator, ArrowSchema schema, CDataDictionaryProvider provider) { - Field structField = importField(allocator, schema, provider); + BufferAllocator allocator, + ArrowSchema schema, + CDataDictionaryProvider provider, + boolean closeImportedStructs) { + Field structField = importField(allocator, schema, provider, closeImportedStructs); if (structField.getType().getTypeID() != ArrowTypeID.Struct) { throw new IllegalArgumentException( "Cannot import schema: ArrowSchema describes non-struct type"); @@ -276,24 +320,67 @@ public static Schema importSchema( return new Schema(structField.getChildren(), structField.getMetadata()); } + /** + * Equivalent to calling {@link #importIntoVector(BufferAllocator, ArrowArray, FieldVector, + * DictionaryProvider, boolean)} importIntoVector(allocator, array, vector, provider, true)}. + * + * @param allocator Buffer allocator + * @param array C data interface struct holding the array data + * @param vector Imported vector object [out] + * @param provider Dictionary provider to load dictionary vectors to (optional) + * @see #importIntoVector(BufferAllocator, ArrowArray, FieldVector, DictionaryProvider, boolean) + */ + public static void importIntoVector( + BufferAllocator allocator, + ArrowArray array, + FieldVector vector, + DictionaryProvider provider) { + importIntoVector(allocator, array, vector, provider, true); + } + /** * Import Java vector from the C data interface. * - *

The ArrowArray struct has its contents moved (as per the C data interface specification) to - * a private object held alive by the resulting array. + *

On successful completion, the ArrowArray struct will have been moved (as per the C data + * interface specification) to a private object held alive by the resulting array. * * @param allocator Buffer allocator * @param array C data interface struct holding the array data * @param vector Imported vector object [out] * @param provider Dictionary provider to load dictionary vectors to (optional) + * @param closeImportedStructs if true, the ArrowArray struct will be closed when this method + * completes successfully. */ public static void importIntoVector( BufferAllocator allocator, ArrowArray array, FieldVector vector, - DictionaryProvider provider) { + DictionaryProvider provider, + boolean closeImportedStructs) { ArrayImporter importer = new ArrayImporter(allocator, vector, provider); importer.importArray(array); + if (closeImportedStructs) { + array.close(); + } + } + + /** + * Equivalent to calling {@link #importVector(BufferAllocator, ArrowArray, ArrowSchema, + * CDataDictionaryProvider, boolean) importVector(allocator, array, schema, provider, true)}. + * + * @param allocator Buffer allocator for allocating the output FieldVector + * @param array C data interface struct holding the array data + * @param schema C data interface struct holding the array type + * @param provider Dictionary provider to load dictionary vectors to (optional) + * @return Imported vector object + * @see #importVector(BufferAllocator, ArrowArray, ArrowSchema, CDataDictionaryProvider, boolean) + */ + public static FieldVector importVector( + BufferAllocator allocator, + ArrowArray array, + ArrowSchema schema, + CDataDictionaryProvider provider) { + return importVector(allocator, array, schema, provider, true); } /** @@ -307,19 +394,42 @@ public static void importIntoVector( * @param array C data interface struct holding the array data * @param schema C data interface struct holding the array type * @param provider Dictionary provider to load dictionary vectors to (optional) + * @param closeImportedStructs if true, the ArrowArray struct will be closed when this method + * completes successfully and the ArrowSchema struct will be always be closed. * @return Imported vector object */ public static FieldVector importVector( BufferAllocator allocator, ArrowArray array, ArrowSchema schema, - CDataDictionaryProvider provider) { - Field field = importField(allocator, schema, provider); + CDataDictionaryProvider provider, + boolean closeImportedStructs) { + Field field = importField(allocator, schema, provider, closeImportedStructs); FieldVector vector = field.createVector(allocator); - importIntoVector(allocator, array, vector, provider); + importIntoVector(allocator, array, vector, provider, closeImportedStructs); return vector; } + /** + * Equivalent to calling {@link #importIntoVectorSchemaRoot(BufferAllocator, ArrowArray, + * VectorSchemaRoot, DictionaryProvider, boolean) importIntoVectorSchemaRoot(allocator, array, + * root, provider, true)}. + * + * @param allocator Buffer allocator + * @param array C data interface struct holding the record batch data + * @param root vector schema root to load into + * @param provider Dictionary provider to load dictionary vectors to (optional) + * @see #importIntoVectorSchemaRoot(BufferAllocator, ArrowArray, VectorSchemaRoot, + * DictionaryProvider, boolean) + */ + public static void importIntoVectorSchemaRoot( + BufferAllocator allocator, + ArrowArray array, + VectorSchemaRoot root, + DictionaryProvider provider) { + importIntoVectorSchemaRoot(allocator, array, root, provider, true); + } + /** * Import record batch from the C data interface into vector schema root. * @@ -333,15 +443,18 @@ public static FieldVector importVector( * @param array C data interface struct holding the record batch data * @param root vector schema root to load into * @param provider Dictionary provider to load dictionary vectors to (optional) + * @param closeImportedStructs if true, the ArrowArray struct will be closed when this method + * completes successfully */ public static void importIntoVectorSchemaRoot( BufferAllocator allocator, ArrowArray array, VectorSchemaRoot root, - DictionaryProvider provider) { + DictionaryProvider provider, + boolean closeImportedStructs) { try (StructVector structVector = StructVector.emptyWithDuplicates("", allocator)) { structVector.initializeChildrenFromFields(root.getSchema().getFields()); - importIntoVector(allocator, array, structVector, provider); + importIntoVector(allocator, array, structVector, provider, closeImportedStructs); StructVectorUnloader unloader = new StructVectorUnloader(structVector); VectorLoader loader = new VectorLoader(root); try (ArrowRecordBatch recordBatch = unloader.getRecordBatch()) { @@ -350,6 +463,21 @@ public static void importIntoVectorSchemaRoot( } } + /** + * Equivalent to calling {@link #importVectorSchemaRoot(BufferAllocator, ArrowSchema, + * CDataDictionaryProvider, boolean) importVectorSchemaRoot(allocator, schema, provider, true)}. + * + * @param allocator Buffer allocator for allocating the output VectorSchemaRoot + * @param schema C data interface struct holding the record batch schema + * @param provider Dictionary provider to load dictionary vectors to (optional) + * @return Imported vector schema root + * @see #importVectorSchemaRoot(BufferAllocator, ArrowSchema, CDataDictionaryProvider, boolean) + */ + public static VectorSchemaRoot importVectorSchemaRoot( + BufferAllocator allocator, ArrowSchema schema, CDataDictionaryProvider provider) { + return importVectorSchemaRoot(allocator, schema, provider, true); + } + /** * Import Java vector schema root from a C data interface Schema. * @@ -360,11 +488,37 @@ public static void importIntoVectorSchemaRoot( * @param allocator Buffer allocator for allocating the output VectorSchemaRoot * @param schema C data interface struct holding the record batch schema * @param provider Dictionary provider to load dictionary vectors to (optional) + * @param closeImportedStructs if true, the ArrowSchema struct will be closed when this method + * completes * @return Imported vector schema root */ public static VectorSchemaRoot importVectorSchemaRoot( - BufferAllocator allocator, ArrowSchema schema, CDataDictionaryProvider provider) { - return importVectorSchemaRoot(allocator, null, schema, provider); + BufferAllocator allocator, + ArrowSchema schema, + CDataDictionaryProvider provider, + boolean closeImportedStructs) { + return importVectorSchemaRoot(allocator, null, schema, provider, closeImportedStructs); + } + + /** + * Equivalent to calling {@link #importVectorSchemaRoot(BufferAllocator, ArrowArray, ArrowSchema, + * CDataDictionaryProvider, boolean) importVectorSchemaRoot(allocator, array, schema, provider, + * true)}. + * + * @param allocator Buffer allocator for allocating the output VectorSchemaRoot + * @param array C data interface struct holding the record batch data (optional) + * @param schema C data interface struct holding the record batch schema + * @param provider Dictionary provider to load dictionary vectors to (optional) + * @return Imported vector schema root + * @see #importVectorSchemaRoot(BufferAllocator, ArrowArray, ArrowSchema, CDataDictionaryProvider, + * boolean) + */ + public static VectorSchemaRoot importVectorSchemaRoot( + BufferAllocator allocator, + ArrowArray array, + ArrowSchema schema, + CDataDictionaryProvider provider) { + return importVectorSchemaRoot(allocator, array, schema, provider, true); } /** @@ -383,29 +537,56 @@ public static VectorSchemaRoot importVectorSchemaRoot( * @param array C data interface struct holding the record batch data (optional) * @param schema C data interface struct holding the record batch schema * @param provider Dictionary provider to load dictionary vectors to (optional) + * @param closeImportedStructs if true, the ArrowArray struct will be closed when this method + * completes successfully and the ArrowSchema struct will be always be closed. * @return Imported vector schema root */ public static VectorSchemaRoot importVectorSchemaRoot( BufferAllocator allocator, ArrowArray array, ArrowSchema schema, - CDataDictionaryProvider provider) { + CDataDictionaryProvider provider, + boolean closeImportedStructs) { VectorSchemaRoot vsr = - VectorSchemaRoot.create(importSchema(allocator, schema, provider), allocator); + VectorSchemaRoot.create( + importSchema(allocator, schema, provider, closeImportedStructs), allocator); if (array != null) { - importIntoVectorSchemaRoot(allocator, array, vsr, provider); + importIntoVectorSchemaRoot(allocator, array, vsr, provider, closeImportedStructs); } return vsr; } /** - * Import an ArrowArrayStream as an {@link ArrowReader}. + * Equivalent to calling {@link #importArrayStream(BufferAllocator, ArrowArrayStream, boolean) + * importArrayStream(allocator, stream, true)}. * * @param allocator Buffer allocator for allocating the output data. * @param stream C stream interface struct to import. * @return Imported reader + * @see #importArrayStream(BufferAllocator, ArrowArrayStream, boolean) */ public static ArrowReader importArrayStream(BufferAllocator allocator, ArrowArrayStream stream) { - return new ArrowArrayStreamReader(allocator, stream); + return importArrayStream(allocator, stream, true); + } + + /** + * Import an ArrowArrayStream as an {@link ArrowReader}. + * + *

On successful completion, the ArrowArrayStream struct will have been moved (as per the C + * data interface specification) to a private object held alive by the resulting ArrowReader. + * + * @param allocator Buffer allocator for allocating the output data. + * @param stream C stream interface struct to import. + * @param closeImportedStructs if true, the ArrowArrayStream struct will be closed when this + * method completes successfully + * @return Imported reader + */ + public static ArrowReader importArrayStream( + BufferAllocator allocator, ArrowArrayStream stream, boolean closeImportedStructs) { + ArrowArrayStreamReader reader = new ArrowArrayStreamReader(allocator, stream); + if (closeImportedStructs) { + stream.close(); + } + return reader; } } diff --git a/c/src/test/java/org/apache/arrow/c/RoundtripTest.java b/c/src/test/java/org/apache/arrow/c/RoundtripTest.java index 6d68449c0b..010a305495 100644 --- a/c/src/test/java/org/apache/arrow/c/RoundtripTest.java +++ b/c/src/test/java/org/apache/arrow/c/RoundtripTest.java @@ -17,9 +17,7 @@ package org.apache.arrow.c; import static org.apache.arrow.vector.testing.ValueVectorDataPopulator.setVector; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -958,6 +956,50 @@ public void testVectorSchemaRootWithDuplicatedFieldNames() { @Test public void testSchema() { + Schema schema = createSchema(); + // Consumer allocates empty ArrowSchema + try (ArrowSchema consumerArrowSchema = ArrowSchema.allocateNew(allocator)) { + // Producer fills the schema with data + exportSchema(schema, consumerArrowSchema); + + // Consumer imports schema + Schema importedSchema = Data.importSchema(allocator, consumerArrowSchema, null); + assertEquals(schema.toJson(), importedSchema.toJson()); + } + } + + @Test + public void testSchemaStructReuse() { + Schema schema = createSchema(); + // Consumer allocates empty ArrowSchema + try (ArrowSchema consumerArrowSchema = ArrowSchema.allocateNew(allocator)) { + // Producer fills the schema with data + exportSchema(schema, consumerArrowSchema); + + // Consumer imports schema + Schema importedSchema = Data.importSchema(allocator, consumerArrowSchema, null, false); + assertEquals(schema.toJson(), importedSchema.toJson()); + + // Imported struct should be released but not closed + assertEquals(0, consumerArrowSchema.snapshot().release); + assertNotEquals(0, consumerArrowSchema.memoryAddress()); + + // Export and import again + exportSchema(schema, consumerArrowSchema); + importedSchema = Data.importSchema(allocator, consumerArrowSchema, null, false); + assertEquals(schema.toJson(), importedSchema.toJson()); + assertEquals(0, consumerArrowSchema.snapshot().release); + assertNotEquals(0, consumerArrowSchema.memoryAddress()); + } + } + + private void exportSchema(Schema schema, ArrowSchema targetArrowSchema) { + try (ArrowSchema arrowSchema = ArrowSchema.wrap(targetArrowSchema.memoryAddress())) { + Data.exportSchema(allocator, schema, null, arrowSchema); + } + } + + private static Schema createSchema() { Field decimalField = new Field("inner1", FieldType.nullable(new ArrowType.Decimal(19, 4, 128)), null); Field strField = new Field("inner2", FieldType.nullable(new ArrowType.Utf8()), null); @@ -968,16 +1010,7 @@ public void testSchema() { Arrays.asList(decimalField, strField)); Field intField = new Field("col2", FieldType.nullable(new ArrowType.Int(32, true)), null); Schema schema = new Schema(Arrays.asList(itemField, intField)); - // Consumer allocates empty ArrowSchema - try (ArrowSchema consumerArrowSchema = ArrowSchema.allocateNew(allocator)) { - // Producer fills the schema with data - try (ArrowSchema arrowSchema = ArrowSchema.wrap(consumerArrowSchema.memoryAddress())) { - Data.exportSchema(allocator, schema, null, arrowSchema); - } - // Consumer imports schema - Schema importedSchema = Data.importSchema(allocator, consumerArrowSchema, null); - assertEquals(schema.toJson(), importedSchema.toJson()); - } + return schema; } @Test @@ -1002,12 +1035,8 @@ public void testImportReleasedArray() { try (ArrowSchema consumerArrowSchema = ArrowSchema.allocateNew(allocator); ArrowArray consumerArrowArray = ArrowArray.allocateNew(allocator)) { // Producer creates structures from existing memory pointers - try (ArrowSchema arrowSchema = ArrowSchema.wrap(consumerArrowSchema.memoryAddress()); - ArrowArray arrowArray = ArrowArray.wrap(consumerArrowArray.memoryAddress())) { - // Producer exports vector into the C Data Interface structures - try (final NullVector vector = new NullVector()) { - Data.exportVector(allocator, vector, null, arrowArray, arrowSchema); - } + try (final NullVector vector = new NullVector()) { + exportFieldVector(vector, consumerArrowSchema, consumerArrowArray); } // Release array structure @@ -1025,6 +1054,45 @@ public void testImportReleasedArray() { } } + @Test + public void testArrayStructReuse() { + // Consumer allocates empty structures + try (ArrowSchema consumerArrowSchema = ArrowSchema.allocateNew(allocator); + ArrowArray consumerArrowArray = ArrowArray.allocateNew(allocator)) { + // Producer creates structures from existing memory pointers + try (final NullVector vector = new NullVector()) { + exportFieldVector(vector, consumerArrowSchema, consumerArrowArray); + } + Data.importVector(allocator, consumerArrowArray, consumerArrowSchema, null, false); + + // Imported structs should be released but not closed + assertEquals(0, consumerArrowSchema.snapshot().release); + assertNotEquals(0, consumerArrowSchema.memoryAddress()); + assertEquals(0, consumerArrowArray.snapshot().release); + assertNotEquals(0, consumerArrowArray.memoryAddress()); + + try (final NullVector vector = new NullVector()) { + exportFieldVector(vector, consumerArrowSchema, consumerArrowArray); + } + Data.importVector(allocator, consumerArrowArray, consumerArrowSchema, null, false); + + // Imported structs should be released but not closed + assertEquals(0, consumerArrowSchema.snapshot().release); + assertNotEquals(0, consumerArrowSchema.memoryAddress()); + assertEquals(0, consumerArrowArray.snapshot().release); + assertNotEquals(0, consumerArrowArray.memoryAddress()); + } + } + + private void exportFieldVector( + FieldVector vector, ArrowSchema consumerArrowSchema, ArrowArray consumerArrowArray) { + try (ArrowSchema arrowSchema = ArrowSchema.wrap(consumerArrowSchema.memoryAddress()); + ArrowArray arrowArray = ArrowArray.wrap(consumerArrowArray.memoryAddress())) { + // Producer exports vector into the C Data Interface structures + Data.exportVector(allocator, vector, null, arrowArray, arrowSchema); + } + } + private VectorSchemaRoot createTestVSR() { BitVector bitVector = new BitVector("boolean", allocator); From 17f85a1fba64aef4552ed2fabc5a76bc038a1ed6 Mon Sep 17 00:00:00 2001 From: rtadepalli <105760760+rtadepalli@users.noreply.github.com> Date: Tue, 27 May 2025 21:06:55 -0400 Subject: [PATCH 180/408] GH-70: Move from `hamcrest` to `assertj` in `flight-sql` (#772) ## What's Changed Series of PRs to consolidate on using `assertj` in tests as part of https://github.com/apache/arrow-java/issues/70. --- flight/flight-sql/pom.xml | 10 +- .../arrow/flight/sql/test/TestFlightSql.java | 264 +++++++++--------- .../sql/test/TestFlightSqlStateless.java | 9 +- .../flight/sql/test/TestFlightSqlStreams.java | 29 +- 4 files changed, 150 insertions(+), 162 deletions(-) diff --git a/flight/flight-sql/pom.xml b/flight/flight-sql/pom.xml index 5f06a5e9eb..15d00e3e18 100644 --- a/flight/flight-sql/pom.xml +++ b/flight/flight-sql/pom.xml @@ -116,16 +116,16 @@ under the License. 1.13.1 test - - org.hamcrest - hamcrest - test - commons-cli commons-cli 1.9.0 true + + org.assertj + assertj-core + test + diff --git a/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSql.java b/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSql.java index 3f769363fb..e2934ab1e9 100644 --- a/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSql.java +++ b/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSql.java @@ -21,10 +21,7 @@ import static java.util.Collections.singletonList; import static org.apache.arrow.flight.sql.util.FlightStreamUtils.getResults; import static org.apache.arrow.util.AutoCloseables.close; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.hamcrest.CoreMatchers.nullValue; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -40,6 +37,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.stream.IntStream; import org.apache.arrow.flight.CancelFlightInfoRequest; @@ -76,8 +74,7 @@ import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.Text; import org.apache.arrow.vector.util.VectorBatchAppender; -import org.hamcrest.Matcher; -import org.hamcrest.MatcherAssert; +import org.assertj.core.api.Condition; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -247,16 +244,15 @@ private static List> getNonConformingResultsForGetSqlInfo( @Test public void testGetTablesSchema() { final FlightInfo info = sqlClient.getTables(null, null, null, null, true); - MatcherAssert.assertThat( - info.getSchemaOptional(), is(Optional.of(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA))); + assertThat(info.getSchemaOptional()) + .isEqualTo(Optional.of(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA)); } @Test public void testGetTablesSchemaExcludeSchema() { final FlightInfo info = sqlClient.getTables(null, null, null, null, false); - MatcherAssert.assertThat( - info.getSchemaOptional(), - is(Optional.of(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA_NO_SCHEMA))); + assertThat(info.getSchemaOptional()) + .isEqualTo(Optional.of(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA_NO_SCHEMA)); } @Test @@ -266,8 +262,8 @@ public void testGetTablesResultNoSchema() throws Exception { sqlClient.getTables(null, null, null, null, false).getEndpoints().get(0).getTicket())) { assertAll( () -> { - MatcherAssert.assertThat( - stream.getSchema(), is(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA_NO_SCHEMA)); + assertThat(stream.getSchema()) + .isEqualTo(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA_NO_SCHEMA); }, () -> { final List> results = getResults(stream); @@ -301,7 +297,7 @@ public void testGetTablesResultNoSchema() throws Exception { asList(null /* TODO No catalog yet */, "SYSIBM", "SYSDUMMY1", "SYSTEM TABLE"), asList(null /* TODO No catalog yet */, "APP", "FOREIGNTABLE", "TABLE"), asList(null /* TODO No catalog yet */, "APP", "INTTABLE", "TABLE")); - MatcherAssert.assertThat(results, is(expectedResults)); + assertThat(results).isEqualTo(expectedResults); }); } } @@ -318,8 +314,8 @@ public void testGetTablesResultFilteredNoSchema() throws Exception { assertAll( () -> - MatcherAssert.assertThat( - stream.getSchema(), is(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA_NO_SCHEMA)), + assertThat(stream.getSchema()) + .isEqualTo(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA_NO_SCHEMA), () -> { final List> results = getResults(stream); final List> expectedResults = @@ -327,7 +323,7 @@ public void testGetTablesResultFilteredNoSchema() throws Exception { // catalog_name | schema_name | table_name | table_type | table_schema asList(null /* TODO No catalog yet */, "APP", "FOREIGNTABLE", "TABLE"), asList(null /* TODO No catalog yet */, "APP", "INTTABLE", "TABLE")); - MatcherAssert.assertThat(results, is(expectedResults)); + assertThat(results).isEqualTo(expectedResults); }); } } @@ -343,11 +339,9 @@ public void testGetTablesResultFilteredWithSchema() throws Exception { .getTicket())) { assertAll( () -> - MatcherAssert.assertThat( - stream.getSchema(), is(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA)), + assertThat(stream.getSchema()).isEqualTo(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA), () -> { - MatcherAssert.assertThat( - stream.getSchema(), is(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA)); + assertThat(stream.getSchema()).isEqualTo(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA); final List> results = getResults(stream); final List> expectedResults = ImmutableList.of( @@ -487,7 +481,7 @@ public void testGetTablesResultFilteredWithSchema() throws Exception { .getMetadataMap()), null))) .toJson())); - MatcherAssert.assertThat(results, is(expectedResults)); + assertThat(results).isEqualTo(expectedResults); }); } } @@ -498,11 +492,11 @@ public void testSimplePreparedStatementSchema() throws Exception { assertAll( () -> { final Schema actualSchema = preparedStatement.getResultSetSchema(); - MatcherAssert.assertThat(actualSchema, is(SCHEMA_INT_TABLE)); + assertThat(actualSchema).isEqualTo(SCHEMA_INT_TABLE); }, () -> { final FlightInfo info = preparedStatement.execute(); - MatcherAssert.assertThat(info.getSchemaOptional(), is(Optional.of(SCHEMA_INT_TABLE))); + assertThat(info.getSchemaOptional()).isEqualTo(Optional.of(SCHEMA_INT_TABLE)); }); } } @@ -513,10 +507,8 @@ public void testSimplePreparedStatementResults() throws Exception { final FlightStream stream = sqlClient.getStream(preparedStatement.execute().getEndpoints().get(0).getTicket())) { assertAll( - () -> MatcherAssert.assertThat(stream.getSchema(), is(SCHEMA_INT_TABLE)), - () -> - MatcherAssert.assertThat( - getResults(stream), is(EXPECTED_RESULTS_FOR_STAR_SELECT_QUERY))); + () -> assertThat(stream.getSchema()).isEqualTo(SCHEMA_INT_TABLE), + () -> assertThat(getResults(stream)).isEqualTo(EXPECTED_RESULTS_FOR_STAR_SELECT_QUERY)); } } @@ -538,10 +530,8 @@ public void testSimplePreparedStatementResultsWithParameterBinding() throws Exce FlightStream stream = sqlClient.getStream(flightInfo.getEndpoints().get(0).getTicket()); assertAll( - () -> MatcherAssert.assertThat(stream.getSchema(), is(SCHEMA_INT_TABLE)), - () -> - MatcherAssert.assertThat( - getResults(stream), is(EXPECTED_RESULTS_FOR_PARAMETER_BINDING))); + () -> assertThat(stream.getSchema()).isEqualTo(SCHEMA_INT_TABLE), + () -> assertThat(getResults(stream)).isEqualTo(EXPECTED_RESULTS_FOR_PARAMETER_BINDING)); } } } @@ -579,8 +569,8 @@ public void testSimplePreparedStatementUpdateResults() throws SQLException { deletedRows = deletePrepare.executeUpdate(); } assertAll( - () -> MatcherAssert.assertThat(updatedRows, is(10L)), - () -> MatcherAssert.assertThat(deletedRows, is(10L))); + () -> assertThat(updatedRows).isEqualTo(10L), + () -> assertThat(deletedRows).isEqualTo(10L)); } } } @@ -647,7 +637,7 @@ public void testBulkIngest() throws IOException { null, null)); - MatcherAssert.assertThat(updatedRows, is(-1L)); + assertThat(updatedRows).isEqualTo(-1L); // Ingest directly using VectorSchemaRoot populateNext10RowsInIngestRootBatch( @@ -672,7 +662,7 @@ public void testBulkIngest() throws IOException { deletedRows = deletePrepare.executeUpdate(); } - MatcherAssert.assertThat(deletedRows, is(30L)); + assertThat(deletedRows).isEqualTo(30L); } } } @@ -709,8 +699,7 @@ public void testSimplePreparedStatementUpdateResultsWithoutParameters() throws S final long deletedRows = deletePrepare.executeUpdate(); assertAll( - () -> MatcherAssert.assertThat(updatedRows, is(1L)), - () -> MatcherAssert.assertThat(deletedRows, is(1L))); + () -> assertThat(updatedRows).isEqualTo(1L), () -> assertThat(deletedRows).isEqualTo(1L)); } } @@ -719,19 +708,19 @@ public void testSimplePreparedStatementClosesProperly() { final PreparedStatement preparedStatement = sqlClient.prepare("SELECT * FROM intTable"); assertAll( () -> { - MatcherAssert.assertThat(preparedStatement.isClosed(), is(false)); + assertThat(preparedStatement.isClosed()).isEqualTo(false); }, () -> { preparedStatement.close(); - MatcherAssert.assertThat(preparedStatement.isClosed(), is(true)); + assertThat(preparedStatement.isClosed()).isEqualTo(true); }); } @Test public void testGetCatalogsSchema() { final FlightInfo info = sqlClient.getCatalogs(); - MatcherAssert.assertThat( - info.getSchemaOptional(), is(Optional.of(FlightSqlProducer.Schemas.GET_CATALOGS_SCHEMA))); + assertThat(info.getSchemaOptional()) + .isEqualTo(Optional.of(FlightSqlProducer.Schemas.GET_CATALOGS_SCHEMA)); } @Test @@ -740,11 +729,11 @@ public void testGetCatalogsResults() throws Exception { sqlClient.getStream(sqlClient.getCatalogs().getEndpoints().get(0).getTicket())) { assertAll( () -> - MatcherAssert.assertThat( - stream.getSchema(), is(FlightSqlProducer.Schemas.GET_CATALOGS_SCHEMA)), + assertThat(stream.getSchema()) + .isEqualTo(FlightSqlProducer.Schemas.GET_CATALOGS_SCHEMA), () -> { List> catalogs = getResults(stream); - MatcherAssert.assertThat(catalogs, is(emptyList())); + assertThat(catalogs).isEqualTo(emptyList()); }); } } @@ -752,9 +741,8 @@ public void testGetCatalogsResults() throws Exception { @Test public void testGetTableTypesSchema() { final FlightInfo info = sqlClient.getTableTypes(); - MatcherAssert.assertThat( - info.getSchemaOptional(), - is(Optional.of(FlightSqlProducer.Schemas.GET_TABLE_TYPES_SCHEMA))); + assertThat(info.getSchemaOptional()) + .isEqualTo(Optional.of(FlightSqlProducer.Schemas.GET_TABLE_TYPES_SCHEMA)); } @Test @@ -763,8 +751,8 @@ public void testGetTableTypesResult() throws Exception { sqlClient.getStream(sqlClient.getTableTypes().getEndpoints().get(0).getTicket())) { assertAll( () -> { - MatcherAssert.assertThat( - stream.getSchema(), is(FlightSqlProducer.Schemas.GET_TABLE_TYPES_SCHEMA)); + assertThat(stream.getSchema()) + .isEqualTo(FlightSqlProducer.Schemas.GET_TABLE_TYPES_SCHEMA); }, () -> { final List> tableTypes = getResults(stream); @@ -775,7 +763,7 @@ public void testGetTableTypesResult() throws Exception { singletonList("SYSTEM TABLE"), singletonList("TABLE"), singletonList("VIEW")); - MatcherAssert.assertThat(tableTypes, is(expectedTableTypes)); + assertThat(tableTypes).isEqualTo(expectedTableTypes); }); } } @@ -783,8 +771,8 @@ public void testGetTableTypesResult() throws Exception { @Test public void testGetSchemasSchema() { final FlightInfo info = sqlClient.getSchemas(null, null); - MatcherAssert.assertThat( - info.getSchemaOptional(), is(Optional.of(FlightSqlProducer.Schemas.GET_SCHEMAS_SCHEMA))); + assertThat(info.getSchemaOptional()) + .isEqualTo(Optional.of(FlightSqlProducer.Schemas.GET_SCHEMAS_SCHEMA)); } @Test @@ -793,8 +781,7 @@ public void testGetSchemasResult() throws Exception { sqlClient.getStream(sqlClient.getSchemas(null, null).getEndpoints().get(0).getTicket())) { assertAll( () -> { - MatcherAssert.assertThat( - stream.getSchema(), is(FlightSqlProducer.Schemas.GET_SCHEMAS_SCHEMA)); + assertThat(stream.getSchema()).isEqualTo(FlightSqlProducer.Schemas.GET_SCHEMAS_SCHEMA); }, () -> { final List> schemas = getResults(stream); @@ -812,7 +799,7 @@ public void testGetSchemasResult() throws Exception { asList(null /* TODO Add catalog. */, "SYSIBM"), asList(null /* TODO Add catalog. */, "SYSPROC"), asList(null /* TODO Add catalog. */, "SYSSTAT")); - MatcherAssert.assertThat(schemas, is(expectedSchemas)); + assertThat(schemas).isEqualTo(expectedSchemas); }); } } @@ -825,24 +812,24 @@ public void testGetPrimaryKey() { final List> results = getResults(stream); assertAll( - () -> MatcherAssert.assertThat(results.size(), is(1)), + () -> assertThat(results.size()).isEqualTo(1), () -> { final List result = results.get(0); assertAll( - () -> MatcherAssert.assertThat(result.get(0), is("")), - () -> MatcherAssert.assertThat(result.get(1), is("APP")), - () -> MatcherAssert.assertThat(result.get(2), is("INTTABLE")), - () -> MatcherAssert.assertThat(result.get(3), is("ID")), - () -> MatcherAssert.assertThat(result.get(4), is("1")), - () -> MatcherAssert.assertThat(result.get(5), notNullValue())); + () -> assertThat(result.get(0)).isEqualTo(""), + () -> assertThat(result.get(1)).isEqualTo("APP"), + () -> assertThat(result.get(2)).isEqualTo("INTTABLE"), + () -> assertThat(result.get(3)).isEqualTo("ID"), + () -> assertThat(result.get(4)).isEqualTo("1"), + () -> assertThat(result.get(5)).isNotNull()); }); } @Test public void testGetSqlInfoSchema() { final FlightInfo info = sqlClient.getSqlInfo(); - MatcherAssert.assertThat( - info.getSchemaOptional(), is(Optional.of(FlightSqlProducer.Schemas.GET_SQL_INFO_SCHEMA))); + assertThat(info.getSchemaOptional()) + .isEqualTo(Optional.of(FlightSqlProducer.Schemas.GET_SQL_INFO_SCHEMA)); } @Test @@ -851,11 +838,11 @@ public void testGetSqlInfoResults() throws Exception { try (final FlightStream stream = sqlClient.getStream(info.getEndpoints().get(0).getTicket())) { assertAll( () -> - MatcherAssert.assertThat( - stream.getSchema(), is(FlightSqlProducer.Schemas.GET_SQL_INFO_SCHEMA)), + assertThat(stream.getSchema()) + .isEqualTo(FlightSqlProducer.Schemas.GET_SQL_INFO_SCHEMA), () -> - MatcherAssert.assertThat( - getNonConformingResultsForGetSqlInfo(getResults(stream)), is(emptyList()))); + assertThat(getNonConformingResultsForGetSqlInfo(getResults(stream))) + .isEqualTo(emptyList())); } } @@ -866,11 +853,11 @@ public void testGetSqlInfoResultsWithSingleArg() throws Exception { try (final FlightStream stream = sqlClient.getStream(info.getEndpoints().get(0).getTicket())) { assertAll( () -> - MatcherAssert.assertThat( - stream.getSchema(), is(FlightSqlProducer.Schemas.GET_SQL_INFO_SCHEMA)), + assertThat(stream.getSchema()) + .isEqualTo(FlightSqlProducer.Schemas.GET_SQL_INFO_SCHEMA), () -> - MatcherAssert.assertThat( - getNonConformingResultsForGetSqlInfo(getResults(stream), arg), is(emptyList()))); + assertThat(getNonConformingResultsForGetSqlInfo(getResults(stream), arg)) + .isEqualTo(emptyList())); } } @@ -895,11 +882,11 @@ public void testGetSqlInfoResultsWithManyArgs() throws Exception { try (final FlightStream stream = sqlClient.getStream(info.getEndpoints().get(0).getTicket())) { assertAll( () -> - MatcherAssert.assertThat( - stream.getSchema(), is(FlightSqlProducer.Schemas.GET_SQL_INFO_SCHEMA)), + assertThat(stream.getSchema()) + .isEqualTo(FlightSqlProducer.Schemas.GET_SQL_INFO_SCHEMA), () -> - MatcherAssert.assertThat( - getNonConformingResultsForGetSqlInfo(getResults(stream), args), is(emptyList()))); + assertThat(getNonConformingResultsForGetSqlInfo(getResults(stream), args)) + .isEqualTo(emptyList())); } } @@ -915,28 +902,30 @@ public void testGetCommandExportedKeys() throws Exception { final List> results = getResults(stream); - final List> matchers = + final List> matchers = asList( - nullValue(String.class), // pk_catalog_name - is("APP"), // pk_schema_name - is("FOREIGNTABLE"), // pk_table_name - is("ID"), // pk_column_name - nullValue(String.class), // fk_catalog_name - is("APP"), // fk_schema_name - is("INTTABLE"), // fk_table_name - is("FOREIGNID"), // fk_column_name - is("1"), // key_sequence - containsString("SQL"), // fk_key_name - containsString("SQL"), // pk_key_name - is("3"), // update_rule - is("3")); // delete_rule + new Condition<>(Objects::isNull, "pk_catalog_name expected to be null"), + new Condition<>(c -> c.equals("APP"), "pk_schema_name expected to equal APP"), + new Condition<>( + c -> c.equals("FOREIGNTABLE"), "pk_table_name should equal FOREIGNTABLE"), + new Condition<>(c -> c.equals("ID"), "pk_column_name should equal ID"), + new Condition<>(Objects::isNull, "fk_catalog_name expected to be null"), + new Condition<>(c -> c.equals("APP"), "fk_schema_name expected to be APP"), + new Condition<>(c -> c.equals("INTTABLE"), "fk_table_name expeced to be INTTABLE"), + new Condition<>( + c -> c.equals("FOREIGNID"), "fk_column_name expected to equal FOREIGNID"), + new Condition<>(c -> c.equals("1"), "key_sequence expected to equal 1"), + new Condition<>(c -> c.contains("SQL"), "fk_key_name expected to contain SQL"), + new Condition<>(c -> c.contains("SQL"), "pk_key_name expected to contain SQL"), + new Condition<>(c -> c.equals("3"), "update_rule expected to equal 3"), + new Condition<>(c -> c.equals("3"), "delete_rule expected to equal 3")); final List assertions = new ArrayList<>(); assertEquals(1, results.size()); for (int i = 0; i < matchers.size(); i++) { final String actual = results.get(0).get(i); - final Matcher expected = matchers.get(i); - assertions.add(() -> MatcherAssert.assertThat(actual, expected)); + final Condition expected = matchers.get(i); + assertions.add(() -> assertThat(actual).satisfies(expected)); } assertAll(assertions); } @@ -954,28 +943,30 @@ public void testGetCommandImportedKeys() throws Exception { final List> results = getResults(stream); - final List> matchers = + final List> matchers = asList( - nullValue(String.class), // pk_catalog_name - is("APP"), // pk_schema_name - is("FOREIGNTABLE"), // pk_table_name - is("ID"), // pk_column_name - nullValue(String.class), // fk_catalog_name - is("APP"), // fk_schema_name - is("INTTABLE"), // fk_table_name - is("FOREIGNID"), // fk_column_name - is("1"), // key_sequence - containsString("SQL"), // fk_key_name - containsString("SQL"), // pk_key_name - is("3"), // update_rule - is("3")); // delete_rule + new Condition<>(Objects::isNull, "pk_catalog_name expected to be null"), + new Condition<>(c -> c.equals("APP"), "pk_schema_name expected to equal APP"), + new Condition<>( + c -> c.equals("FOREIGNTABLE"), "pk_table_name should equal FOREIGNTABLE"), + new Condition<>(c -> c.equals("ID"), "pk_column_name should equal ID"), + new Condition<>(Objects::isNull, "fk_catalog_name expected to be null"), + new Condition<>(c -> c.equals("APP"), "fk_schema_name expected to be APP"), + new Condition<>(c -> c.equals("INTTABLE"), "fk_table_name expeced to be INTTABLE"), + new Condition<>( + c -> c.equals("FOREIGNID"), "fk_column_name expected to equal FOREIGNID"), + new Condition<>(c -> c.equals("1"), "key_sequence expected to equal 1"), + new Condition<>(c -> c.contains("SQL"), "fk_key_name expected to contain SQL"), + new Condition<>(c -> c.contains("SQL"), "pk_key_name expected to contain SQL"), + new Condition<>(c -> c.equals("3"), "update_rule expected to equal 3"), + new Condition<>(c -> c.equals("3"), "delete_rule expected to equal 3")); assertEquals(1, results.size()); final List assertions = new ArrayList<>(); for (int i = 0; i < matchers.size(); i++) { final String actual = results.get(0).get(i); - final Matcher expected = matchers.get(i); - assertions.add(() -> MatcherAssert.assertThat(actual, expected)); + final Condition expected = matchers.get(i); + assertions.add(() -> assertThat(actual).satisfies(expected)); } assertAll(assertions); } @@ -1431,7 +1422,7 @@ public void testGetTypeInfo() throws Exception { null, null, null)); - MatcherAssert.assertThat(results, is(matchers)); + assertThat(results).isEqualTo(matchers); } } @@ -1465,7 +1456,7 @@ public void testGetTypeInfoWithFiltering() throws Exception { null, "10", null)); - MatcherAssert.assertThat(results, is(matchers)); + assertThat(results).isEqualTo(matchers); } } @@ -1479,28 +1470,30 @@ public void testGetCommandCrossReference() throws Exception { final List> results = getResults(stream); - final List> matchers = + final List> matchers = asList( - nullValue(String.class), // pk_catalog_name - is("APP"), // pk_schema_name - is("FOREIGNTABLE"), // pk_table_name - is("ID"), // pk_column_name - nullValue(String.class), // fk_catalog_name - is("APP"), // fk_schema_name - is("INTTABLE"), // fk_table_name - is("FOREIGNID"), // fk_column_name - is("1"), // key_sequence - containsString("SQL"), // fk_key_name - containsString("SQL"), // pk_key_name - is("3"), // update_rule - is("3")); // delete_rule + new Condition<>(Objects::isNull, "pk_catalog_name expected to be null"), + new Condition<>(c -> c.equals("APP"), "pk_schema_name expected to equal APP"), + new Condition<>( + c -> c.equals("FOREIGNTABLE"), "pk_table_name should equal FOREIGNTABLE"), + new Condition<>(c -> c.equals("ID"), "pk_column_name should equal ID"), + new Condition<>(Objects::isNull, "fk_catalog_name expected to be null"), + new Condition<>(c -> c.equals("APP"), "fk_schema_name expected to be APP"), + new Condition<>(c -> c.equals("INTTABLE"), "fk_table_name expeced to be INTTABLE"), + new Condition<>( + c -> c.equals("FOREIGNID"), "fk_column_name expected to equal FOREIGNID"), + new Condition<>(c -> c.equals("1"), "key_sequence expected to equal 1"), + new Condition<>(c -> c.contains("SQL"), "fk_key_name expected to contain SQL"), + new Condition<>(c -> c.contains("SQL"), "pk_key_name expected to contain SQL"), + new Condition<>(c -> c.equals("3"), "update_rule expected to equal 3"), + new Condition<>(c -> c.equals("3"), "delete_rule expected to equal 3")); assertEquals(1, results.size()); final List assertions = new ArrayList<>(); for (int i = 0; i < matchers.size(); i++) { final String actual = results.get(0).get(i); - final Matcher expected = matchers.get(i); - assertions.add(() -> MatcherAssert.assertThat(actual, expected)); + final Condition expected = matchers.get(i); + assertions.add(() -> assertThat(actual).satisfies(expected)); } assertAll(assertions); } @@ -1509,7 +1502,7 @@ public void testGetCommandCrossReference() throws Exception { @Test public void testCreateStatementSchema() throws Exception { final FlightInfo info = sqlClient.execute("SELECT * FROM intTable"); - MatcherAssert.assertThat(info.getSchemaOptional(), is(Optional.of(SCHEMA_INT_TABLE))); + assertThat(info.getSchemaOptional()).isEqualTo(Optional.of(SCHEMA_INT_TABLE)); // Consume statement to close connection before cache eviction try (FlightStream stream = sqlClient.getStream(info.getEndpoints().get(0).getTicket())) { @@ -1526,11 +1519,10 @@ public void testCreateStatementResults() throws Exception { sqlClient.execute("SELECT * FROM intTable").getEndpoints().get(0).getTicket())) { assertAll( () -> { - MatcherAssert.assertThat(stream.getSchema(), is(SCHEMA_INT_TABLE)); + assertThat(stream.getSchema()).isEqualTo(SCHEMA_INT_TABLE); }, () -> { - MatcherAssert.assertThat( - getResults(stream), is(EXPECTED_RESULTS_FOR_STAR_SELECT_QUERY)); + assertThat(getResults(stream)).isEqualTo(EXPECTED_RESULTS_FOR_STAR_SELECT_QUERY); }); } } @@ -1543,19 +1535,19 @@ public void testExecuteUpdate() { sqlClient.executeUpdate( "INSERT INTO INTTABLE (keyName, value) VALUES " + "('KEYNAME1', 1001), ('KEYNAME2', 1002), ('KEYNAME3', 1003)"); - MatcherAssert.assertThat(insertedCount, is(3L)); + assertThat(insertedCount).isEqualTo(3L); }, () -> { long updatedCount = sqlClient.executeUpdate( "UPDATE INTTABLE SET keyName = 'KEYNAME1' " + "WHERE keyName = 'KEYNAME2' OR keyName = 'KEYNAME3'"); - MatcherAssert.assertThat(updatedCount, is(2L)); + assertThat(updatedCount).isEqualTo(2L); }, () -> { long deletedCount = sqlClient.executeUpdate("DELETE FROM INTTABLE WHERE keyName = 'KEYNAME1'"); - MatcherAssert.assertThat(deletedCount, is(3L)); + assertThat(deletedCount).isEqualTo(3L); }); } @@ -1566,10 +1558,10 @@ public void testQueryWithNoResultsShouldNotHang() throws Exception { final FlightStream stream = sqlClient.getStream(preparedStatement.execute().getEndpoints().get(0).getTicket())) { assertAll( - () -> MatcherAssert.assertThat(stream.getSchema(), is(SCHEMA_INT_TABLE)), + () -> assertThat(stream.getSchema()).isEqualTo(SCHEMA_INT_TABLE), () -> { final List> result = getResults(stream); - MatcherAssert.assertThat(result, is(emptyList())); + assertThat(result).isEqualTo(emptyList()); }); } } diff --git a/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStateless.java b/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStateless.java index 36d621ad64..ee1507b6af 100644 --- a/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStateless.java +++ b/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStateless.java @@ -18,7 +18,7 @@ import static org.apache.arrow.flight.sql.util.FlightStreamUtils.getResults; import static org.apache.arrow.util.AutoCloseables.close; -import static org.hamcrest.CoreMatchers.is; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertAll; import org.apache.arrow.flight.FlightClient; @@ -34,7 +34,6 @@ import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.types.pojo.Schema; -import org.hamcrest.MatcherAssert; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -89,10 +88,10 @@ public void testSimplePreparedStatementResultsWithParameterBinding() throws Exce for (FlightEndpoint endpoint : flightInfo.getEndpoints()) { try (FlightStream stream = sqlClient.getStream(endpoint.getTicket())) { assertAll( - () -> MatcherAssert.assertThat(stream.getSchema(), is(SCHEMA_INT_TABLE)), + () -> assertThat(stream.getSchema()).isEqualTo(SCHEMA_INT_TABLE), () -> - MatcherAssert.assertThat( - getResults(stream), is(EXPECTED_RESULTS_FOR_PARAMETER_BINDING))); + assertThat(getResults(stream)) + .isEqualTo(EXPECTED_RESULTS_FOR_PARAMETER_BINDING)); } } } diff --git a/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStreams.java b/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStreams.java index 71c0dc88e4..3f527f961e 100644 --- a/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStreams.java +++ b/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStreams.java @@ -22,7 +22,7 @@ import static org.apache.arrow.flight.sql.util.FlightStreamUtils.getResults; import static org.apache.arrow.util.AutoCloseables.close; import static org.apache.arrow.vector.types.Types.MinorType.INT; -import static org.hamcrest.CoreMatchers.is; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertAll; import com.google.common.collect.ImmutableList; @@ -53,7 +53,6 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.Text; -import org.hamcrest.MatcherAssert; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -245,15 +244,15 @@ public void testGetTablesResultNoSchema() throws Exception { sqlClient.getTables(null, null, null, null, false).getEndpoints().get(0).getTicket())) { assertAll( () -> - MatcherAssert.assertThat( - stream.getSchema(), is(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA_NO_SCHEMA)), + assertThat(stream.getSchema()) + .isEqualTo(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA_NO_SCHEMA), () -> { final List> results = getResults(stream); final List> expectedResults = ImmutableList.of( // catalog_name | schema_name | table_name | table_type | table_schema asList(null, null, "test_table", "TABLE")); - MatcherAssert.assertThat(results, is(expectedResults)); + assertThat(results).isEqualTo(expectedResults); }); } } @@ -264,15 +263,15 @@ public void testGetTableTypesResult() throws Exception { sqlClient.getStream(sqlClient.getTableTypes().getEndpoints().get(0).getTicket())) { assertAll( () -> - MatcherAssert.assertThat( - stream.getSchema(), is(FlightSqlProducer.Schemas.GET_TABLE_TYPES_SCHEMA)), + assertThat(stream.getSchema()) + .isEqualTo(FlightSqlProducer.Schemas.GET_TABLE_TYPES_SCHEMA), () -> { final List> tableTypes = getResults(stream); final List> expectedTableTypes = ImmutableList.of( // table_type singletonList("TABLE")); - MatcherAssert.assertThat(tableTypes, is(expectedTableTypes)); + assertThat(tableTypes).isEqualTo(expectedTableTypes); }); } } @@ -283,9 +282,9 @@ public void testGetSqlInfoResults() throws Exception { try (final FlightStream stream = sqlClient.getStream(info.getEndpoints().get(0).getTicket())) { assertAll( () -> - MatcherAssert.assertThat( - stream.getSchema(), is(FlightSqlProducer.Schemas.GET_SQL_INFO_SCHEMA)), - () -> MatcherAssert.assertThat(getResults(stream), is(emptyList()))); + assertThat(stream.getSchema()) + .isEqualTo(FlightSqlProducer.Schemas.GET_SQL_INFO_SCHEMA), + () -> assertThat(getResults(stream)).isEqualTo(emptyList())); } } @@ -303,7 +302,7 @@ public void testGetTypeInfo() throws Exception { "Integer", "4", "400", null, null, "3", "true", null, "true", null, "true", "Integer", null, null, "4", null, "10", null)); - MatcherAssert.assertThat(results, is(matchers)); + assertThat(results).isEqualTo(matchers); } } @@ -317,10 +316,8 @@ public void testExecuteQuery() throws Exception { .get(0) .getTicket())) { assertAll( - () -> - MatcherAssert.assertThat(stream.getSchema(), is(FlightSqlTestProducer.FIXED_SCHEMA)), - () -> - MatcherAssert.assertThat(getResults(stream), is(singletonList(singletonList("1"))))); + () -> assertThat(stream.getSchema()).isEqualTo(FlightSqlTestProducer.FIXED_SCHEMA), + () -> assertThat(getResults(stream)).isEqualTo(singletonList(singletonList("1")))); } } } From fad4e142f398492533bb1dea32accfcad0a33be8 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 2 Jun 2025 11:42:45 +0900 Subject: [PATCH 181/408] MINOR: Add missing permission to milestone assignment bot (#673) ## What's Changed This step needs permissions to write to issues so we can set the milestone. --- .github/workflows/dev_pr.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml index 34b3363c50..7352137b09 100644 --- a/.github/workflows/dev_pr.yml +++ b/.github/workflows/dev_pr.yml @@ -80,5 +80,9 @@ jobs: if: '! github.event.pull_request.draft' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + permissions: + contents: read + issues: write + pull-requests: write run: | ./.github/workflows/dev_pr_milestone.sh "${GITHUB_REPOSITORY}" ${{ github.event.number }} From 17958324c65e0c2109733ed4022ccd25b23e1ff2 Mon Sep 17 00:00:00 2001 From: rtadepalli <105760760+rtadepalli@users.noreply.github.com> Date: Wed, 4 Jun 2025 08:15:34 -0400 Subject: [PATCH 182/408] GH-774: Consoliate `BitVectorHelper.getValidityBufferSize` and `BaseValueVector.getValidityBufferSizeFromCount` (#775) ## What's Changed Just removing some duplicate functions in anticipation of cleaning out some transferPair duplication across complex vectors. Closes #774 --------- Co-authored-by: David Li --- .../arrow/vector/BaseFixedWidthVector.java | 10 ++++---- .../vector/BaseLargeVariableWidthVector.java | 6 ++--- .../apache/arrow/vector/BaseValueVector.java | 9 +++++++- .../arrow/vector/BaseVariableWidthVector.java | 6 ++--- .../vector/BaseVariableWidthViewVector.java | 6 ++--- .../org/apache/arrow/vector/BitVector.java | 6 ++--- .../apache/arrow/vector/BitVectorHelper.java | 12 +++++----- .../vector/complex/FixedSizeListVector.java | 16 +++++++------ .../arrow/vector/complex/LargeListVector.java | 18 ++++++++------- .../vector/complex/LargeListViewVector.java | 20 ++++++++-------- .../arrow/vector/complex/ListVector.java | 20 ++++++++-------- .../arrow/vector/complex/ListViewVector.java | 20 ++++++++-------- .../arrow/vector/complex/MapVector.java | 2 +- .../arrow/vector/complex/StructVector.java | 23 ++++++++++--------- .../arrow/vector/ipc/JsonFileReader.java | 3 ++- .../arrow/vector/TestLargeListVector.java | 3 ++- .../arrow/vector/TestLargeListViewVector.java | 3 ++- .../apache/arrow/vector/TestListVector.java | 3 ++- .../arrow/vector/TestListViewVector.java | 3 ++- .../apache/arrow/vector/TestValueVector.java | 13 ++++++----- .../vector/TestVariableWidthViewVector.java | 3 ++- .../arrow/vector/TestVectorUnloadLoad.java | 3 ++- 22 files changed, 117 insertions(+), 91 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java index 4be55396b7..d126266cf5 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java @@ -359,7 +359,7 @@ public int getBufferSizeFor(final int count) { if (count == 0) { return 0; } - return (count * typeWidth) + getValidityBufferSizeFromCount(count); + return (count * typeWidth) + BitVectorHelper.getValidityBufferSizeFromCount(count); } /** @@ -372,7 +372,7 @@ public int getBufferSize() { if (valueCount == 0) { return 0; } - return (valueCount * typeWidth) + getValidityBufferSizeFromCount(valueCount); + return (valueCount * typeWidth) + BitVectorHelper.getValidityBufferSizeFromCount(valueCount); } /** @@ -536,10 +536,10 @@ private void setReaderAndWriterIndex() { validityBuffer.writerIndex(0); valueBuffer.writerIndex(0); } else { - validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); if (typeWidth == 0) { /* specialized handling for BitVector */ - valueBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + valueBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); } else { valueBuffer.writerIndex((long) valueCount * typeWidth); } @@ -664,7 +664,7 @@ private void splitAndTransferValidityBuffer( int startIndex, int length, BaseFixedWidthVector target) { int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getValidityBufferSizeFromCount(length); + int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); int offset = startIndex % 8; if (length > 0) { diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java index 7e0d0affc6..4245e0053b 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java @@ -379,7 +379,7 @@ private void setReaderAndWriterIndex() { valueBuffer.writerIndex(0); } else { final long lastDataOffset = getStartOffset(valueCount); - validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); offsetBuffer.writerIndex((long) (valueCount + 1) * OFFSET_WIDTH); valueBuffer.writerIndex(lastDataOffset); } @@ -633,7 +633,7 @@ public int getBufferSizeFor(final int valueCount) { return 0; } - final long validityBufferSize = getValidityBufferSizeFromCount(valueCount); + final long validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount); final long offsetBufferSize = (long) (valueCount + 1) * OFFSET_WIDTH; /* get the end offset for this valueCount */ final long dataBufferSize = getStartOffset(valueCount); @@ -816,7 +816,7 @@ private void splitAndTransferValidityBuffer( int startIndex, int length, BaseLargeVariableWidthVector target) { int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getValidityBufferSizeFromCount(length); + int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); int offset = startIndex % 8; if (length > 0) { diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java index 9befcb890f..7bff431e40 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java @@ -110,7 +110,14 @@ protected ArrowBuf releaseBuffer(ArrowBuf buffer) { return buffer; } - /* number of bytes for the validity buffer for the given valueCount */ + /** + * Compute the size of validity buffer required to manage a given number of elements in a vector. + * + * @param valueCount number of elements in the vector + * @return buffer size + * @deprecated -- use {@link BitVectorHelper#getValidityBufferSizeFromCount} instead. + */ + @Deprecated(forRemoval = true, since = "18.4.0") protected static int getValidityBufferSizeFromCount(final int valueCount) { return DataSizeRoundingUtil.divideBy8Ceil(valueCount); } diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java index 1609e64ca5..4f681311ed 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java @@ -395,7 +395,7 @@ private void setReaderAndWriterIndex() { valueBuffer.writerIndex(0); } else { final int lastDataOffset = getStartOffset(valueCount); - validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); offsetBuffer.writerIndex((long) (valueCount + 1) * OFFSET_WIDTH); valueBuffer.writerIndex(lastDataOffset); } @@ -673,7 +673,7 @@ public int getBufferSizeFor(final int valueCount) { return 0; } - final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); + final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount); final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; /* get the end offset for this valueCount */ final int dataBufferSize = offsetBuffer.getInt((long) valueCount * OFFSET_WIDTH); @@ -867,7 +867,7 @@ private void splitAndTransferValidityBuffer( final int firstByteSource = BitVectorHelper.byteIndex(startIndex); final int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - final int byteSizeTarget = getValidityBufferSizeFromCount(length); + final int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); final int offset = startIndex % 8; if (offset == 0) { diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java index beda91dc3f..5e25ffa568 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java @@ -400,7 +400,7 @@ private void setReaderAndWriterIndex() { validityBuffer.writerIndex(0); viewBuffer.writerIndex(0); } else { - validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); viewBuffer.writerIndex(valueCount * ELEMENT_SIZE); } } @@ -683,7 +683,7 @@ public int getBufferSizeFor(final int valueCount) { return 0; } - final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); + final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount); final int viewBufferSize = valueCount * ELEMENT_SIZE; final int dataBufferSize = getDataBufferSize(); return validityBufferSize + viewBufferSize + dataBufferSize; @@ -872,7 +872,7 @@ private void splitAndTransferValidityBuffer( final int firstByteSource = BitVectorHelper.byteIndex(startIndex); final int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - final int byteSizeTarget = getValidityBufferSizeFromCount(length); + final int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); final int offset = startIndex % 8; if (offset == 0) { diff --git a/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/vector/src/main/java/org/apache/arrow/vector/BitVector.java index f8e3342625..ecee02f665 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BitVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BitVector.java @@ -98,7 +98,7 @@ public MinorType getMinorType() { */ @Override public void setInitialCapacity(int valueCount) { - final int size = getValidityBufferSizeFromCount(valueCount); + final int size = BitVectorHelper.getValidityBufferSizeFromCount(valueCount); if (size * 2L > MAX_ALLOCATION_SIZE) { throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); } @@ -121,7 +121,7 @@ public int getBufferSizeFor(final int count) { if (count == 0) { return 0; } - return 2 * getValidityBufferSizeFromCount(count); + return 2 * BitVectorHelper.getValidityBufferSizeFromCount(count); } /** @@ -165,7 +165,7 @@ private ArrowBuf splitAndTransferBuffer( int startIndex, int length, ArrowBuf sourceBuffer, ArrowBuf destBuffer) { int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getValidityBufferSizeFromCount(length); + int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); int offset = startIndex % 8; if (length > 0) { diff --git a/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java b/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java index 0ac56691a6..bc2c3da98f 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java +++ b/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java @@ -135,11 +135,11 @@ public static void setValidityBit(ArrowBuf validityBuffer, int index, int value) public static ArrowBuf setValidityBit( ArrowBuf validityBuffer, BufferAllocator allocator, int valueCount, int index, int value) { if (validityBuffer == null) { - validityBuffer = allocator.buffer(getValidityBufferSize(valueCount)); + validityBuffer = allocator.buffer(getValidityBufferSizeFromCount(valueCount)); } setValidityBit(validityBuffer, index, value); if (index == (valueCount - 1)) { - validityBuffer.writerIndex(getValidityBufferSize(valueCount)); + validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); } return validityBuffer; @@ -165,7 +165,7 @@ public static int get(final ArrowBuf buffer, int index) { * @param valueCount number of elements in the vector * @return buffer size */ - public static int getValidityBufferSize(int valueCount) { + public static int getValidityBufferSizeFromCount(int valueCount) { return DataSizeRoundingUtil.divideBy8Ceil(valueCount); } @@ -182,7 +182,7 @@ public static int getNullCount(final ArrowBuf validityBuffer, final int valueCou return 0; } int count = 0; - final int sizeInBytes = getValidityBufferSize(valueCount); + final int sizeInBytes = getValidityBufferSizeFromCount(valueCount); // If value count is not a multiple of 8, then calculate number of used bits in the last byte final int remainder = valueCount % 8; final int fullBytesCount = remainder == 0 ? sizeInBytes : sizeInBytes - 1; @@ -233,7 +233,7 @@ public static boolean checkAllBitsEqualTo( if (valueCount == 0) { return true; } - final int sizeInBytes = getValidityBufferSize(valueCount); + final int sizeInBytes = getValidityBufferSizeFromCount(valueCount); // boundary check validityBuffer.checkBytes(0, sizeInBytes); @@ -325,7 +325,7 @@ public static ArrowBuf loadValidityBuffer( sourceValidityBuffer == null || sourceValidityBuffer.capacity() == 0; if (isValidityBufferNull && (fieldNode.getNullCount() == 0 || fieldNode.getNullCount() == valueCount)) { - newBuffer = allocator.buffer(getValidityBufferSize(valueCount)); + newBuffer = allocator.buffer(getValidityBufferSizeFromCount(valueCount)); newBuffer.setZero(0, newBuffer.capacity()); if (fieldNode.getNullCount() != 0) { /* all NULLs */ diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index c762eb5172..36d9ff40ed 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -110,7 +110,8 @@ public FixedSizeListVector( this.listSize = ((ArrowType.FixedSizeList) field.getFieldType().getType()).getListSize(); Preconditions.checkArgument(listSize >= 0, "list size must be non-negative"); this.valueCount = 0; - this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); + this.validityAllocationSizeInBytes = + BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); } @Override @@ -189,7 +190,7 @@ public List getFieldBuffers() { private void setReaderAndWriterIndex() { validityBuffer.readerIndex(0); - validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); } /** @@ -268,7 +269,8 @@ private void reallocValidityBuffer() { if (validityAllocationSizeInBytes > 0) { newAllocationSize = validityAllocationSizeInBytes; } else { - newAllocationSize = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L; + newAllocationSize = + BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L; } } @@ -311,7 +313,7 @@ public UnionFixedSizeListWriter getWriter() { @Override public void setInitialCapacity(int numRecords) { - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords); vector.setInitialCapacity(numRecords * listSize); } @@ -328,7 +330,7 @@ public int getBufferSize() { if (getValueCount() == 0) { return 0; } - return getValidityBufferSizeFromCount(valueCount) + vector.getBufferSize(); + return BitVectorHelper.getValidityBufferSizeFromCount(valueCount) + vector.getBufferSize(); } @Override @@ -336,7 +338,7 @@ public int getBufferSizeFor(int valueCount) { if (valueCount == 0) { return 0; } - return getValidityBufferSizeFromCount(valueCount) + return BitVectorHelper.getValidityBufferSizeFromCount(valueCount) + vector.getBufferSizeFor(valueCount * listSize); } @@ -654,7 +656,7 @@ private void splitAndTransferValidityBuffer( int startIndex, int length, FixedSizeListVector target) { int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getValidityBufferSizeFromCount(length); + int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); int offset = startIndex % 8; if (length > 0) { diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java index ed075352c9..71633441cb 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java @@ -131,7 +131,8 @@ public LargeListVector(Field field, BufferAllocator allocator, CallBack callBack this.field = field; this.validityBuffer = allocator.getEmpty(); this.callBack = callBack; - this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); + this.validityAllocationSizeInBytes = + BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); this.lastSet = -1; this.offsetBuffer = allocator.getEmpty(); this.vector = vector == null ? DEFAULT_DATA_VECTOR : vector; @@ -156,7 +157,7 @@ public void initializeChildrenFromFields(List children) { @Override public void setInitialCapacity(int numRecords) { - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords); offsetAllocationSizeInBytes = (long) (numRecords + 1) * OFFSET_WIDTH; if (vector instanceof BaseFixedWidthVector || vector instanceof BaseVariableWidthVector) { vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD); @@ -184,7 +185,7 @@ public void setInitialCapacity(int numRecords) { */ @Override public void setInitialCapacity(int numRecords, double density) { - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords); if ((numRecords * density) >= Integer.MAX_VALUE) { throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); } @@ -311,7 +312,7 @@ private void setReaderAndWriterIndex() { validityBuffer.writerIndex(0); offsetBuffer.writerIndex(0); } else { - validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH); } } @@ -442,7 +443,8 @@ private void reallocValidityBuffer() { if (validityAllocationSizeInBytes > 0) { newAllocationSize = validityAllocationSizeInBytes; } else { - newAllocationSize = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L; + newAllocationSize = + BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L; } } newAllocationSize = CommonUtil.nextPowerOfTwo(newAllocationSize); @@ -699,7 +701,7 @@ private void splitAndTransferValidityBuffer( int startIndex, int length, LargeListVector target) { int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getValidityBufferSizeFromCount(length); + int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); int offset = startIndex % 8; if (length > 0) { @@ -821,7 +823,7 @@ public int getBufferSize() { return 0; } final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; - final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); + final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount); return offsetBufferSize + validityBufferSize + vector.getBufferSize(); } @@ -830,7 +832,7 @@ public int getBufferSizeFor(int valueCount) { if (valueCount == 0) { return 0; } - final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); + final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount); long innerVectorValueCount = offsetBuffer.getLong((long) valueCount * OFFSET_WIDTH); return ((valueCount + 1) * OFFSET_WIDTH) diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java index 84c6f03edb..1b7e6b2280 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java @@ -113,7 +113,8 @@ public LargeListViewVector(Field field, BufferAllocator allocator, CallBack call this.validityBuffer = allocator.getEmpty(); this.field = field; this.callBack = callBack; - this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); + this.validityAllocationSizeInBytes = + BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); } @Override @@ -134,7 +135,7 @@ public void initializeChildrenFromFields(List children) { @Override public void setInitialCapacity(int numRecords) { - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords); super.setInitialCapacity(numRecords); } @@ -157,7 +158,7 @@ public void setInitialCapacity(int numRecords) { */ @Override public void setInitialCapacity(int numRecords, double density) { - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords); super.setInitialCapacity(numRecords, density); } @@ -176,7 +177,7 @@ public void setInitialCapacity(int numRecords, double density) { */ @Override public void setInitialTotalCapacity(int numRecords, int totalNumberOfElements) { - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords); super.setInitialTotalCapacity(numRecords, totalNumberOfElements); } @@ -226,7 +227,7 @@ private void setReaderAndWriterIndex() { offsetBuffer.writerIndex(0); sizeBuffer.writerIndex(0); } else { - validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); offsetBuffer.writerIndex((long) valueCount * OFFSET_WIDTH); sizeBuffer.writerIndex((long) valueCount * SIZE_WIDTH); } @@ -323,7 +324,8 @@ private long getNewAllocationSize(int currentBufferCapacity) { if (validityAllocationSizeInBytes > 0) { newAllocationSize = validityAllocationSizeInBytes; } else { - newAllocationSize = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L; + newAllocationSize = + BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L; } } newAllocationSize = CommonUtil.nextPowerOfTwo(newAllocationSize); @@ -536,7 +538,7 @@ private void splitAndTransferValidityBuffer( int startIndex, int length, LargeListViewVector target) { int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getValidityBufferSizeFromCount(length); + int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); int offset = startIndex % 8; if (length > 0) { @@ -629,7 +631,7 @@ public int getBufferSize() { } final int offsetBufferSize = valueCount * OFFSET_WIDTH; final int sizeBufferSize = valueCount * SIZE_WIDTH; - final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); + final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount); return offsetBufferSize + sizeBufferSize + validityBufferSize + vector.getBufferSize(); } @@ -644,7 +646,7 @@ public int getBufferSizeFor(int valueCount) { if (valueCount == 0) { return 0; } - final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); + final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount); return super.getBufferSizeFor(valueCount) + validityBufferSize; } diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 3daeb6d77b..a8e8dcc436 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -108,7 +108,8 @@ public ListVector(Field field, BufferAllocator allocator, CallBack callBack) { this.validityBuffer = allocator.getEmpty(); this.field = field; this.callBack = callBack; - this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); + this.validityAllocationSizeInBytes = + BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); this.lastSet = -1; } @@ -130,7 +131,7 @@ public void initializeChildrenFromFields(List children) { @Override public void setInitialCapacity(int numRecords) { - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords); super.setInitialCapacity(numRecords); } @@ -153,7 +154,7 @@ public void setInitialCapacity(int numRecords) { */ @Override public void setInitialCapacity(int numRecords, double density) { - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords); super.setInitialCapacity(numRecords, density); } @@ -172,7 +173,7 @@ public void setInitialCapacity(int numRecords, double density) { */ @Override public void setInitialTotalCapacity(int numRecords, int totalNumberOfElements) { - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords); super.setInitialTotalCapacity(numRecords, totalNumberOfElements); } @@ -269,7 +270,7 @@ private void setReaderAndWriterIndex() { validityBuffer.writerIndex(0); offsetBuffer.writerIndex(0); } else { - validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH); } } @@ -366,7 +367,8 @@ private long getNewAllocationSize(int currentBufferCapacity) { if (validityAllocationSizeInBytes > 0) { newAllocationSize = validityAllocationSizeInBytes; } else { - newAllocationSize = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L; + newAllocationSize = + BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L; } } newAllocationSize = CommonUtil.nextPowerOfTwo(newAllocationSize); @@ -579,7 +581,7 @@ public void splitAndTransfer(int startIndex, int length) { private void splitAndTransferValidityBuffer(int startIndex, int length, ListVector target) { int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getValidityBufferSizeFromCount(length); + int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); int offset = startIndex % 8; if (length > 0) { @@ -678,7 +680,7 @@ public int getBufferSize() { return 0; } final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; - final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); + final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount); return offsetBufferSize + validityBufferSize + vector.getBufferSize(); } @@ -687,7 +689,7 @@ public int getBufferSizeFor(int valueCount) { if (valueCount == 0) { return 0; } - final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); + final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount); return super.getBufferSizeFor(valueCount) + validityBufferSize; } diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java index 9b4e6b4c0c..ada25bbaf5 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java @@ -112,7 +112,8 @@ public ListViewVector(Field field, BufferAllocator allocator, CallBack callBack) this.validityBuffer = allocator.getEmpty(); this.field = field; this.callBack = callBack; - this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); + this.validityAllocationSizeInBytes = + BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); } @Override @@ -133,7 +134,7 @@ public void initializeChildrenFromFields(List children) { @Override public void setInitialCapacity(int numRecords) { - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords); super.setInitialCapacity(numRecords); } @@ -156,7 +157,7 @@ public void setInitialCapacity(int numRecords) { */ @Override public void setInitialCapacity(int numRecords, double density) { - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords); super.setInitialCapacity(numRecords, density); } @@ -175,7 +176,7 @@ public void setInitialCapacity(int numRecords, double density) { */ @Override public void setInitialTotalCapacity(int numRecords, int totalNumberOfElements) { - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords); super.setInitialTotalCapacity(numRecords, totalNumberOfElements); } @@ -225,7 +226,7 @@ private void setReaderAndWriterIndex() { offsetBuffer.writerIndex(0); sizeBuffer.writerIndex(0); } else { - validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); offsetBuffer.writerIndex(valueCount * OFFSET_WIDTH); sizeBuffer.writerIndex(valueCount * SIZE_WIDTH); } @@ -322,7 +323,8 @@ private long getNewAllocationSize(int currentBufferCapacity) { if (validityAllocationSizeInBytes > 0) { newAllocationSize = validityAllocationSizeInBytes; } else { - newAllocationSize = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L; + newAllocationSize = + BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L; } } newAllocationSize = CommonUtil.nextPowerOfTwo(newAllocationSize); @@ -542,7 +544,7 @@ public void splitAndTransfer(int startIndex, int length) { private void splitAndTransferValidityBuffer(int startIndex, int length, ListViewVector target) { int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getValidityBufferSizeFromCount(length); + int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); int offset = startIndex % 8; if (length > 0) { @@ -634,7 +636,7 @@ public int getBufferSize() { } final int offsetBufferSize = valueCount * OFFSET_WIDTH; final int sizeBufferSize = valueCount * SIZE_WIDTH; - final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); + final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount); return offsetBufferSize + sizeBufferSize + validityBufferSize + vector.getBufferSize(); } @@ -649,7 +651,7 @@ public int getBufferSizeFor(int valueCount) { if (valueCount == 0) { return 0; } - final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); + final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount); return super.getBufferSizeFor(valueCount) + validityBufferSize; } diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java index 23cda8401b..5eb857ab94 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java @@ -238,7 +238,7 @@ public void splitAndTransfer(int startIndex, int length) { private void splitAndTransferValidityBuffer(int startIndex, int length, MapVector target) { int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getValidityBufferSizeFromCount(length); + int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); int offset = startIndex % 8; if (length > 0) { diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java index ca5f572034..5e5bb7fc21 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java @@ -18,6 +18,7 @@ import static org.apache.arrow.memory.util.LargeMemoryUtil.checkedCastToInt; import static org.apache.arrow.util.Preconditions.checkNotNull; +import static org.apache.arrow.vector.BitVectorHelper.getValidityBufferSizeFromCount; import java.util.ArrayList; import java.util.Arrays; @@ -89,7 +90,7 @@ public StructVector( super(name, checkNotNull(allocator), fieldType, callBack); this.validityBuffer = allocator.getEmpty(); this.validityAllocationSizeInBytes = - BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION); + getValidityBufferSizeFromCount(BaseValueVector.INITIAL_VALUE_ALLOCATION); } /** @@ -118,7 +119,7 @@ public StructVector( allowConflictPolicyChanges); this.validityBuffer = allocator.getEmpty(); this.validityAllocationSizeInBytes = - BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION); + getValidityBufferSizeFromCount(BaseValueVector.INITIAL_VALUE_ALLOCATION); } /** @@ -132,7 +133,7 @@ public StructVector(Field field, BufferAllocator allocator, CallBack callBack) { super(field, checkNotNull(allocator), callBack); this.validityBuffer = allocator.getEmpty(); this.validityAllocationSizeInBytes = - BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION); + getValidityBufferSizeFromCount(BaseValueVector.INITIAL_VALUE_ALLOCATION); } /** @@ -153,7 +154,7 @@ public StructVector( super(field, checkNotNull(allocator), callBack, conflictPolicy, allowConflictPolicyChanges); this.validityBuffer = allocator.getEmpty(); this.validityAllocationSizeInBytes = - BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION); + getValidityBufferSizeFromCount(BaseValueVector.INITIAL_VALUE_ALLOCATION); } @Override @@ -182,7 +183,7 @@ public List getFieldBuffers() { private void setReaderAndWriterIndex() { validityBuffer.readerIndex(0); - validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSize(valueCount)); + validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); } /** @@ -318,7 +319,7 @@ public void splitAndTransfer(int startIndex, int length) { private void splitAndTransferValidityBuffer(int startIndex, int length, StructVector target) { int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = BitVectorHelper.getValidityBufferSize(length); + int byteSizeTarget = getValidityBufferSizeFromCount(length); int offset = startIndex % 8; if (length > 0) { @@ -464,7 +465,7 @@ public int getBufferSize() { if (valueCount == 0) { return 0; } - return super.getBufferSize() + BitVectorHelper.getValidityBufferSize(valueCount); + return super.getBufferSize() + getValidityBufferSizeFromCount(valueCount); } /** @@ -478,18 +479,18 @@ public int getBufferSizeFor(final int valueCount) { if (valueCount == 0) { return 0; } - return super.getBufferSizeFor(valueCount) + BitVectorHelper.getValidityBufferSize(valueCount); + return super.getBufferSizeFor(valueCount) + getValidityBufferSizeFromCount(valueCount); } @Override public void setInitialCapacity(int numRecords) { - validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(numRecords); + validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); super.setInitialCapacity(numRecords); } @Override public void setInitialCapacity(int numRecords, double density) { - validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(numRecords); + validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); super.setInitialCapacity(numRecords, density); } @@ -547,7 +548,7 @@ private long getNewAllocationSize(int currentBufferCapacity) { newAllocationSize = validityAllocationSizeInBytes; } else { newAllocationSize = - BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION) * 2L; + getValidityBufferSizeFromCount(BaseValueVector.INITIAL_VALUE_ALLOCATION) * 2L; } } newAllocationSize = CommonUtil.nextPowerOfTwo(newAllocationSize); diff --git a/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java b/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java index fe0803d298..e4bab7eb80 100644 --- a/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java +++ b/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java @@ -20,6 +20,7 @@ import static com.fasterxml.jackson.core.JsonToken.END_OBJECT; import static com.fasterxml.jackson.core.JsonToken.START_ARRAY; import static com.fasterxml.jackson.core.JsonToken.START_OBJECT; +import static org.apache.arrow.vector.BitVectorHelper.getValidityBufferSizeFromCount; import static org.apache.arrow.vector.BufferLayout.BufferType.DATA; import static org.apache.arrow.vector.BufferLayout.BufferType.OFFSET; import static org.apache.arrow.vector.BufferLayout.BufferType.SIZE; @@ -381,7 +382,7 @@ private class BufferHelper { new BufferReader() { @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { - final int bufferSize = BitVectorHelper.getValidityBufferSize(count); + final int bufferSize = getValidityBufferSizeFromCount(count); ArrowBuf buf = allocator.buffer(bufferSize); // C++ integration test fails without this. diff --git a/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java index 101d942d2a..d5cbf925b2 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.vector; +import static org.apache.arrow.vector.BitVectorHelper.getValidityBufferSizeFromCount; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; @@ -943,7 +944,7 @@ public void testGetBufferSizeFor() { int[] indices = new int[] {0, 2, 4, 6, 10, 14}; for (int valueCount = 1; valueCount <= 5; valueCount++) { - int validityBufferSize = BitVectorHelper.getValidityBufferSize(valueCount); + int validityBufferSize = getValidityBufferSizeFromCount(valueCount); int offsetBufferSize = (valueCount + 1) * LargeListVector.OFFSET_WIDTH; int expectedSize = diff --git a/vector/src/test/java/org/apache/arrow/vector/TestLargeListViewVector.java b/vector/src/test/java/org/apache/arrow/vector/TestLargeListViewVector.java index 26e7bb4a0d..256aa99687 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestLargeListViewVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestLargeListViewVector.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.vector; +import static org.apache.arrow.vector.BitVectorHelper.getValidityBufferSizeFromCount; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertSame; @@ -1062,7 +1063,7 @@ public void testGetBufferSizeFor() { int[] indices = new int[] {0, 2, 4, 6, 10, 14}; for (int valueCount = 1; valueCount <= 5; valueCount++) { - int validityBufferSize = BitVectorHelper.getValidityBufferSize(valueCount); + int validityBufferSize = getValidityBufferSizeFromCount(valueCount); int offsetBufferSize = valueCount * BaseLargeRepeatedValueViewVector.OFFSET_WIDTH; int sizeBufferSize = valueCount * BaseLargeRepeatedValueViewVector.SIZE_WIDTH; diff --git a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index 1d6fa39f9e..5b2043a014 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.vector; +import static org.apache.arrow.vector.BitVectorHelper.getValidityBufferSizeFromCount; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; @@ -1123,7 +1124,7 @@ public void testGetBufferSizeFor() { int[] indices = new int[] {0, 2, 4, 6, 10, 14}; for (int valueCount = 1; valueCount <= 5; valueCount++) { - int validityBufferSize = BitVectorHelper.getValidityBufferSize(valueCount); + int validityBufferSize = getValidityBufferSizeFromCount(valueCount); int offsetBufferSize = (valueCount + 1) * BaseRepeatedValueVector.OFFSET_WIDTH; int expectedSize = diff --git a/vector/src/test/java/org/apache/arrow/vector/TestListViewVector.java b/vector/src/test/java/org/apache/arrow/vector/TestListViewVector.java index 639585fc48..2f282e1988 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestListViewVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestListViewVector.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.vector; +import static org.apache.arrow.vector.BitVectorHelper.getValidityBufferSizeFromCount; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -1075,7 +1076,7 @@ public void testGetBufferSizeFor() { int[] indices = new int[] {0, 2, 4, 6, 10, 14}; for (int valueCount = 1; valueCount <= 5; valueCount++) { - int validityBufferSize = BitVectorHelper.getValidityBufferSize(valueCount); + int validityBufferSize = getValidityBufferSizeFromCount(valueCount); int offsetBufferSize = valueCount * BaseRepeatedValueViewVector.OFFSET_WIDTH; int sizeBufferSize = valueCount * BaseRepeatedValueViewVector.SIZE_WIDTH; diff --git a/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index daec331831..ac82246671 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.vector; +import static org.apache.arrow.vector.BitVectorHelper.getValidityBufferSizeFromCount; import static org.apache.arrow.vector.TestUtils.newVarBinaryVector; import static org.apache.arrow.vector.TestUtils.newVarCharVector; import static org.apache.arrow.vector.TestUtils.newVector; @@ -1233,7 +1234,7 @@ public void testSplitAndTransfer3() { // the size needed for the validity buffer final long validitySize = DefaultRoundingPolicy.DEFAULT_ROUNDING_POLICY.getRoundedSize( - BaseValueVector.getValidityBufferSizeFromCount(2)); + getValidityBufferSizeFromCount(2)); assertEquals(allocatedMem + validitySize, allocator.getAllocatedMemory()); // The validity and offset buffers are sliced from a same buffer.See // BaseFixedWidthVector#allocateBytes. @@ -2464,7 +2465,7 @@ public void testDefaultAllocNewAll() { assertTrue(intVector.getValueCapacity() >= defaultCapacity); expectedSize = (defaultCapacity * IntVector.TYPE_WIDTH) - + BaseFixedWidthVector.getValidityBufferSizeFromCount(defaultCapacity); + + getValidityBufferSizeFromCount(defaultCapacity); assertTrue(childAllocator.getAllocatedMemory() - beforeSize <= expectedSize * 1.05); // verify that the wastage is within bounds for BigIntVector. @@ -2473,7 +2474,7 @@ public void testDefaultAllocNewAll() { assertTrue(bigIntVector.getValueCapacity() >= defaultCapacity); expectedSize = (defaultCapacity * bigIntVector.TYPE_WIDTH) - + BaseFixedWidthVector.getValidityBufferSizeFromCount(defaultCapacity); + + getValidityBufferSizeFromCount(defaultCapacity); assertTrue(childAllocator.getAllocatedMemory() - beforeSize <= expectedSize * 1.05); // verify that the wastage is within bounds for DecimalVector. @@ -2482,7 +2483,7 @@ public void testDefaultAllocNewAll() { assertTrue(decimalVector.getValueCapacity() >= defaultCapacity); expectedSize = (defaultCapacity * decimalVector.TYPE_WIDTH) - + BaseFixedWidthVector.getValidityBufferSizeFromCount(defaultCapacity); + + getValidityBufferSizeFromCount(defaultCapacity); assertTrue(childAllocator.getAllocatedMemory() - beforeSize <= expectedSize * 1.05); // verify that the wastage is within bounds for VarCharVector. @@ -2492,7 +2493,7 @@ public void testDefaultAllocNewAll() { assertTrue(varCharVector.getValueCapacity() >= defaultCapacity - 1); expectedSize = (defaultCapacity * VarCharVector.OFFSET_WIDTH) - + BaseFixedWidthVector.getValidityBufferSizeFromCount(defaultCapacity) + + getValidityBufferSizeFromCount(defaultCapacity) + defaultCapacity * 8; // wastage should be less than 5%. assertTrue(childAllocator.getAllocatedMemory() - beforeSize <= expectedSize * 1.05); @@ -2501,7 +2502,7 @@ public void testDefaultAllocNewAll() { beforeSize = childAllocator.getAllocatedMemory(); bitVector.allocateNew(); assertTrue(bitVector.getValueCapacity() >= defaultCapacity); - expectedSize = BaseFixedWidthVector.getValidityBufferSizeFromCount(defaultCapacity) * 2; + expectedSize = getValidityBufferSizeFromCount(defaultCapacity) * 2; assertTrue(childAllocator.getAllocatedMemory() - beforeSize <= expectedSize * 1.05); } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestVariableWidthViewVector.java b/vector/src/test/java/org/apache/arrow/vector/TestVariableWidthViewVector.java index 7a3a1bae63..f7c66a00be 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestVariableWidthViewVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestVariableWidthViewVector.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.vector; +import static org.apache.arrow.vector.BitVectorHelper.getValidityBufferSizeFromCount; import static org.apache.arrow.vector.TestUtils.newVector; import static org.apache.arrow.vector.TestUtils.newViewVarBinaryVector; import static org.apache.arrow.vector.TestUtils.newViewVarCharVector; @@ -2367,7 +2368,7 @@ private void testSplitAndTransferOnValiditySplitHelper( // the allocation only consists in the size needed for the validity buffer final long validitySize = DefaultRoundingPolicy.DEFAULT_ROUNDING_POLICY.getRoundedSize( - BaseValueVector.getValidityBufferSizeFromCount(2)); + getValidityBufferSizeFromCount(2)); // we allocate view and data buffers for the target vector assertTrue(allocatedMem + validitySize < allocator.getAllocatedMemory()); // The validity is sliced from the same buffer.See BaseFixedWidthViewVector#allocateBytes. diff --git a/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java index 6121fb67fe..782535fccc 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java @@ -17,6 +17,7 @@ package org.apache.arrow.vector; import static java.util.Arrays.asList; +import static org.apache.arrow.vector.BitVectorHelper.getValidityBufferSizeFromCount; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -215,7 +216,7 @@ public void testLoadValidityBuffer() throws IOException { int count = 10; ArrowBuf[] values = new ArrowBuf[4]; for (int i = 0; i < 4; i += 2) { - ArrowBuf buf1 = allocator.buffer(BitVectorHelper.getValidityBufferSize(count)); + ArrowBuf buf1 = allocator.buffer(getValidityBufferSizeFromCount(count)); ArrowBuf buf2 = allocator.buffer(count * 4); // integers buf1.setZero(0, buf1.capacity()); buf2.setZero(0, buf2.capacity()); From acbc138f75711fc94e842f9b306a48a5a233bca8 Mon Sep 17 00:00:00 2001 From: rtadepalli <105760760+rtadepalli@users.noreply.github.com> Date: Wed, 4 Jun 2025 22:57:25 -0400 Subject: [PATCH 183/408] GH-79: Move `splitAndTransferValidityBuffer` to `BaseValueVector` (#777) ## What's Changed Move `splitAndTransferValidityBuffer` up to `BaseValueVector`. This PR is not touching the implementation of this function in `StructVector` -- that is not being derived from `BaseValueVector` so some amount of duplication is probably fine. Closes #79 --- .../arrow/vector/BaseFixedWidthVector.java | 86 +++---------- .../vector/BaseLargeVariableWidthVector.java | 79 ++---------- .../apache/arrow/vector/BaseValueVector.java | 116 ++++++++++++++++++ .../arrow/vector/BaseVariableWidthVector.java | 79 ++---------- .../vector/BaseVariableWidthViewVector.java | 85 ++----------- .../BaseLargeRepeatedValueViewVector.java | 1 - .../complex/BaseRepeatedValueVector.java | 1 - .../complex/BaseRepeatedValueViewVector.java | 1 - .../vector/complex/FixedSizeListVector.java | 77 +----------- .../arrow/vector/complex/LargeListVector.java | 77 +----------- .../vector/complex/LargeListViewVector.java | 74 +---------- .../arrow/vector/complex/ListVector.java | 73 +---------- .../arrow/vector/complex/ListViewVector.java | 73 +---------- .../arrow/vector/complex/MapVector.java | 65 ---------- 14 files changed, 182 insertions(+), 705 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java index d126266cf5..f6e2a3b225 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java @@ -49,9 +49,7 @@ public abstract class BaseFixedWidthVector extends BaseValueVector protected final Field field; private int allocationMonitor; - protected ArrowBuf validityBuffer; protected ArrowBuf valueBuffer; - protected int valueCount; /** * Constructs a new instance. @@ -87,7 +85,7 @@ public String getName() { /* TODO: * Once the entire hierarchy has been refactored, move common functions - * like getNullCount(), splitAndTransferValidityBuffer to top level + * like getNullCount() to top level * base class BaseValueVector. * * Along with this, some class members (validityBuffer) can also be @@ -342,9 +340,9 @@ private void allocateBytes(int valueCount) { * slice the source buffer so we have to explicitly allocate the validityBuffer of the target * vector. This is unlike the databuffer which we can always slice for the target vector. */ - private void allocateValidityBuffer(final int validityBufferSize) { - validityBuffer = allocator.buffer(validityBufferSize); - validityBuffer.readerIndex(0); + @Override + protected void allocateValidityBuffer(final long validityBufferSize) { + super.allocateValidityBuffer(validityBufferSize); refreshValueCapacity(); } @@ -656,72 +654,18 @@ private void splitAndTransferValueBuffer( target.refreshValueCapacity(); } - /** - * Validity buffer has multiple cases of split and transfer depending on the starting position of - * the source index. - */ - private void splitAndTransferValidityBuffer( - int startIndex, int length, BaseFixedWidthVector target) { - int firstByteSource = BitVectorHelper.byteIndex(startIndex); - int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); - int offset = startIndex % 8; - - if (length > 0) { - if (offset == 0) { - /* slice */ - if (target.validityBuffer != null) { - target.validityBuffer.getReferenceManager().release(); - } - ArrowBuf slicedValidityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer = transferBuffer(slicedValidityBuffer, target.allocator); - target.refreshValueCapacity(); - } else { - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ - target.allocateValidityBuffer(byteSizeTarget); - - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - this.validityBuffer, firstByteSource + i, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - this.validityBuffer, firstByteSource + i + 1, offset); - - target.validityBuffer.setByte(i, (b1 + b2)); - } - - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ - if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - this.validityBuffer, firstByteSource + byteSizeTarget, offset); - - target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } else { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - target.validityBuffer.setByte(byteSizeTarget - 1, b1); - } - } + @Override + protected void sliceAndTransferValidityBuffer( + int startIndex, int length, BaseValueVector target) { + final int firstByteSource = BitVectorHelper.byteIndex(startIndex); + final int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); + + if (target.validityBuffer != null) { + target.validityBuffer.getReferenceManager().release(); } + ArrowBuf slicedValidityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer = transferBuffer(slicedValidityBuffer, target.allocator); + ((BaseFixedWidthVector) target).refreshValueCapacity(); } /*----------------------------------------------------------------* diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java index 4245e0053b..6c451f10a7 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java @@ -52,10 +52,8 @@ public abstract class BaseLargeVariableWidthVector extends BaseValueVector /* protected members */ public static final int OFFSET_WIDTH = 8; /* 8 byte unsigned int to track offsets */ protected static final byte[] emptyByteArray = new byte[] {}; - protected ArrowBuf validityBuffer; protected ArrowBuf valueBuffer; protected ArrowBuf offsetBuffer; - protected int valueCount; protected int lastSet; protected final Field field; @@ -501,10 +499,9 @@ private ArrowBuf allocateOffsetBuffer(final long size) { } /* allocate validity buffer */ - private void allocateValidityBuffer(final long size) { - validityBuffer = allocator.buffer(size); - validityBuffer.readerIndex(0); - initValidityBuffer(); + @Override + protected void allocateValidityBuffer(final long size) { + super.allocateValidityBuffer(size); } /** @@ -809,69 +806,17 @@ private void splitAndTransferOffsetBuffer( target.valueBuffer = transferBuffer(slicedBuffer, target.allocator); } - /* - * Transfer the validity. - */ - private void splitAndTransferValidityBuffer( - int startIndex, int length, BaseLargeVariableWidthVector target) { - int firstByteSource = BitVectorHelper.byteIndex(startIndex); - int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); - int offset = startIndex % 8; + @Override + protected void sliceAndTransferValidityBuffer( + int startIndex, int length, BaseValueVector target) { + final int firstByteSource = BitVectorHelper.byteIndex(startIndex); + final int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); - if (length > 0) { - if (offset == 0) { - // slice - if (target.validityBuffer != null) { - target.validityBuffer.getReferenceManager().release(); - } - target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer.getReferenceManager().retain(); - } else { - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ - target.allocateValidityBuffer(byteSizeTarget); - - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - this.validityBuffer, firstByteSource + i, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - this.validityBuffer, firstByteSource + i + 1, offset); - - target.validityBuffer.setByte(i, (b1 + b2)); - } - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ - if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - this.validityBuffer, firstByteSource + byteSizeTarget, offset); - - target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } else { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - target.validityBuffer.setByte(byteSizeTarget - 1, b1); - } - } + if (target.validityBuffer != null) { + target.validityBuffer.getReferenceManager().release(); } + target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer.getReferenceManager().retain(); } /*----------------------------------------------------------------* diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java index 7bff431e40..37dfa20616 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java @@ -48,6 +48,10 @@ public abstract class BaseValueVector implements ValueVector { protected volatile FieldReader fieldReader; + protected ArrowBuf validityBuffer; + + protected int valueCount; + protected BaseValueVector(BufferAllocator allocator) { this.allocator = Preconditions.checkNotNull(allocator, "allocator cannot be null"); } @@ -255,4 +259,116 @@ public void copyFrom(int fromIndex, int thisIndex, ValueVector from) { public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { throw new UnsupportedOperationException(); } + + /** + * Transfer the validity buffer from `validityBuffer` to the target vector's `validityBuffer`. + * Start at `startIndex` and copy `length` number of elements. If the starting index is 8 byte + * aligned, then the buffer is sliced from that index and ownership is transferred. If not, + * individual bytes are copied. + * + * @param startIndex starting index + * @param length number of elements to be copied + * @param target target vector + */ + protected void splitAndTransferValidityBuffer( + int startIndex, int length, BaseValueVector target) { + int offset = startIndex % 8; + + if (length <= 0) { + return; + } + if (offset == 0) { + sliceAndTransferValidityBuffer(startIndex, length, target); + } else { + copyValidityBuffer(startIndex, length, target); + } + } + + /** + * If the start index is 8 byte aligned, slice `validityBuffer` and transfer ownership to + * `target`'s `validityBuffer`. + * + * @param startIndex starting index + * @param length number of elements to be copied + * @param target target vector + */ + protected void sliceAndTransferValidityBuffer( + int startIndex, int length, BaseValueVector target) { + final int firstByteSource = BitVectorHelper.byteIndex(startIndex); + final int byteSizeTarget = getValidityBufferSizeFromCount(length); + + if (target.validityBuffer != null) { + target.validityBuffer.getReferenceManager().release(); + } + target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer.getReferenceManager().retain(1); + } + + /** + * Allocate new validity buffer for `target` and copy bytes from `validityBuffer`. Precise details + * in the comments below. + * + * @param startIndex starting index + * @param length number of elements to be copied + * @param target target vector + */ + protected void copyValidityBuffer(int startIndex, int length, BaseValueVector target) { + final int firstByteSource = BitVectorHelper.byteIndex(startIndex); + final int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); + final int byteSizeTarget = getValidityBufferSizeFromCount(length); + final int offset = startIndex % 8; + + /* Copy data + * When the first bit starts from the middle of a byte (offset != 0), + * copy data from src BitVector. + * Each byte in the target is composed by a part in i-th byte, + * another part in (i+1)-th byte. + */ + target.allocateValidityBuffer(byteSizeTarget); + + for (int i = 0; i < byteSizeTarget - 1; i++) { + byte b1 = + BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset); + byte b2 = + BitVectorHelper.getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset); + + target.validityBuffer.setByte(i, (b1 + b2)); + } + + /* Copying the last piece is done in the following manner: + * if the source vector has 1 or more bytes remaining, we copy + * the last piece as a byte formed by shifting data + * from the current byte and the next byte. + * + * if the source vector has no more bytes remaining + * (we are at the last byte), we copy the last piece as a byte + * by shifting data from the current byte. + */ + if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + byte b1 = + BitVectorHelper.getBitsFromCurrentByte( + this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); + byte b2 = + BitVectorHelper.getBitsFromNextByte( + this.validityBuffer, firstByteSource + byteSizeTarget, offset); + + target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); + } else { + byte b1 = + BitVectorHelper.getBitsFromCurrentByte( + this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); + target.validityBuffer.setByte(byteSizeTarget - 1, b1); + } + } + + /** + * Allocate new validity buffer for when the bytes need to be copied over. + * + * @param byteSizeTarget desired size of the buffer + */ + protected void allocateValidityBuffer(long byteSizeTarget) { + validityBuffer = allocator.buffer(byteSizeTarget); + validityBuffer.readerIndex(0); + validityBuffer.setZero(0, validityBuffer.capacity()); + } } diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java index 4f681311ed..96e2afbd29 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java @@ -50,10 +50,8 @@ public abstract class BaseVariableWidthVector extends BaseValueVector /* protected members */ public static final int OFFSET_WIDTH = 4; /* 4 byte unsigned int to track offsets */ protected static final byte[] emptyByteArray = new byte[] {}; - protected ArrowBuf validityBuffer; protected ArrowBuf valueBuffer; protected ArrowBuf offsetBuffer; - protected int valueCount; protected int lastSet; protected final Field field; @@ -87,7 +85,7 @@ public String getName() { /* TODO: * Once the entire hierarchy has been refactored, move common functions - * like getNullCount(), splitAndTransferValidityBuffer to top level + * like getNullCount() to top level * base class BaseValueVector. * * Along with this, some class members (validityBuffer) can also be @@ -519,11 +517,9 @@ private ArrowBuf allocateOffsetBuffer(final long size) { } /* allocate validity buffer */ - private void allocateValidityBuffer(final long size) { - final int curSize = (int) size; - validityBuffer = allocator.buffer(curSize); - validityBuffer.readerIndex(0); - initValidityBuffer(); + @Override + protected void allocateValidityBuffer(final long size) { + super.allocateValidityBuffer(size); } /** @@ -856,70 +852,17 @@ private void splitAndTransferOffsetBuffer( target.valueBuffer = transferBuffer(slicedBuffer, target.allocator); } - /* - * Transfer the validity. - */ - private void splitAndTransferValidityBuffer( - int startIndex, int length, BaseVariableWidthVector target) { - if (length <= 0) { - return; - } - + @Override + protected void sliceAndTransferValidityBuffer( + int startIndex, int length, BaseValueVector target) { final int firstByteSource = BitVectorHelper.byteIndex(startIndex); - final int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); final int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); - final int offset = startIndex % 8; - if (offset == 0) { - // slice - if (target.validityBuffer != null) { - target.validityBuffer.getReferenceManager().release(); - } - final ArrowBuf slicedValidityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer = transferBuffer(slicedValidityBuffer, target.allocator); - return; - } - - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ - target.allocateValidityBuffer(byteSizeTarget); - - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset); - - target.validityBuffer.setByte(i, (b1 + b2)); - } - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ - if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - this.validityBuffer, firstByteSource + byteSizeTarget, offset); - - target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } else { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - target.validityBuffer.setByte(byteSizeTarget - 1, b1); + if (target.validityBuffer != null) { + target.validityBuffer.getReferenceManager().release(); } + final ArrowBuf slicedValidityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer = transferBuffer(slicedValidityBuffer, target.allocator); } /*----------------------------------------------------------------* diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java index 5e25ffa568..ea9de8320e 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java @@ -78,13 +78,11 @@ public abstract class BaseVariableWidthViewVector extends BaseValueVector // The third 4 bytes of view are allocated for buffer index public static final int BUF_INDEX_WIDTH = 4; public static final byte[] EMPTY_BYTE_ARRAY = new byte[] {}; - protected ArrowBuf validityBuffer; // The view buffer is used to store the variable width view elements protected ArrowBuf viewBuffer; // The external buffer which stores the long strings protected List dataBuffers; protected int initialDataBufferSize; - protected int valueCount; protected int lastSet; protected final Field field; @@ -117,7 +115,7 @@ public String getName() { /* TODO: * Once the entire hierarchy has been refactored, move common functions - * like getNullCount(), splitAndTransferValidityBuffer to top level + * like getNullCount() to top level * base class BaseValueVector. * * Along with this, some class members (validityBuffer) can also be @@ -129,12 +127,6 @@ public String getName() { * the top class as of now is not a good idea. */ - /* TODO: - * Implement TransferPair functionality - * https://github.com/apache/arrow/issues/40932 - * - */ - /** * Get buffer that manages the validity (NULL or NON-NULL nature) of elements in the vector. * Consider it as a buffer for internal bit vector data structure. @@ -854,77 +846,22 @@ public void splitAndTransferTo(int startIndex, int length, BaseVariableWidthView } /* allocate validity buffer */ - private void allocateValidityBuffer(final long size) { - final int curSize = (int) size; - validityBuffer = allocator.buffer(curSize); - validityBuffer.readerIndex(0); - initValidityBuffer(); + @Override + protected void allocateValidityBuffer(final long size) { + super.allocateValidityBuffer(size); } - /* - * Transfer the validity. - */ - private void splitAndTransferValidityBuffer( - int startIndex, int length, BaseVariableWidthViewVector target) { - if (length <= 0) { - return; - } - + @Override + protected void sliceAndTransferValidityBuffer( + int startIndex, int length, BaseValueVector target) { final int firstByteSource = BitVectorHelper.byteIndex(startIndex); - final int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); final int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); - final int offset = startIndex % 8; - - if (offset == 0) { - // slice - if (target.validityBuffer != null) { - target.validityBuffer.getReferenceManager().release(); - } - final ArrowBuf slicedValidityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer = transferBuffer(slicedValidityBuffer, target.allocator); - return; - } - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ - target.allocateValidityBuffer(byteSizeTarget); - - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset); - - target.validityBuffer.setByte(i, (b1 + b2)); - } - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ - if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - this.validityBuffer, firstByteSource + byteSizeTarget, offset); - - target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } else { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - target.validityBuffer.setByte(byteSizeTarget - 1, b1); + if (target.validityBuffer != null) { + target.validityBuffer.getReferenceManager().release(); } + final ArrowBuf slicedValidityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer = transferBuffer(slicedValidityBuffer, target.allocator); } /** diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/BaseLargeRepeatedValueViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/BaseLargeRepeatedValueViewVector.java index 12edd6557b..fac3f86bba 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/BaseLargeRepeatedValueViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/BaseLargeRepeatedValueViewVector.java @@ -52,7 +52,6 @@ public abstract class BaseLargeRepeatedValueViewVector extends BaseValueVector protected ArrowBuf sizeBuffer; protected FieldVector vector; protected final CallBack repeatedCallBack; - protected int valueCount; protected long offsetAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * OFFSET_WIDTH; protected long sizeAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * SIZE_WIDTH; private final String name; diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java index fbe83bad52..ee1d65d3e3 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java @@ -54,7 +54,6 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector protected ArrowBuf offsetBuffer; protected FieldVector vector; protected final CallBack repeatedCallBack; - protected int valueCount; protected long offsetAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * OFFSET_WIDTH; private final String name; diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueViewVector.java index e6213316b5..fd7a4ff2c6 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueViewVector.java @@ -52,7 +52,6 @@ public abstract class BaseRepeatedValueViewVector extends BaseValueVector protected ArrowBuf sizeBuffer; protected FieldVector vector; protected final CallBack repeatedCallBack; - protected int valueCount; protected long offsetAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * OFFSET_WIDTH; protected long sizeAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * SIZE_WIDTH; private final String name; diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index 36d9ff40ed..e3b4ab477f 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -69,12 +69,10 @@ public static FixedSizeListVector empty(String name, int size, BufferAllocator a } private FieldVector vector; - private ArrowBuf validityBuffer; private final int listSize; private Field field; private UnionFixedSizeListReader reader; - private int valueCount; private int validityAllocationSizeInBytes; /** @@ -248,12 +246,10 @@ public boolean allocateNewSafe() { return success; } - private void allocateValidityBuffer(final long size) { - final int curSize = (int) size; - validityBuffer = allocator.buffer(curSize); - validityBuffer.readerIndex(0); - validityAllocationSizeInBytes = curSize; - validityBuffer.setZero(0, validityBuffer.capacity()); + @Override + protected void allocateValidityBuffer(final long size) { + super.allocateValidityBuffer(size); + validityAllocationSizeInBytes = (int) size; } @Override @@ -649,71 +645,6 @@ public void splitAndTransfer(int startIndex, int length) { to.setValueCount(length); } - /* - * transfer the validity. - */ - private void splitAndTransferValidityBuffer( - int startIndex, int length, FixedSizeListVector target) { - int firstByteSource = BitVectorHelper.byteIndex(startIndex); - int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); - int offset = startIndex % 8; - - if (length > 0) { - if (offset == 0) { - // slice - if (target.validityBuffer != null) { - target.validityBuffer.getReferenceManager().release(); - } - target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer.getReferenceManager().retain(1); - } else { - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ - target.allocateValidityBuffer(byteSizeTarget); - - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - validityBuffer, firstByteSource + i + 1, offset); - - target.validityBuffer.setByte(i, (b1 + b2)); - } - - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ - if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - validityBuffer, firstByteSource + byteSizeTarget, offset); - - target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } else { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - target.validityBuffer.setByte(byteSizeTarget - 1, b1); - } - } - } - } - @Override public ValueVector getTo() { return to; diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java index 71633441cb..835d3468f3 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java @@ -94,11 +94,9 @@ public static LargeListVector empty(String name, BufferAllocator allocator) { protected ArrowBuf offsetBuffer; protected FieldVector vector; protected final CallBack callBack; - protected int valueCount; protected long offsetAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * OFFSET_WIDTH; protected String defaultDataVectorName = DATA_VECTOR_NAME; - protected ArrowBuf validityBuffer; protected UnionLargeListReader reader; private Field field; private int validityAllocationSizeInBytes; @@ -375,12 +373,10 @@ public boolean allocateNewSafe() { return success; } - private void allocateValidityBuffer(final long size) { - final int curSize = (int) size; - validityBuffer = allocator.buffer(curSize); - validityBuffer.readerIndex(0); - validityAllocationSizeInBytes = curSize; - validityBuffer.setZero(0, validityBuffer.capacity()); + @Override + protected void allocateValidityBuffer(final long size) { + super.allocateValidityBuffer(size); + validityAllocationSizeInBytes = (int) size; } protected ArrowBuf allocateOffsetBuffer(final long size) { @@ -694,71 +690,6 @@ public void splitAndTransfer(int startIndex, int length) { to.setValueCount(length); } - /* - * transfer the validity. - */ - private void splitAndTransferValidityBuffer( - int startIndex, int length, LargeListVector target) { - int firstByteSource = BitVectorHelper.byteIndex(startIndex); - int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); - int offset = startIndex % 8; - - if (length > 0) { - if (offset == 0) { - // slice - if (target.validityBuffer != null) { - target.validityBuffer.getReferenceManager().release(); - } - target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer.getReferenceManager().retain(1); - } else { - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ - target.allocateValidityBuffer(byteSizeTarget); - - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - validityBuffer, firstByteSource + i + 1, offset); - - target.validityBuffer.setByte(i, (b1 + b2)); - } - - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ - if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - validityBuffer, firstByteSource + byteSizeTarget, offset); - - target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } else { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - target.validityBuffer.setByte(byteSizeTarget - 1, b1); - } - } - } - } - @Override public ValueVector getTo() { return to; diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java index 1b7e6b2280..394c3c67bb 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java @@ -77,7 +77,6 @@ public class LargeListViewVector extends BaseLargeRepeatedValueViewVector implements PromotableVector, ValueIterableVector> { - protected ArrowBuf validityBuffer; protected UnionLargeListViewReader reader; private CallBack callBack; protected Field field; @@ -285,12 +284,10 @@ public boolean allocateNewSafe() { return success; } + @Override protected void allocateValidityBuffer(final long size) { - final int curSize = (int) size; - validityBuffer = allocator.buffer(curSize); - validityBuffer.readerIndex(0); - validityAllocationSizeInBytes = curSize; - validityBuffer.setZero(0, validityBuffer.capacity()); + super.allocateValidityBuffer(size); + validityAllocationSizeInBytes = (int) size; } @Override @@ -531,71 +528,6 @@ public void splitAndTransfer(int startIndex, int length) { } } - /* - * transfer the validity. - */ - private void splitAndTransferValidityBuffer( - int startIndex, int length, LargeListViewVector target) { - int firstByteSource = BitVectorHelper.byteIndex(startIndex); - int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); - int offset = startIndex % 8; - - if (length > 0) { - if (offset == 0) { - // slice - if (target.validityBuffer != null) { - target.validityBuffer.getReferenceManager().release(); - } - target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer.getReferenceManager().retain(1); - } else { - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ - target.allocateValidityBuffer(byteSizeTarget); - - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - validityBuffer, firstByteSource + i + 1, offset); - - target.validityBuffer.setByte(i, (b1 + b2)); - } - - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ - if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - validityBuffer, firstByteSource + byteSizeTarget, offset); - - target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } else { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - target.validityBuffer.setByte(byteSizeTarget - 1, b1); - } - } - } - } - @Override public ValueVector getTo() { return to; diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index a8e8dcc436..2b2817515f 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -74,7 +74,6 @@ public static ListVector empty(String name, BufferAllocator allocator) { return new ListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null); } - protected ArrowBuf validityBuffer; protected UnionListReader reader; private CallBack callBack; protected Field field; @@ -324,12 +323,10 @@ public boolean allocateNewSafe() { return success; } + @Override protected void allocateValidityBuffer(final long size) { - final int curSize = (int) size; - validityBuffer = allocator.buffer(curSize); - validityBuffer.readerIndex(0); - validityAllocationSizeInBytes = curSize; - validityBuffer.setZero(0, validityBuffer.capacity()); + super.allocateValidityBuffer(size); + validityAllocationSizeInBytes = (int) size; } /** @@ -575,70 +572,6 @@ public void splitAndTransfer(int startIndex, int length) { } } - /* - * transfer the validity. - */ - private void splitAndTransferValidityBuffer(int startIndex, int length, ListVector target) { - int firstByteSource = BitVectorHelper.byteIndex(startIndex); - int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); - int offset = startIndex % 8; - - if (length > 0) { - if (offset == 0) { - // slice - if (target.validityBuffer != null) { - target.validityBuffer.getReferenceManager().release(); - } - target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer.getReferenceManager().retain(1); - } else { - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ - target.allocateValidityBuffer(byteSizeTarget); - - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - validityBuffer, firstByteSource + i + 1, offset); - - target.validityBuffer.setByte(i, (b1 + b2)); - } - - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ - if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - validityBuffer, firstByteSource + byteSizeTarget, offset); - - target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } else { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - target.validityBuffer.setByte(byteSizeTarget - 1, b1); - } - } - } - } - @Override public ValueVector getTo() { return to; diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java index ada25bbaf5..2b80101926 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java @@ -76,7 +76,6 @@ public class ListViewVector extends BaseRepeatedValueViewVector implements PromotableVector, ValueIterableVector> { - protected ArrowBuf validityBuffer; protected UnionListViewReader reader; private CallBack callBack; protected Field field; @@ -284,12 +283,10 @@ public boolean allocateNewSafe() { return success; } + @Override protected void allocateValidityBuffer(final long size) { - final int curSize = (int) size; - validityBuffer = allocator.buffer(curSize); - validityBuffer.readerIndex(0); - validityAllocationSizeInBytes = curSize; - validityBuffer.setZero(0, validityBuffer.capacity()); + super.allocateValidityBuffer(size); + validityAllocationSizeInBytes = (int) size; } @Override @@ -538,70 +535,6 @@ public void splitAndTransfer(int startIndex, int length) { } } - /* - * transfer the validity. - */ - private void splitAndTransferValidityBuffer(int startIndex, int length, ListViewVector target) { - int firstByteSource = BitVectorHelper.byteIndex(startIndex); - int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); - int offset = startIndex % 8; - - if (length > 0) { - if (offset == 0) { - // slice - if (target.validityBuffer != null) { - target.validityBuffer.getReferenceManager().release(); - } - target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer.getReferenceManager().retain(1); - } else { - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ - target.allocateValidityBuffer(byteSizeTarget); - - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - validityBuffer, firstByteSource + i + 1, offset); - - target.validityBuffer.setByte(i, (b1 + b2)); - } - - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ - if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - validityBuffer, firstByteSource + byteSizeTarget, offset); - - target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } else { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - target.validityBuffer.setByte(byteSizeTarget - 1, b1); - } - } - } - } - @Override public ValueVector getTo() { return to; diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java index 5eb857ab94..3f98322ba9 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java @@ -22,7 +22,6 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.AddOrGetResult; -import org.apache.arrow.vector.BitVectorHelper; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.ZeroVector; @@ -232,70 +231,6 @@ public void splitAndTransfer(int startIndex, int length) { } } - /* - * transfer the validity. - */ - private void splitAndTransferValidityBuffer(int startIndex, int length, MapVector target) { - int firstByteSource = BitVectorHelper.byteIndex(startIndex); - int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); - int offset = startIndex % 8; - - if (length > 0) { - if (offset == 0) { - // slice - if (target.validityBuffer != null) { - target.validityBuffer.getReferenceManager().release(); - } - target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer.getReferenceManager().retain(1); - } else { - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ - target.allocateValidityBuffer(byteSizeTarget); - - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - validityBuffer, firstByteSource + i + 1, offset); - - target.validityBuffer.setByte(i, (b1 + b2)); - } - - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ - if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - validityBuffer, firstByteSource + byteSizeTarget, offset); - - target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } else { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - target.validityBuffer.setByte(byteSizeTarget - 1, b1); - } - } - } - } - @Override public ValueVector getTo() { return to; From 43b6b6ccf9107353d22cd2a90632ef2608d872c5 Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Tue, 1 Jul 2025 09:16:52 +0800 Subject: [PATCH 184/408] GH-759: Get length of byte[] in TryCopyLastError (#760) ## What's Changed We should get the length of byte[] by `GetArrayLength`, not `strlen` which may cause invalid memory access. Closes #759. --- c/src/main/cpp/jni_wrapper.cc | 3 +- .../org/apache/arrow/c/ExceptionTest.java | 150 ++++++++++++++++++ 2 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 c/src/test/java/org/apache/arrow/c/ExceptionTest.java diff --git a/c/src/main/cpp/jni_wrapper.cc b/c/src/main/cpp/jni_wrapper.cc index 35c2b7787e..436cbdc806 100644 --- a/c/src/main/cpp/jni_wrapper.cc +++ b/c/src/main/cpp/jni_wrapper.cc @@ -205,8 +205,9 @@ void TryCopyLastError(JNIEnv* env, InnerPrivateData* private_data) { return; } + jsize error_bytes_len = env->GetArrayLength(arr); char* error_str = reinterpret_cast(error_bytes); - private_data->last_error_ = std::string(error_str, std::strlen(error_str)); + private_data->last_error_ = std::string(error_str, error_bytes_len); env->ReleaseByteArrayElements(arr, error_bytes, JNI_ABORT); } diff --git a/c/src/test/java/org/apache/arrow/c/ExceptionTest.java b/c/src/test/java/org/apache/arrow/c/ExceptionTest.java new file mode 100644 index 0000000000..5bc96a8f99 --- /dev/null +++ b/c/src/test/java/org/apache/arrow/c/ExceptionTest.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.c; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowableOfType; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.VectorLoader; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.dictionary.Dictionary; +import org.apache.arrow.vector.dictionary.DictionaryProvider; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.Test; + +// Regression test for https://github.com/apache/arrow-java/issues/759 +final class ExceptionTest { + @Test + public void testException() throws IOException { + final Schema schema = + new Schema(Collections.singletonList(Field.nullable("ints", new ArrowType.Int(32, true)))); + final List batches = new ArrayList<>(); + + try (BufferAllocator allocator = new RootAllocator(); + VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { + + final String exceptionMessage = "This is a message for testing exception."; + + RuntimeException exToThrow = new RuntimeException(exceptionMessage); + batches.add(exToThrow); + + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + exToThrow.printStackTrace(pw); + final String expectExceptionMessage = sw.toString(); + + ArrowReader source = new ExceptionMemoryArrowReader(allocator, schema, batches); + + try (final ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator); + final VectorSchemaRoot importRoot = VectorSchemaRoot.create(schema, allocator)) { + final VectorLoader loader = new VectorLoader(importRoot); + Data.exportArrayStream(allocator, source, stream); + + try (final ArrowReader reader = Data.importArrayStream(allocator, stream)) { + IOException jniException = catchThrowableOfType(IOException.class, reader::loadNextBatch); + final String jniMessage = jniException.getMessage(); + assertThat(jniMessage.endsWith(expectExceptionMessage + "}")); + } + } + } + } + + static class ExceptionMemoryArrowReader extends ArrowReader { + private final Schema schema; + private final List batches; // set ArrowRecordBatch or Exception + private final DictionaryProvider provider; + private int nextBatch; + + ExceptionMemoryArrowReader(BufferAllocator allocator, Schema schema, List batches) { + super(allocator); + this.schema = schema; + this.batches = batches; + this.provider = new CDataDictionaryProvider(); + this.nextBatch = 0; + } + + @Override + public Dictionary lookup(long id) { + return provider.lookup(id); + } + + @Override + public Set getDictionaryIds() { + return provider.getDictionaryIds(); + } + + @Override + public Map getDictionaryVectors() { + return getDictionaryIds().stream() + .collect(Collectors.toMap(Function.identity(), this::lookup)); + } + + @Override + public boolean loadNextBatch() throws IOException { + if (nextBatch < batches.size()) { + Object object = batches.get(nextBatch++); + if (object instanceof RuntimeException) { + throw (RuntimeException) object; + } + VectorLoader loader = new VectorLoader(getVectorSchemaRoot()); + loader.load((ArrowRecordBatch) object); + return true; + } + return false; + } + + @Override + public long bytesRead() { + return 0; + } + + @Override + protected void closeReadSource() throws IOException { + try { + for (Object object : batches) { + if (object instanceof ArrowRecordBatch) { + ArrowRecordBatch batch = (ArrowRecordBatch) object; + batch.close(); + } + } + } catch (Exception e) { + throw new IOException(e); + } + } + + @Override + protected Schema readSchema() { + return schema; + } + } +} From 45295a589d64da5f9838cfad7cd29ec4db481047 Mon Sep 17 00:00:00 2001 From: Martin Traverse Date: Tue, 8 Jul 2025 01:57:20 +0100 Subject: [PATCH 185/408] GH-731: Avro adapter, output dictionary-encoded fields as enums (#779) ## What's Changed Updated ArrowToAvro to output dictionary-encoded string vectors as Avro enums, where possible. Apologies for the delay - busy as usual! To output dict encoded vectors as enums, a dictionary provider must be supplied to the top level methods with all the required dictionaries. All dictionary values must be present when the schema is written, i.e. before the data blocks are produced. If data is being written as a schema followed by multiple blocks, values added to a dictionary in between blocks will not be included in the schema resulting in an invalid Avro file (in general supply an invalid dictionary mapping will result in invalid output). Dictionary encoded fields are checked to ensure they are valid Avro enums. If the dictionary encoded field is not a string field, or the string values are not valid Avro enums, the field is decoded and output as literal values. This is done by calling DictionaryEncoder.decode(vector, dictionary), which will consume memory for the vector. An alternative approach would be to decode values one-by-one, however this would require a significant change to the producer pattern since the current producers expect concrete vectors of the output type. Another option would be to throw an error if there are dictionary-encoded vectors that are not string types, i.e. push the responsibility onto client code. I'm not sure which approach is best - happy to take any guidance and I will update the code accordingly. To read enums back the current approach for decoding is unchanged (the AvroToArrow config has to be set up with a MapDictionaryProvider which is populated when data is read). The last part of the Avro work is to add the capability for reading / writing whole files block-by-block, so there is an opportunity to do something with the top level APIs there, for now the current API works and I've used it in the round trip tests. Please let me know any feedback, happy to update as needed! Closes #731. --- .../arrow/adapter/avro/ArrowToAvroUtils.java | 226 +++++++++++++++--- .../avro/producers/AvroEnumProducer.java | 12 +- .../producers/DictionaryDecodingProducer.java | 47 ++++ .../adapter/avro/ArrowToAvroDataTest.java | 81 +++++++ .../adapter/avro/ArrowToAvroSchemaTest.java | 129 ++++++++++ .../arrow/adapter/avro/RoundTripDataTest.java | 110 ++++++++- .../adapter/avro/RoundTripSchemaTest.java | 63 ++++- 7 files changed, 618 insertions(+), 50 deletions(-) create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/DictionaryDecodingProducer.java diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/ArrowToAvroUtils.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/ArrowToAvroUtils.java index 87b594af9e..e09b99f670 100644 --- a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/ArrowToAvroUtils.java +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/ArrowToAvroUtils.java @@ -17,10 +17,14 @@ package org.apache.arrow.adapter.avro; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; import org.apache.arrow.adapter.avro.producers.AvroBigIntProducer; import org.apache.arrow.adapter.avro.producers.AvroBooleanProducer; import org.apache.arrow.adapter.avro.producers.AvroBytesProducer; +import org.apache.arrow.adapter.avro.producers.AvroEnumProducer; import org.apache.arrow.adapter.avro.producers.AvroFixedSizeBinaryProducer; import org.apache.arrow.adapter.avro.producers.AvroFixedSizeListProducer; import org.apache.arrow.adapter.avro.producers.AvroFloat2Producer; @@ -41,6 +45,7 @@ import org.apache.arrow.adapter.avro.producers.AvroUint8Producer; import org.apache.arrow.adapter.avro.producers.BaseAvroProducer; import org.apache.arrow.adapter.avro.producers.CompositeAvroProducer; +import org.apache.arrow.adapter.avro.producers.DictionaryDecodingProducer; import org.apache.arrow.adapter.avro.producers.Producer; import org.apache.arrow.adapter.avro.producers.logical.AvroDateDayProducer; import org.apache.arrow.adapter.avro.producers.logical.AvroDateMilliProducer; @@ -59,6 +64,7 @@ import org.apache.arrow.adapter.avro.producers.logical.AvroTimestampSecProducer; import org.apache.arrow.adapter.avro.producers.logical.AvroTimestampSecTzProducer; import org.apache.arrow.util.Preconditions; +import org.apache.arrow.vector.BaseIntVector; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.DateDayVector; @@ -96,11 +102,14 @@ import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.StructVector; +import org.apache.arrow.vector.dictionary.Dictionary; +import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.util.Text; import org.apache.avro.LogicalType; import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; @@ -162,17 +171,29 @@ public class ArrowToAvroUtils { * may be nullable. Record types must contain at least one child field and cannot contain multiple * fields with the same name * + *

String fields that are dictionary-encoded will be represented as an Avro enum, so long as + * all the values meet the restrictions on Avro enums (non-null, valid identifiers). Other data + * types that are dictionary encoded, or string fields that do not meet the avro requirements, + * will be output as their decoded type. + * * @param arrowFields The arrow fields used to generate the Avro schema * @param typeName Name of the top level Avro record type * @param namespace Namespace of the top level Avro record type + * @param dictionaries A dictionary provider is required if any fields use dictionary encoding * @return An Avro record schema for the given list of fields, with the specified name and * namespace */ public static Schema createAvroSchema( - List arrowFields, String typeName, String namespace) { + List arrowFields, String typeName, String namespace, DictionaryProvider dictionaries) { SchemaBuilder.RecordBuilder assembler = SchemaBuilder.record(typeName).namespace(namespace); - return buildRecordSchema(assembler, arrowFields, namespace); + return buildRecordSchema(assembler, arrowFields, namespace, dictionaries); + } + + /** Overload provided for convenience, sets dictionaries = null. */ + public static Schema createAvroSchema( + List arrowFields, String typeName, String namespace) { + return createAvroSchema(arrowFields, typeName, namespace, null); } /** Overload provided for convenience, sets namespace = null. */ @@ -185,61 +206,83 @@ public static Schema createAvroSchema(List arrowFields) { return createAvroSchema(arrowFields, GENERIC_RECORD_TYPE_NAME); } + /** + * Overload provided for convenience, sets name = GENERIC_RECORD_TYPE_NAME and namespace = null. + */ + public static Schema createAvroSchema(List arrowFields, DictionaryProvider dictionaries) { + return createAvroSchema(arrowFields, GENERIC_RECORD_TYPE_NAME, null, dictionaries); + } + private static T buildRecordSchema( - SchemaBuilder.RecordBuilder builder, List fields, String namespace) { + SchemaBuilder.RecordBuilder builder, + List fields, + String namespace, + DictionaryProvider dictionaries) { if (fields.isEmpty()) { throw new IllegalArgumentException("Record field must have at least one child field"); } SchemaBuilder.FieldAssembler assembler = builder.namespace(namespace).fields(); for (Field field : fields) { - assembler = buildFieldSchema(assembler, field, namespace); + assembler = buildFieldSchema(assembler, field, namespace, dictionaries); } return assembler.endRecord(); } private static SchemaBuilder.FieldAssembler buildFieldSchema( - SchemaBuilder.FieldAssembler assembler, Field field, String namespace) { + SchemaBuilder.FieldAssembler assembler, + Field field, + String namespace, + DictionaryProvider dictionaries) { return assembler .name(field.getName()) - .type(buildTypeSchema(SchemaBuilder.builder(), field, namespace)) + .type(buildTypeSchema(SchemaBuilder.builder(), field, namespace, dictionaries)) .noDefault(); } private static T buildTypeSchema( - SchemaBuilder.TypeBuilder builder, Field field, String namespace) { + SchemaBuilder.TypeBuilder builder, + Field field, + String namespace, + DictionaryProvider dictionaries) { // Nullable unions need special handling, since union types cannot be directly nested if (field.getType().getTypeID() == ArrowType.ArrowTypeID.Union) { boolean unionNullable = field.getChildren().stream().anyMatch(Field::isNullable); if (unionNullable) { SchemaBuilder.UnionAccumulator union = builder.unionOf().nullType(); - return addTypesToUnion(union, field.getChildren(), namespace); + return addTypesToUnion(union, field.getChildren(), namespace, dictionaries); } else { Field headType = field.getChildren().get(0); List tailTypes = field.getChildren().subList(1, field.getChildren().size()); SchemaBuilder.UnionAccumulator union = - buildBaseTypeSchema(builder.unionOf(), headType, namespace); - return addTypesToUnion(union, tailTypes, namespace); + buildBaseTypeSchema(builder.unionOf(), headType, namespace, dictionaries); + return addTypesToUnion(union, tailTypes, namespace, dictionaries); } } else if (field.isNullable()) { - return buildBaseTypeSchema(builder.nullable(), field, namespace); + return buildBaseTypeSchema(builder.nullable(), field, namespace, dictionaries); } else { - return buildBaseTypeSchema(builder, field, namespace); + return buildBaseTypeSchema(builder, field, namespace, dictionaries); } } private static T buildArraySchema( - SchemaBuilder.ArrayBuilder builder, Field listField, String namespace) { + SchemaBuilder.ArrayBuilder builder, + Field listField, + String namespace, + DictionaryProvider dictionaries) { if (listField.getChildren().size() != 1) { throw new IllegalArgumentException("List field must have exactly one child field"); } Field itemField = listField.getChildren().get(0); - return buildTypeSchema(builder.items(), itemField, namespace); + return buildTypeSchema(builder.items(), itemField, namespace, dictionaries); } private static T buildMapSchema( - SchemaBuilder.MapBuilder builder, Field mapField, String namespace) { + SchemaBuilder.MapBuilder builder, + Field mapField, + String namespace, + DictionaryProvider dictionaries) { if (mapField.getChildren().size() != 1) { throw new IllegalArgumentException("Map field must have exactly one child field"); } @@ -253,11 +296,14 @@ private static T buildMapSchema( throw new IllegalArgumentException( "Map keys must be of type string and cannot be nullable for conversion to Avro"); } - return buildTypeSchema(builder.values(), valueField, namespace); + return buildTypeSchema(builder.values(), valueField, namespace, dictionaries); } private static T buildBaseTypeSchema( - SchemaBuilder.BaseTypeBuilder builder, Field field, String namespace) { + SchemaBuilder.BaseTypeBuilder builder, + Field field, + String namespace, + DictionaryProvider dictionaries) { ArrowType.ArrowTypeID typeID = field.getType().getTypeID(); @@ -269,6 +315,33 @@ private static T buildBaseTypeSchema( return builder.booleanType(); case Int: + if (field.getDictionary() != null) { + if (dictionaries == null) { + throw new IllegalArgumentException( + "Field references a dictionary but no dictionaries were provided: " + + field.getName()); + } + Dictionary dictionary = dictionaries.lookup(field.getDictionary().getId()); + if (dictionary == null) { + throw new IllegalArgumentException( + "Field references a dictionary that does not exist: " + + field.getName() + + ", dictionary ID = " + + field.getDictionary().getId()); + } + if (dictionaryIsValidEnum(dictionary)) { + String[] symbols = dictionarySymbols(dictionary); + return builder.enumeration(field.getName()).symbols(symbols); + } else { + Field decodedField = + new Field( + field.getName(), + dictionary.getVector().getField().getFieldType(), + dictionary.getVector().getField().getChildren()); + return buildBaseTypeSchema(builder, decodedField, namespace, dictionaries); + } + } + ArrowType.Int intType = (ArrowType.Int) field.getType(); if (intType.getBitWidth() > 32 || (intType.getBitWidth() == 32 && !intType.getIsSigned())) { return builder.longType(); @@ -328,7 +401,7 @@ private static T buildBaseTypeSchema( String childNamespace = namespace == null ? field.getName() : namespace + "." + field.getName(); return buildRecordSchema( - builder.record(field.getName()), field.getChildren(), childNamespace); + builder.record(field.getName()), field.getChildren(), childNamespace, dictionaries); case List: case FixedSizeList: @@ -339,13 +412,13 @@ private static T buildBaseTypeSchema( new Field("item", itemField.getFieldType(), itemField.getChildren()); Field safeListField = new Field(field.getName(), field.getFieldType(), List.of(safeItemField)); - return buildArraySchema(builder.array(), safeListField, namespace); + return buildArraySchema(builder.array(), safeListField, namespace, dictionaries); } else { - return buildArraySchema(builder.array(), field, namespace); + return buildArraySchema(builder.array(), field, namespace, dictionaries); } case Map: - return buildMapSchema(builder.map(), field, namespace); + return buildMapSchema(builder.map(), field, namespace, dictionaries); default: throw new IllegalArgumentException( @@ -354,9 +427,12 @@ private static T buildBaseTypeSchema( } private static T addTypesToUnion( - SchemaBuilder.UnionAccumulator accumulator, List unionFields, String namespace) { + SchemaBuilder.UnionAccumulator accumulator, + List unionFields, + String namespace, + DictionaryProvider dictionaries) { for (var field : unionFields) { - accumulator = buildBaseTypeSchema(accumulator.and(), field, namespace); + accumulator = buildBaseTypeSchema(accumulator.and(), field, namespace, dictionaries); } return accumulator.endUnion(); } @@ -373,30 +449,88 @@ private static LogicalType timestampLogicalType(ArrowType.Timestamp timestampTyp } } + private static boolean dictionaryIsValidEnum(Dictionary dictionary) { + + if (dictionary.getVectorType().getTypeID() != ArrowType.ArrowTypeID.Utf8) { + return false; + } + + VarCharVector vector = (VarCharVector) dictionary.getVector(); + Set symbols = new HashSet<>(); + + for (int i = 0; i < vector.getValueCount(); i++) { + if (vector.isNull(i)) { + return false; + } + Text text = vector.getObject(i); + if (text == null) { + return false; + } + String symbol = text.toString(); + if (!ENUM_REGEX.matcher(symbol).matches()) { + return false; + } + if (symbols.contains(symbol)) { + return false; + } + symbols.add(symbol); + } + + return true; + } + + private static String[] dictionarySymbols(Dictionary dictionary) { + + VarCharVector vector = (VarCharVector) dictionary.getVector(); + String[] symbols = new String[vector.getValueCount()]; + + for (int i = 0; i < vector.getValueCount(); i++) { + Text text = vector.getObject(i); + // This should never happen if dictionaryIsValidEnum() succeeded + if (text == null) { + throw new IllegalArgumentException("Illegal null value in enum"); + } + symbols[i] = text.toString(); + } + + return symbols; + } + + private static final Pattern ENUM_REGEX = Pattern.compile("^[A-Za-z_][A-Za-z0-9_]*$"); + /** * Create a composite Avro producer for a set of field vectors (typically the root set of a VSR). * * @param vectors The vectors that will be used to produce Avro data * @return The resulting composite Avro producer */ - public static CompositeAvroProducer createCompositeProducer(List vectors) { + public static CompositeAvroProducer createCompositeProducer( + List vectors, DictionaryProvider dictionaries) { List> producers = new ArrayList<>(vectors.size()); for (FieldVector vector : vectors) { - BaseAvroProducer producer = createProducer(vector); + BaseAvroProducer producer = createProducer(vector, dictionaries); producers.add(producer); } return new CompositeAvroProducer(producers); } - private static BaseAvroProducer createProducer(FieldVector vector) { + /** Overload provided for convenience, sets dictionaries = null. */ + public static CompositeAvroProducer createCompositeProducer(List vectors) { + + return createCompositeProducer(vectors, null); + } + + private static BaseAvroProducer createProducer( + FieldVector vector, DictionaryProvider dictionaries) { boolean nullable = vector.getField().isNullable(); - return createProducer(vector, nullable); + return createProducer(vector, nullable, dictionaries); } - private static BaseAvroProducer createProducer(FieldVector vector, boolean nullable) { + private static BaseAvroProducer createProducer( + FieldVector vector, boolean nullable, DictionaryProvider dictionaries) { Preconditions.checkNotNull(vector, "Arrow vector object can't be null"); @@ -405,10 +539,34 @@ private static BaseAvroProducer createProducer(FieldVector vector, boolean nu // Avro understands nullable types as a union of type | null // Most nullable fields in a VSR will not be unions, so provide a special wrapper if (nullable && minorType != Types.MinorType.UNION) { - final BaseAvroProducer innerProducer = createProducer(vector, false); + final BaseAvroProducer innerProducer = createProducer(vector, false, dictionaries); return new AvroNullableProducer<>(innerProducer); } + if (vector.getField().getDictionary() != null) { + if (dictionaries == null) { + throw new IllegalArgumentException( + "Field references a dictionary but no dictionaries were provided: " + + vector.getField().getName()); + } + Dictionary dictionary = dictionaries.lookup(vector.getField().getDictionary().getId()); + if (dictionary == null) { + throw new IllegalArgumentException( + "Field references a dictionary that does not exist: " + + vector.getField().getName() + + ", dictionary ID = " + + vector.getField().getDictionary().getId()); + } + // If a field is dictionary-encoded but cannot be represented as an Avro enum, + // then decode it before writing + if (dictionaryIsValidEnum(dictionary)) { + return new AvroEnumProducer((BaseIntVector) vector); + } else { + BaseAvroProducer dictProducer = createProducer(dictionary.getVector(), false, null); + return new DictionaryDecodingProducer<>((BaseIntVector) vector, dictProducer); + } + } + switch (minorType) { case NULL: return new AvroNullProducer((NullVector) vector); @@ -486,21 +644,23 @@ private static BaseAvroProducer createProducer(FieldVector vector, boolean nu Producer[] childProducers = new Producer[childVectors.size()]; for (int i = 0; i < childVectors.size(); i++) { FieldVector childVector = childVectors.get(i); - childProducers[i] = createProducer(childVector, childVector.getField().isNullable()); + childProducers[i] = + createProducer(childVector, childVector.getField().isNullable(), dictionaries); } return new AvroStructProducer(structVector, childProducers); case LIST: ListVector listVector = (ListVector) vector; FieldVector itemVector = listVector.getDataVector(); - Producer itemProducer = createProducer(itemVector, itemVector.getField().isNullable()); + Producer itemProducer = + createProducer(itemVector, itemVector.getField().isNullable(), dictionaries); return new AvroListProducer(listVector, itemProducer); case FIXED_SIZE_LIST: FixedSizeListVector fixedListVector = (FixedSizeListVector) vector; FieldVector fixedItemVector = fixedListVector.getDataVector(); Producer fixedItemProducer = - createProducer(fixedItemVector, fixedItemVector.getField().isNullable()); + createProducer(fixedItemVector, fixedItemVector.getField().isNullable(), dictionaries); return new AvroFixedSizeListProducer(fixedListVector, fixedItemProducer); case MAP: @@ -514,7 +674,7 @@ private static BaseAvroProducer createProducer(FieldVector vector, boolean nu FieldVector valueVector = entryVector.getChildrenFromFields().get(1); Producer keyProducer = new AvroStringProducer(keyVector); Producer valueProducer = - createProducer(valueVector, valueVector.getField().isNullable()); + createProducer(valueVector, valueVector.getField().isNullable(), dictionaries); Producer entryProducer = new AvroStructProducer(entryVector, new Producer[] {keyProducer, valueProducer}); return new AvroMapProducer(mapVector, entryProducer); diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroEnumProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroEnumProducer.java index 068566493e..eebfb7d241 100644 --- a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroEnumProducer.java +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroEnumProducer.java @@ -17,22 +17,22 @@ package org.apache.arrow.adapter.avro.producers; import java.io.IOException; -import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.BaseIntVector; import org.apache.avro.io.Encoder; /** - * Producer that produces enum values from a dictionary-encoded {@link IntVector}, writes data to an - * Avro encoder. + * Producer that produces enum values from a dictionary-encoded {@link BaseIntVector}, writes data + * to an Avro encoder. */ -public class AvroEnumProducer extends BaseAvroProducer { +public class AvroEnumProducer extends BaseAvroProducer { /** Instantiate an AvroEnumProducer. */ - public AvroEnumProducer(IntVector vector) { + public AvroEnumProducer(BaseIntVector vector) { super(vector); } @Override public void produce(Encoder encoder) throws IOException { - encoder.writeEnum(vector.get(currentIndex++)); + encoder.writeEnum((int) vector.getValueAsLong(currentIndex++)); } } diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/DictionaryDecodingProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/DictionaryDecodingProducer.java new file mode 100644 index 0000000000..afeba08511 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/DictionaryDecodingProducer.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers; + +import java.io.IOException; +import org.apache.arrow.vector.BaseIntVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.avro.io.Encoder; + +/** + * Producer that decodes values from a dictionary-encoded {@link FieldVector}, writes the resulting + * values to an Avro encoder. + * + * @param Type of the underlying dictionary vector + */ +public class DictionaryDecodingProducer + extends BaseAvroProducer { + + private final Producer dictProducer; + + /** Instantiate a DictionaryDecodingProducer. */ + public DictionaryDecodingProducer(BaseIntVector indexVector, Producer dictProducer) { + super(indexVector); + this.dictProducer = dictProducer; + } + + @Override + public void produce(Encoder encoder) throws IOException { + int dicIndex = (int) vector.getValueAsLong(currentIndex++); + dictProducer.setPosition(dicIndex); + dictProducer.produce(encoder); + } +} diff --git a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroDataTest.java b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroDataTest.java index 2d70b45021..6d66ee9d45 100644 --- a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroDataTest.java +++ b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroDataTest.java @@ -76,10 +76,14 @@ import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.complex.writer.BaseWriter; import org.apache.arrow.vector.complex.writer.FieldWriter; +import org.apache.arrow.vector.dictionary.Dictionary; +import org.apache.arrow.vector.dictionary.DictionaryEncoder; +import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.types.DateUnit; import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.DictionaryEncoding; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.JsonStringArrayList; @@ -2817,4 +2821,81 @@ record = datumReader.read(record, decoder); } } } + + @Test + public void testWriteDictEnumEncoded() throws Exception { + + BufferAllocator allocator = new RootAllocator(); + + // Create a dictionary + FieldType dictionaryField = new FieldType(false, new ArrowType.Utf8(), null); + VarCharVector dictionaryVector = + new VarCharVector(new Field("dictionary", dictionaryField, null), allocator); + + dictionaryVector.allocateNew(3); + dictionaryVector.set(0, "apple".getBytes()); + dictionaryVector.set(1, "banana".getBytes()); + dictionaryVector.set(2, "cherry".getBytes()); + dictionaryVector.setValueCount(3); + + Dictionary dictionary = + new Dictionary(dictionaryVector, new DictionaryEncoding(1L, false, null)); + DictionaryProvider dictionaries = new DictionaryProvider.MapDictionaryProvider(dictionary); + + // Field definition + FieldType stringField = new FieldType(false, new ArrowType.Utf8(), null); + VarCharVector stringVector = + new VarCharVector(new Field("enumField", stringField, null), allocator); + stringVector.allocateNew(10); + stringVector.setSafe(0, "apple".getBytes()); + stringVector.setSafe(1, "banana".getBytes()); + stringVector.setSafe(2, "cherry".getBytes()); + stringVector.setSafe(3, "cherry".getBytes()); + stringVector.setSafe(4, "apple".getBytes()); + stringVector.setSafe(5, "banana".getBytes()); + stringVector.setSafe(6, "apple".getBytes()); + stringVector.setSafe(7, "cherry".getBytes()); + stringVector.setSafe(8, "banana".getBytes()); + stringVector.setSafe(9, "apple".getBytes()); + stringVector.setValueCount(10); + + IntVector encodedVector = (IntVector) DictionaryEncoder.encode(stringVector, dictionary); + + // Set up VSR + List vectors = Arrays.asList(encodedVector); + int rowCount = 10; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + File dataFile = new File(TMP, "testWriteEnumEncoded.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = + ArrowToAvroUtils.createCompositeProducer(vectors, dictionaries); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields(), dictionaries); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + GenericRecord record = null; + + // Read and check values + for (int row = 0; row < rowCount; row++) { + record = datumReader.read(record, decoder); + // Values read from Avro should be the decoded enum values + assertEquals(stringVector.getObject(row).toString(), record.get("enumField").toString()); + } + } + } + } } diff --git a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroSchemaTest.java b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroSchemaTest.java index d3e12e763a..d5e0357a8c 100644 --- a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroSchemaTest.java +++ b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroSchemaTest.java @@ -20,11 +20,18 @@ import java.util.Arrays; import java.util.List; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.dictionary.Dictionary; +import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.types.DateUnit; import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.UnionMode; import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.DictionaryEncoding; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.avro.LogicalTypes; @@ -1389,4 +1396,126 @@ public void testConvertUnionTypes() { Schema.Type.STRING, schema.getField("nullableDenseUnionField").schema().getTypes().get(3).getType()); } + + @Test + public void testWriteDictEnumEncoded() { + + BufferAllocator allocator = new RootAllocator(); + + // Create a dictionary + FieldType dictionaryField = new FieldType(false, new ArrowType.Utf8(), null); + VarCharVector dictionaryVector = + new VarCharVector(new Field("dictionary", dictionaryField, null), allocator); + + dictionaryVector.allocateNew(3); + dictionaryVector.set(0, "apple".getBytes()); + dictionaryVector.set(1, "banana".getBytes()); + dictionaryVector.set(2, "cherry".getBytes()); + dictionaryVector.setValueCount(3); + + Dictionary dictionary = + new Dictionary( + dictionaryVector, new DictionaryEncoding(0L, false, new ArrowType.Int(8, true))); + DictionaryProvider dictionaries = new DictionaryProvider.MapDictionaryProvider(dictionary); + + List fields = + Arrays.asList( + new Field( + "enumField", + new FieldType(false, new ArrowType.Int(8, true), dictionary.getEncoding(), null), + null)); + + Schema schema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord", null, dictionaries); + + assertEquals(Schema.Type.RECORD, schema.getType()); + assertEquals(1, schema.getFields().size()); + + Schema.Field enumField = schema.getField("enumField"); + + assertEquals(Schema.Type.ENUM, enumField.schema().getType()); + assertEquals(3, enumField.schema().getEnumSymbols().size()); + assertEquals("apple", enumField.schema().getEnumSymbols().get(0)); + assertEquals("banana", enumField.schema().getEnumSymbols().get(1)); + assertEquals("cherry", enumField.schema().getEnumSymbols().get(2)); + } + + @Test + public void testWriteDictEnumInvalid() { + + BufferAllocator allocator = new RootAllocator(); + + // Create a dictionary + FieldType dictionaryField = new FieldType(false, new ArrowType.Utf8(), null); + VarCharVector dictionaryVector = + new VarCharVector(new Field("dictionary", dictionaryField, null), allocator); + + dictionaryVector.allocateNew(3); + dictionaryVector.set(0, "passion fruit".getBytes()); + dictionaryVector.set(1, "banana".getBytes()); + dictionaryVector.set(2, "cherry".getBytes()); + dictionaryVector.setValueCount(3); + + Dictionary dictionary = + new Dictionary( + dictionaryVector, new DictionaryEncoding(0L, false, new ArrowType.Int(8, true))); + DictionaryProvider dictionaries = new DictionaryProvider.MapDictionaryProvider(dictionary); + + List fields = + Arrays.asList( + new Field( + "enumField", + new FieldType(false, new ArrowType.Int(8, true), dictionary.getEncoding(), null), + null)); + + // Dictionary field contains values that are not valid enums + // Should be decoded and output as a string field + + Schema schema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord", null, dictionaries); + + assertEquals(Schema.Type.RECORD, schema.getType()); + assertEquals(1, schema.getFields().size()); + + Schema.Field enumField = schema.getField("enumField"); + assertEquals(Schema.Type.STRING, enumField.schema().getType()); + } + + @Test + public void testWriteDictEnumInvalid2() { + + BufferAllocator allocator = new RootAllocator(); + + // Create a dictionary + FieldType dictionaryField = new FieldType(false, new ArrowType.Int(64, true), null); + BigIntVector dictionaryVector = + new BigIntVector(new Field("dictionary", dictionaryField, null), allocator); + + dictionaryVector.allocateNew(3); + dictionaryVector.set(0, 123L); + dictionaryVector.set(1, 456L); + dictionaryVector.set(2, 789L); + dictionaryVector.setValueCount(3); + + Dictionary dictionary = + new Dictionary( + dictionaryVector, new DictionaryEncoding(0L, false, new ArrowType.Int(8, true))); + DictionaryProvider dictionaries = new DictionaryProvider.MapDictionaryProvider(dictionary); + + List fields = + Arrays.asList( + new Field( + "enumField", + new FieldType(false, new ArrowType.Int(8, true), dictionary.getEncoding(), null), + null)); + + // Dictionary field encodes LONG values rather than STRING + // Should be doecded and output as a LONG field + + Schema schema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord", null, dictionaries); + + assertEquals(Schema.Type.RECORD, schema.getType()); + assertEquals(1, schema.getFields().size()); + + Schema.Field enumField = schema.getField("enumField"); + assertEquals(Schema.Type.LONG, enumField.schema().getType()); + } } diff --git a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripDataTest.java b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripDataTest.java index 85e6a960b0..ceaf59aa72 100644 --- a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripDataTest.java +++ b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripDataTest.java @@ -52,6 +52,7 @@ import org.apache.arrow.vector.TimeStampMilliVector; import org.apache.arrow.vector.TimeStampNanoTZVector; import org.apache.arrow.vector.TimeStampNanoVector; +import org.apache.arrow.vector.TinyIntVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; @@ -60,10 +61,14 @@ import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.complex.writer.BaseWriter; import org.apache.arrow.vector.complex.writer.FieldWriter; +import org.apache.arrow.vector.dictionary.Dictionary; +import org.apache.arrow.vector.dictionary.DictionaryEncoder; +import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.types.DateUnit; import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.DictionaryEncoding; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.avro.Schema; @@ -78,16 +83,21 @@ public class RoundTripDataTest { @TempDir public static File TMP; - private static AvroToArrowConfig basicConfig(BufferAllocator allocator) { - return new AvroToArrowConfig(allocator, 1000, null, Collections.emptySet(), false); + private static AvroToArrowConfig basicConfig( + BufferAllocator allocator, DictionaryProvider.MapDictionaryProvider dictionaries) { + return new AvroToArrowConfig(allocator, 1000, dictionaries, Collections.emptySet(), false); } private static VectorSchemaRoot readDataFile( - Schema schema, File dataFile, BufferAllocator allocator) throws Exception { + Schema schema, + File dataFile, + BufferAllocator allocator, + DictionaryProvider.MapDictionaryProvider dictionaries) + throws Exception { try (FileInputStream fis = new FileInputStream(dataFile)) { BinaryDecoder decoder = new DecoderFactory().directBinaryDecoder(fis, null); - return AvroToArrow.avroToArrow(schema, decoder, basicConfig(allocator)); + return AvroToArrow.avroToArrow(schema, decoder, basicConfig(allocator, dictionaries)); } } @@ -95,11 +105,22 @@ private static void roundTripTest( VectorSchemaRoot root, BufferAllocator allocator, File dataFile, int rowCount) throws Exception { + roundTripTest(root, allocator, dataFile, rowCount, null); + } + + private static void roundTripTest( + VectorSchemaRoot root, + BufferAllocator allocator, + File dataFile, + int rowCount, + DictionaryProvider dictionaries) + throws Exception { + // Write an AVRO block using the producer classes try (FileOutputStream fos = new FileOutputStream(dataFile)) { BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); CompositeAvroProducer producer = - ArrowToAvroUtils.createCompositeProducer(root.getFieldVectors()); + ArrowToAvroUtils.createCompositeProducer(root.getFieldVectors(), dictionaries); for (int row = 0; row < rowCount; row++) { producer.produce(encoder); } @@ -107,10 +128,14 @@ private static void roundTripTest( } // Generate AVRO schema - Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields(), dictionaries); + + DictionaryProvider.MapDictionaryProvider roundTripDictionaries = + new DictionaryProvider.MapDictionaryProvider(); // Read back in and compare - try (VectorSchemaRoot roundTrip = readDataFile(schema, dataFile, allocator)) { + try (VectorSchemaRoot roundTrip = + readDataFile(schema, dataFile, allocator, roundTripDictionaries)) { assertEquals(root.getSchema(), roundTrip.getSchema()); assertEquals(rowCount, roundTrip.getRowCount()); @@ -119,6 +144,21 @@ private static void roundTripTest( for (int row = 0; row < rowCount; row++) { assertEquals(root.getVector(0).getObject(row), roundTrip.getVector(0).getObject(row)); } + + if (dictionaries != null) { + for (long id : dictionaries.getDictionaryIds()) { + Dictionary originalDictionary = dictionaries.lookup(id); + Dictionary roundTripDictionary = roundTripDictionaries.lookup(id); + assertEquals( + originalDictionary.getVector().getValueCount(), + roundTripDictionary.getVector().getValueCount()); + for (int j = 0; j < originalDictionary.getVector().getValueCount(); j++) { + assertEquals( + originalDictionary.getVector().getObject(j), + roundTripDictionary.getVector().getObject(j)); + } + } + } } } @@ -141,7 +181,7 @@ private static void roundTripByteArrayTest( Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); // Read back in and compare - try (VectorSchemaRoot roundTrip = readDataFile(schema, dataFile, allocator)) { + try (VectorSchemaRoot roundTrip = readDataFile(schema, dataFile, allocator, null)) { assertEquals(root.getSchema(), roundTrip.getSchema()); assertEquals(rowCount, roundTrip.getRowCount()); @@ -1603,4 +1643,58 @@ public void testRoundTripNullableStructs() throws Exception { roundTripTest(root, allocator, dataFile, rowCount); } } + + @Test + public void testRoundTripEnum() throws Exception { + + BufferAllocator allocator = new RootAllocator(); + + // Create a dictionary + FieldType dictionaryField = new FieldType(false, new ArrowType.Utf8(), null); + VarCharVector dictionaryVector = + new VarCharVector(new Field("dictionary", dictionaryField, null), allocator); + + dictionaryVector.allocateNew(3); + dictionaryVector.set(0, "apple".getBytes()); + dictionaryVector.set(1, "banana".getBytes()); + dictionaryVector.set(2, "cherry".getBytes()); + dictionaryVector.setValueCount(3); + + // For simplicity, ensure the index type matches what will be decoded during Avro enum decoding + Dictionary dictionary = + new Dictionary( + dictionaryVector, new DictionaryEncoding(0L, false, new ArrowType.Int(8, true))); + DictionaryProvider dictionaries = new DictionaryProvider.MapDictionaryProvider(dictionary); + + // Field definition + FieldType stringField = new FieldType(false, new ArrowType.Utf8(), null); + VarCharVector stringVector = + new VarCharVector(new Field("enumField", stringField, null), allocator); + stringVector.allocateNew(10); + stringVector.setSafe(0, "apple".getBytes()); + stringVector.setSafe(1, "banana".getBytes()); + stringVector.setSafe(2, "cherry".getBytes()); + stringVector.setSafe(3, "cherry".getBytes()); + stringVector.setSafe(4, "apple".getBytes()); + stringVector.setSafe(5, "banana".getBytes()); + stringVector.setSafe(6, "apple".getBytes()); + stringVector.setSafe(7, "cherry".getBytes()); + stringVector.setSafe(8, "banana".getBytes()); + stringVector.setSafe(9, "apple".getBytes()); + stringVector.setValueCount(10); + + TinyIntVector encodedVector = + (TinyIntVector) DictionaryEncoder.encode(stringVector, dictionary); + + // Set up VSR + List vectors = Arrays.asList(encodedVector); + int rowCount = 10; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + File dataFile = new File(TMP, "testRoundTripEnums.avro"); + + roundTripTest(root, allocator, dataFile, rowCount, dictionaries); + } + } } diff --git a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripSchemaTest.java b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripSchemaTest.java index 864e2c8b59..37c0b4d9fe 100644 --- a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripSchemaTest.java +++ b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripSchemaTest.java @@ -21,27 +21,50 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.dictionary.Dictionary; +import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.types.DateUnit; import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.DictionaryEncoding; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.avro.Schema; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class RoundTripSchemaTest { private void doRoundTripTest(List fields) { + doRoundTripTest(fields, null); + } - AvroToArrowConfig config = new AvroToArrowConfig(null, 1, null, Collections.emptySet(), false); + private void doRoundTripTest(List fields, DictionaryProvider dictionaries) { - Schema avroSchema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord"); + DictionaryProvider.MapDictionaryProvider decodeDictionaries = + new DictionaryProvider.MapDictionaryProvider(); + AvroToArrowConfig decodeConfig = + new AvroToArrowConfig(null, 1, decodeDictionaries, Collections.emptySet(), false); + + Schema avroSchema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord", null, dictionaries); org.apache.arrow.vector.types.pojo.Schema arrowSchema = - AvroToArrowUtils.createArrowSchema(avroSchema, config); + AvroToArrowUtils.createArrowSchema(avroSchema, decodeConfig); // Compare string representations - equality not defined for logical types assertEquals(fields, arrowSchema.getFields()); + + for (int i = 0; i < fields.size(); i++) { + Field field = fields.get(i); + Field rtField = arrowSchema.getFields().get(i); + if (field.getDictionary() != null) { + // Dictionary content is not decoded until the data is consumed + Assertions.assertNotNull(rtField.getDictionary()); + } + } } // Schema round trip for primitive types, nullable and non-nullable @@ -440,4 +463,38 @@ public void testRoundTripStructType() { doRoundTripTest(fields); } + + @Test + public void testRoundTripEnumType() { + + BufferAllocator allocator = new RootAllocator(); + + FieldType dictionaryField = new FieldType(false, new ArrowType.Utf8(), null); + VarCharVector dictionaryVector = + new VarCharVector(new Field("dictionary", dictionaryField, null), allocator); + + dictionaryVector.allocateNew(3); + dictionaryVector.set(0, "apple".getBytes()); + dictionaryVector.set(1, "banana".getBytes()); + dictionaryVector.set(2, "cherry".getBytes()); + dictionaryVector.setValueCount(3); + + // For simplicity, ensure the index type matches what will be decoded during Avro enum decoding + Dictionary dictionary = + new Dictionary( + dictionaryVector, new DictionaryEncoding(0L, false, new ArrowType.Int(8, true))); + DictionaryProvider dictionaries = new DictionaryProvider.MapDictionaryProvider(dictionary); + + List fields = + Arrays.asList( + new Field( + "enumField", + new FieldType( + true, + new ArrowType.Int(8, true), + new DictionaryEncoding(0L, false, new ArrowType.Int(8, true))), + null)); + + doRoundTripTest(fields, dictionaries); + } } From d0e9d147a924f0cada7bfb477bd3e3d23b7f1216 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 08:05:58 +0200 Subject: [PATCH 186/408] MINOR: Bump com.gradle:common-custom-user-data-maven-extension from 2.0.1 to 2.0.3 (#785) Bumps [com.gradle:common-custom-user-data-maven-extension](https://github.com/gradle/common-custom-user-data-maven-extension) from 2.0.1 to 2.0.3.

Release notes

Sourced from com.gradle:common-custom-user-data-maven-extension's releases.

2.0.3

  • [NEW] Added tagging VS Code builds
  • [FIX] Redact user info from git urls when the user info contains a URL-encoded character

2.0.2

  • [FIX] Update to Groovy 4 to handle Java 24
Commits
  • c744e1a [maven-release-plugin] prepare release v2.0.3
  • 1aff439 Prepare changes.md for release
  • 1a81cf2 Redact URL-encoded characters in userinfo of URLs (#289)
  • e480f07 Combined PRs (#288)
  • feda61d Publish to Maven Central with central-publishing-maven-plugin (#285)
  • 9a96149 Bump org.junit.jupiter:junit-jupiter from 5.12.2 to 5.13.0 (#283)
  • dbc804b Merge pull request #282 from gradle/dependabot/maven/com.gradle-develocity-ma...
  • 328a79c Bump com.gradle:develocity-maven-extension from 2.0 to 2.0.1
  • 23680e3 Merge pull request #281 from gradle/dependabot/maven/org.apache.groovy-groovy...
  • 953c7aa Bump org.apache.groovy:groovy from 4.0.26 to 4.0.27
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.gradle:common-custom-user-data-maven-extension&package-manager=maven&previous-version=2.0.1&new-version=2.0.3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .mvn/extensions.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index 6fd036232f..943140738d 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -28,6 +28,6 @@ com.gradle common-custom-user-data-maven-extension - 2.0.1 + 2.0.3 From 7618274f9d537aa2e626f97ae5c1fa3cc0df8c99 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 08:07:29 +0200 Subject: [PATCH 187/408] MINOR: Bump io.grpc:grpc-bom from 1.71.0 to 1.73.0 (#781) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [io.grpc:grpc-bom](https://github.com/grpc/grpc-java) from 1.71.0 to 1.73.0.
Release notes

Sourced from io.grpc:grpc-bom's releases.

V1.72.0

API Changes

  • util: Remove deprecated method GracefulSwitchLb.switchTo() (f207be39a). It is rarely used outside of gRPC itself. The configuration is passed as lb policy configuration instead
  • xds: Add support for custom per-target credentials on the transport (#11951) (1958e4237)
  • xds: Explicitly set request hash key for the ring hash LB policy (892144dca)

Bug Fixes

  • core: Apply ManagedChannelImpl's updateBalancingState() immediately (ca4819ac6)
  • xds: Fix cluster selection races when updating config selector (d82613a74)
  • otel: Fix span names as per the A72 gRFC changes (#11974) (94f8e9369)
  • xds: ClusterResolverLoadBalancer handle update for both resolved addresses and errors via ResolutionResult (#11997) (868178651)

Improvements

  • netty: Avoid allocating an exception on transport shutdown. This reduces allocation rate for connection-heavy workloads/load testing (a57c14a51)
  • servlet: Set an explicit description for CANCELLED status (#11927) (fca1d3cf4)
  • xds: gRFC A74 xDS Config Tears implementation in the XdsNameResolver (e80c19745). While there is more remaining, users may already see reduced latency when resources are replaced. For example, if changing a route from one backend service to another, RPCs may see less latency during the transition
  • core: Log any exception during channel panic because of exception (3961a923a). This prevents the exception from propagating up the stack on an arbitrary thread. Such exceptions are rarely interesting. Instead, the exception that caused the channel panic is the important one, and RPCs will still fail with its details
  • util: Graceful switch to new LB when leaving CONNECTING (2e260a4bb). Previously when using xDS and the configuration changes the LB policy, the old LB policy is used until the new one is READY. Now the old LB policy is used until the new policy becomes READY, TRANSIENT_FAILURE, or IDLE
  • core: Use java.time.Time.getNano directly in InstantTimeProvider. Previously reflection was used which would confuse R8 full mode (#11977) (7507a9ec0)
  • core: Avoid cancellation exceptions when notifying watchers that already have their connections cancelled (#11934) (350f90e1a)
  • rls: allow maxAge in RLS config to exceed 5 minutes if staleAge is set. Previously, the limit was 5 minutes, which isn't enough for some gRPC clients (#11931) (c340f4a2f)
  • xds: avoid unnecessary dns lookup for CIDR addresses (#11932) (602aece08)
  • netty: Swap to UniformStreamByteDistributor (#11954) (2f52a0036). gRPC will no longer observe the HTTP/2 priorities, which were not used directly by gRPC and deprecated in RFC 9113
  • core: Avoid Set.removeAll() when passing a possibly-large List (#11994) (666136b4b)
  • stub: trailersFromThrowable() metadata should be copied (#11979) (a6e1c1f09)

New Features

  • xds: xDS-based HTTP CONNECT configuration (#11861) (12197065f)
  • netty: Per-rpc authority verification against peer cert subject names. Overriding transport authority at rpc time is only allowed when using TlsChannelCredentials. The per-rpc authority verification feature is guarded by the environment variable GRPC_ENABLE_PER_RPC_AUTHORITY_CHECK in this release. When this is false or not set, the rpc will not fail when the authority verification fails but a warning will be logged. In a subsequent release the usage of this environment variable will be removed and RPCs will start failing if the authority doesn't match the peer certificate names. The environment variable is temporary; if you are depending on the existing insecure behavior, please file an issue (#11724) (cdab410b8)

Thanks to

@​panchenko @​emmanuel-ferdman @​JoeCqupt

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=io.grpc:grpc-bom&package-manager=maven&previous-version=1.71.0&new-version=1.73.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d40f428b22..49e0c47c60 100644 --- a/pom.xml +++ b/pom.xml @@ -97,7 +97,7 @@ under the License. 2.0.17 33.4.8-jre 4.1.119.Final - 1.71.0 + 1.73.0 4.30.2 2.18.3 3.4.1 From e6da71e871a3678a737a88dbe79491b4111496b4 Mon Sep 17 00:00:00 2001 From: Ivan Chesnov Date: Tue, 5 Aug 2025 05:02:48 +0300 Subject: [PATCH 188/408] GH-725: Added ExtensionReader (#726) ## What's Changed ExtensionReader was added to support reading extension types from a complex vector. It contains **read(ExtensionHolder)** method for reading to the holder. And **readObject** - for reading the value explicitly. Closes #725. --- .../templates/AbstractFieldReader.java | 17 +++++ .../AbstractPromotableFieldWriter.java | 4 +- .../main/codegen/templates/BaseReader.java | 2 +- .../main/codegen/templates/NullReader.java | 4 ++ .../codegen/templates/PromotableWriter.java | 4 ++ .../codegen/templates/UnionListWriter.java | 11 ++- .../complex/reader/ExtensionReader.java | 44 ++++++++++++ .../apache/arrow/vector/TestListVector.java | 72 +++++++++++++++++++ .../org/apache/arrow/vector/UuidVector.java | 13 ++++ .../complex/impl/TestPromotableWriter.java | 25 +++++++ .../vector/complex/impl/UuidReaderImpl.java | 64 +++++++++++++++++ .../complex/writer/TestComplexWriter.java | 41 +++++++++++ .../complex/writer/TestSimpleWriter.java | 20 ++++++ 13 files changed, 315 insertions(+), 6 deletions(-) create mode 100644 vector/src/main/java/org/apache/arrow/vector/complex/reader/ExtensionReader.java create mode 100644 vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java diff --git a/vector/src/main/codegen/templates/AbstractFieldReader.java b/vector/src/main/codegen/templates/AbstractFieldReader.java index 25b071fab7..7e84323b64 100644 --- a/vector/src/main/codegen/templates/AbstractFieldReader.java +++ b/vector/src/main/codegen/templates/AbstractFieldReader.java @@ -108,6 +108,23 @@ public void copyAsField(String name, ${name}Writer writer) { } + + public void read(ExtensionHolder holder) { + fail("Extension"); + } + + public void read(int arrayIndex, ExtensionHolder holder) { + fail("RepeatedExtension"); + } + + public void copyAsValue(AbstractExtensionTypeWriter writer) { + fail("CopyAsValueExtension"); + } + + public void copyAsField(String name, AbstractExtensionTypeWriter writer) { + fail("CopyAsFieldExtension"); + } + public FieldReader reader(String name) { fail("reader(String name)"); return null; diff --git a/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java b/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java index 951edd5eee..2e7792fcfe 100644 --- a/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java +++ b/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java @@ -295,7 +295,7 @@ public MapWriter map(boolean keysSorted) { @Override public ExtensionWriter extension(ArrowType arrowType) { - return getWriter(MinorType.EXTENSIONTYPE).extension(arrowType); + return getWriter(MinorType.LIST).extension(arrowType); } @Override @@ -325,7 +325,7 @@ public MapWriter map(String name, boolean keysSorted) { @Override public ExtensionWriter extension(String name, ArrowType arrowType) { - return getWriter(MinorType.EXTENSIONTYPE).extension(name, arrowType); + return getWriter(MinorType.STRUCT).extension(name, arrowType); } <#list vv.types as type><#list type.minor as minor> diff --git a/vector/src/main/codegen/templates/BaseReader.java b/vector/src/main/codegen/templates/BaseReader.java index e75e8a2974..c52345af21 100644 --- a/vector/src/main/codegen/templates/BaseReader.java +++ b/vector/src/main/codegen/templates/BaseReader.java @@ -73,7 +73,7 @@ public interface RepeatedMapReader extends MapReader{ public interface ScalarReader extends <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> ${name}Reader, - BaseReader {} + ExtensionReader, BaseReader {} interface ComplexReader{ StructReader rootAsStruct(); diff --git a/vector/src/main/codegen/templates/NullReader.java b/vector/src/main/codegen/templates/NullReader.java index 1d77248e96..88e6ea98ea 100644 --- a/vector/src/main/codegen/templates/NullReader.java +++ b/vector/src/main/codegen/templates/NullReader.java @@ -86,6 +86,10 @@ public void read(int arrayIndex, Nullable${name}Holder holder){ } + public void read(ExtensionHolder holder) { + holder.isSet = 0; + } + public int size(){ return 0; } diff --git a/vector/src/main/codegen/templates/PromotableWriter.java b/vector/src/main/codegen/templates/PromotableWriter.java index 8d7d57bb9d..d22eb00b2c 100644 --- a/vector/src/main/codegen/templates/PromotableWriter.java +++ b/vector/src/main/codegen/templates/PromotableWriter.java @@ -550,6 +550,10 @@ public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory factory) { getWriter(MinorType.EXTENSIONTYPE).addExtensionTypeWriterFactory(factory); } + public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory factory, ArrowType arrowType) { + getWriter(MinorType.EXTENSIONTYPE, arrowType).addExtensionTypeWriterFactory(factory); + } + @Override public void allocate() { getWriter().allocate(); diff --git a/vector/src/main/codegen/templates/UnionListWriter.java b/vector/src/main/codegen/templates/UnionListWriter.java index 9424533f29..94723e6c9d 100644 --- a/vector/src/main/codegen/templates/UnionListWriter.java +++ b/vector/src/main/codegen/templates/UnionListWriter.java @@ -53,6 +53,7 @@ public class Union${listName}Writer extends AbstractFieldWriter { private boolean inStruct = false; private boolean listStarted = false; private String structName; + private ArrowType extensionType; <#if listName == "LargeList" || listName == "LargeListView"> private static final long OFFSET_WIDTH = 8; <#else> @@ -203,8 +204,8 @@ public MapWriter map(String name, boolean keysSorted) { @Override public ExtensionWriter extension(ArrowType arrowType) { - writer.extension(arrowType); - return writer; + this.extensionType = arrowType; + return this; } @Override public ExtensionWriter extension(String name, ArrowType arrowType) { @@ -337,13 +338,17 @@ public void writeNull() { @Override public void writeExtension(Object value) { writer.writeExtension(value); + writer.setPosition(writer.idx() + 1); } + @Override public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory var1) { - writer.addExtensionTypeWriterFactory(var1); + writer.addExtensionTypeWriterFactory(var1, extensionType); } + public void write(ExtensionHolder var1) { writer.write(var1); + writer.setPosition(writer.idx() + 1); } <#list vv.types as type> diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/reader/ExtensionReader.java b/vector/src/main/java/org/apache/arrow/vector/complex/reader/ExtensionReader.java new file mode 100644 index 0000000000..1ba7b27156 --- /dev/null +++ b/vector/src/main/java/org/apache/arrow/vector/complex/reader/ExtensionReader.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.complex.reader; + +import org.apache.arrow.vector.holders.ExtensionHolder; + +/** Interface for reading extension types. Extends the functionality of {@link BaseReader}. */ +public interface ExtensionReader extends BaseReader { + + /** + * Reads to the given extension holder. + * + * @param holder the {@link ExtensionHolder} to read + */ + void read(ExtensionHolder holder); + + /** + * Reads and returns an object representation of the extension type. + * + * @return the object representation of the extension type + */ + Object readObject(); + + /** + * Checks if the current value is set. + * + * @return true if the value is set, false otherwise + */ + boolean isSet(); +} diff --git a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index 5b2043a014..d58b7cc941 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -24,16 +24,22 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.UUID; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.util.AutoCloseables; import org.apache.arrow.vector.complex.BaseRepeatedValueVector; import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.impl.UnionListReader; import org.apache.arrow.vector.complex.impl.UnionListWriter; +import org.apache.arrow.vector.complex.impl.UuidWriterFactory; import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.complex.writer.BaseWriter.ExtensionWriter; +import org.apache.arrow.vector.holder.UuidHolder; import org.apache.arrow.vector.holders.DurationHolder; import org.apache.arrow.vector.holders.FixedSizeBinaryHolder; import org.apache.arrow.vector.holders.TimeStampMilliTZHolder; @@ -42,6 +48,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.UuidType; import org.apache.arrow.vector.util.TransferPair; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -1199,6 +1206,71 @@ public void testGetTransferPairWithField() { } } + @Test + public void testListVectorWithExtensionType() throws Exception { + final FieldType type = FieldType.nullable(new UuidType()); + try (final ListVector inVector = new ListVector("list", allocator, type, null)) { + UnionListWriter writer = inVector.getWriter(); + writer.allocate(); + writer.setPosition(0); + UUID u1 = UUID.randomUUID(); + UUID u2 = UUID.randomUUID(); + writer.startList(); + ExtensionWriter extensionWriter = writer.extension(new UuidType()); + extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); + extensionWriter.writeExtension(u1); + extensionWriter.writeExtension(u2); + writer.endList(); + + writer.setValueCount(1); + + FieldReader reader = inVector.getReader(); + assertTrue(reader.isSet(), "shouldn't be null"); + Object result = inVector.getObject(0); + ArrayList resultSet = (ArrayList) result; + assertEquals(2, resultSet.size()); + assertEquals(u1, resultSet.get(0)); + assertEquals(u2, resultSet.get(1)); + } + } + + @Test + public void testListVectorReaderForExtensionType() throws Exception { + final FieldType type = FieldType.nullable(new UuidType()); + try (final ListVector inVector = new ListVector("list", allocator, type, null)) { + UnionListWriter writer = inVector.getWriter(); + writer.allocate(); + writer.setPosition(0); + UUID u1 = UUID.randomUUID(); + UUID u2 = UUID.randomUUID(); + writer.startList(); + ExtensionWriter extensionWriter = writer.extension(new UuidType()); + extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); + extensionWriter.writeExtension(u1); + extensionWriter.writeExtension(u2); + writer.endList(); + + writer.setValueCount(1); + + UnionListReader reader = inVector.getReader(); + assertTrue(reader.isSet(), "shouldn't be null"); + reader.setPosition(0); + reader.next(); + FieldReader uuidReader = reader.reader(); + UuidHolder holder = new UuidHolder(); + uuidReader.read(holder); + ByteBuffer bb = ByteBuffer.wrap(holder.value); + UUID actualUuid = new UUID(bb.getLong(), bb.getLong()); + assertEquals(u1, actualUuid); + reader.next(); + uuidReader = reader.reader(); + uuidReader.read(holder); + bb = ByteBuffer.wrap(holder.value); + actualUuid = new UUID(bb.getLong(), bb.getLong()); + assertEquals(u2, actualUuid); + } + } + private void writeIntValues(UnionListWriter writer, int[] values) { writer.startList(); for (int v : values) { diff --git a/vector/src/test/java/org/apache/arrow/vector/UuidVector.java b/vector/src/test/java/org/apache/arrow/vector/UuidVector.java index 5c90d45f60..72ba4aa555 100644 --- a/vector/src/test/java/org/apache/arrow/vector/UuidVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/UuidVector.java @@ -20,6 +20,9 @@ import java.util.UUID; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.util.hash.ArrowBufHasher; +import org.apache.arrow.vector.complex.impl.UuidReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holder.UuidHolder; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.UuidType; @@ -79,11 +82,21 @@ public TransferPair makeTransferPair(ValueVector to) { return new TransferImpl((UuidVector) to); } + @Override + protected FieldReader getReaderImpl() { + return new UuidReaderImpl(this); + } + public void setSafe(int index, byte[] value) { getUnderlyingVector().setIndexDefined(index); getUnderlyingVector().setSafe(index, value); } + public void get(int index, UuidHolder holder) { + holder.value = getUnderlyingVector().get(index); + holder.isSet = 1; + } + public class TransferImpl implements TransferPair { UuidVector to; ValueVector targetUnderlyingVector; diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java index 1556852c5a..7b8b1f9ef9 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java @@ -805,4 +805,29 @@ public void testExtensionType() throws Exception { assertEquals(u2, uuidVector.getObject(1)); } } + + @Test + public void testExtensionTypeForList() throws Exception { + try (final ListVector container = ListVector.empty(EMPTY_SCHEMA_PATH, allocator); + final UuidVector v = + (UuidVector) container.addOrGetVector(FieldType.nullable(new UuidType())).getVector(); + final PromotableWriter writer = new PromotableWriter(v, container)) { + UUID u1 = UUID.randomUUID(); + UUID u2 = UUID.randomUUID(); + container.allocateNew(); + container.setValueCount(1); + writer.addExtensionTypeWriterFactory(new UuidWriterFactory()); + + writer.setPosition(0); + writer.writeExtension(u1); + writer.setPosition(1); + writer.writeExtension(u2); + + container.setValueCount(2); + + UuidVector uuidVector = (UuidVector) container.getDataVector(); + assertEquals(u1, uuidVector.getObject(0)); + assertEquals(u2, uuidVector.getObject(1)); + } + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java new file mode 100644 index 0000000000..16dd734de8 --- /dev/null +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.complex.impl; + +import org.apache.arrow.vector.UuidVector; +import org.apache.arrow.vector.holder.UuidHolder; +import org.apache.arrow.vector.holders.ExtensionHolder; +import org.apache.arrow.vector.types.Types.MinorType; +import org.apache.arrow.vector.types.pojo.Field; + +public class UuidReaderImpl extends AbstractFieldReader { + + private final UuidVector vector; + + public UuidReaderImpl(UuidVector vector) { + super(); + this.vector = vector; + } + + @Override + public MinorType getMinorType() { + return vector.getMinorType(); + } + + @Override + public Field getField() { + return vector.getField(); + } + + @Override + public boolean isSet() { + return !vector.isNull(idx()); + } + + @Override + public void read(ExtensionHolder holder) { + vector.get(idx(), (UuidHolder) holder); + } + + @Override + public void read(int arrayIndex, ExtensionHolder holder) { + vector.get(arrayIndex, (UuidHolder) holder); + } + + @Override + public void copyAsValue(AbstractExtensionTypeWriter writer) { + UuidWriterImpl impl = (UuidWriterImpl) writer; + impl.vector.copyFromSafe(idx(), impl.idx(), vector); + } +} diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index 2745386db4..f374eb41e4 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -19,6 +19,7 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -31,6 +32,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.UUID; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; @@ -64,6 +66,7 @@ import org.apache.arrow.vector.complex.impl.UnionMapReader; import org.apache.arrow.vector.complex.impl.UnionReader; import org.apache.arrow.vector.complex.impl.UnionWriter; +import org.apache.arrow.vector.complex.impl.UuidWriterFactory; import org.apache.arrow.vector.complex.reader.BaseReader.StructReader; import org.apache.arrow.vector.complex.reader.BigIntReader; import org.apache.arrow.vector.complex.reader.FieldReader; @@ -71,9 +74,11 @@ import org.apache.arrow.vector.complex.reader.Float8Reader; import org.apache.arrow.vector.complex.reader.IntReader; import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter; +import org.apache.arrow.vector.complex.writer.BaseWriter.ExtensionWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.StructWriter; +import org.apache.arrow.vector.holder.UuidHolder; import org.apache.arrow.vector.holders.DecimalHolder; import org.apache.arrow.vector.holders.DurationHolder; import org.apache.arrow.vector.holders.FixedSizeBinaryHolder; @@ -84,6 +89,7 @@ import org.apache.arrow.vector.holders.NullableTimeStampNanoTZHolder; import org.apache.arrow.vector.holders.TimeStampMilliTZHolder; import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID; import org.apache.arrow.vector.types.pojo.ArrowType.Int; @@ -93,6 +99,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType.Utf8; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.UuidType; import org.apache.arrow.vector.util.CallBack; import org.apache.arrow.vector.util.DecimalUtility; import org.apache.arrow.vector.util.JsonStringArrayList; @@ -2489,4 +2496,38 @@ public void unionWithVarCharAndBinaryHelpers() throws Exception { "row12", new String(vector.getLargeVarBinaryVector().get(11), StandardCharsets.UTF_8)); } } + + @Test + public void extensionWriterReader() throws Exception { + // test values + UUID u1 = UUID.randomUUID(); + + try (NonNullableStructVector parent = NonNullableStructVector.empty("parent", allocator)) { + // write + + ComplexWriter writer = new ComplexWriterImpl("root", parent); + StructWriter rootWriter = writer.rootAsStruct(); + + { + ExtensionWriter extensionWriter = rootWriter.extension("uuid1", new UuidType()); + extensionWriter.setPosition(0); + extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); + extensionWriter.writeExtension(u1); + } + // read + StructReader rootReader = new SingleStructReaderImpl(parent).reader("root"); + { + FieldReader uuidReader = rootReader.reader("uuid1"); + uuidReader.setPosition(0); + UuidHolder uuidHolder = new UuidHolder(); + uuidReader.read(uuidHolder); + final ByteBuffer bb = ByteBuffer.wrap(uuidHolder.value); + UUID actualUuid = new UUID(bb.getLong(), bb.getLong()); + assertEquals(u1, actualUuid); + assertTrue(uuidReader.isSet()); + assertEquals(uuidReader.getMinorType(), MinorType.EXTENSIONTYPE); + assertInstanceOf(UuidType.class, uuidReader.getField().getFieldType().getType()); + } + } + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java index bf1b9b0dfa..269cff0670 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java @@ -30,6 +30,7 @@ import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.complex.impl.LargeVarBinaryWriterImpl; import org.apache.arrow.vector.complex.impl.LargeVarCharWriterImpl; +import org.apache.arrow.vector.complex.impl.UuidReaderImpl; import org.apache.arrow.vector.complex.impl.UuidWriterImpl; import org.apache.arrow.vector.complex.impl.VarBinaryWriterImpl; import org.apache.arrow.vector.complex.impl.VarCharWriterImpl; @@ -204,4 +205,23 @@ public void testWriteToExtensionVector() throws Exception { assertEquals(uuid, result); } } + + @Test + public void testReaderCopyAsValueExtensionVector() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator); + UuidVector vectorForRead = new UuidVector("test2", allocator); + UuidWriterImpl writer = new UuidWriterImpl(vector)) { + UUID uuid = UUID.randomUUID(); + vectorForRead.setValueCount(1); + vectorForRead.set(0, uuid); + UuidReaderImpl reader = (UuidReaderImpl) vectorForRead.getReader(); + reader.copyAsValue(writer); + UuidReaderImpl reader2 = (UuidReaderImpl) vector.getReader(); + UuidHolder holder = new UuidHolder(); + reader2.read(0, holder); + final ByteBuffer bb = ByteBuffer.wrap(holder.value); + UUID actualUuid = new UUID(bb.getLong(), bb.getLong()); + assertEquals(uuid, actualUuid); + } + } } From e4f64269db0a08299fa25be491570be5ba71d623 Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 5 Aug 2025 12:37:33 +0900 Subject: [PATCH 189/408] MINOR: Fix format (#809) ## What's Changed Apply pre-commit since I forgot. --- vector/src/main/codegen/templates/UnionListWriter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vector/src/main/codegen/templates/UnionListWriter.java b/vector/src/main/codegen/templates/UnionListWriter.java index 94723e6c9d..3c41ac72b6 100644 --- a/vector/src/main/codegen/templates/UnionListWriter.java +++ b/vector/src/main/codegen/templates/UnionListWriter.java @@ -340,12 +340,12 @@ public void writeExtension(Object value) { writer.writeExtension(value); writer.setPosition(writer.idx() + 1); } - + @Override public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory var1) { writer.addExtensionTypeWriterFactory(var1, extensionType); } - + public void write(ExtensionHolder var1) { writer.write(var1); writer.setPosition(writer.idx() + 1); From a6245ab87c782e2b873526acbd876cdf96d1f0ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Thu, 7 Aug 2025 01:29:41 +0100 Subject: [PATCH 190/408] GH-804: Prepend JDBC FlightSQL version to user agent (#806) ## What's Changed * Driver version is passed on to NettyBuilderClient to append it to the user-agent header * NettyBuilderClient now prepends `JDBC Flight SQL Client ` to the header (e.g. `JDBC Flight SQL Client 19.0.0-SNAPSHOT grpc-java-netty/1.73.0`) Closes #804. --- .../driver/jdbc/ArrowFlightConnection.java | 9 +++- .../client/ArrowFlightSqlClientHandler.java | 27 +++++++++++ .../ArrowFlightJdbcConnectionCookieTest.java | 4 +- .../arrow/driver/jdbc/ConnectionTest.java | 46 +++++++++++++++++++ .../jdbc/FlightServerTestExtension.java | 33 +++++++++---- ...rrowFlightSqlClientHandlerBuilderTest.java | 1 + 6 files changed, 107 insertions(+), 13 deletions(-) diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java index 747287ed13..f6f17770f1 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java @@ -32,6 +32,7 @@ import org.apache.arrow.util.Preconditions; import org.apache.calcite.avatica.AvaticaConnection; import org.apache.calcite.avatica.AvaticaFactory; +import org.apache.calcite.avatica.DriverVersion; /** Connection to the Arrow Flight server. */ public final class ArrowFlightConnection extends AvaticaConnection { @@ -86,13 +87,16 @@ static ArrowFlightConnection createNewConnection( throws SQLException { url = replaceSemiColons(url); final ArrowFlightConnectionConfigImpl config = new ArrowFlightConnectionConfigImpl(properties); - final ArrowFlightSqlClientHandler clientHandler = createNewClientHandler(config, allocator); + final ArrowFlightSqlClientHandler clientHandler = + createNewClientHandler(config, allocator, driver.getDriverVersion()); return new ArrowFlightConnection( driver, factory, url, properties, config, allocator, clientHandler); } private static ArrowFlightSqlClientHandler createNewClientHandler( - final ArrowFlightConnectionConfigImpl config, final BufferAllocator allocator) + final ArrowFlightConnectionConfigImpl config, + final BufferAllocator allocator, + final DriverVersion driverVersion) throws SQLException { try { return new ArrowFlightSqlClientHandler.Builder() @@ -116,6 +120,7 @@ private static ArrowFlightSqlClientHandler createNewClientHandler( .withCatalog(config.getCatalog()) .withClientCache(config.useClientCache() ? new FlightClientCache() : null) .withConnectTimeout(config.getConnectTimeout()) + .withDriverVersion(driverVersion) .build(); } catch (final SQLException e) { try { diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java index 17c2c16ebf..a3f6900373 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java @@ -66,6 +66,7 @@ import org.apache.arrow.util.VisibleForTesting; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.calcite.avatica.DriverVersion; import org.apache.calcite.avatica.Meta.StatementType; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; @@ -548,6 +549,9 @@ public FlightInfo getCrossReference( /** Builder for {@link ArrowFlightSqlClientHandler}. */ public static final class Builder { + static final String USER_AGENT_TEMPLATE = "JDBC Flight SQL Driver %s"; + static final String DEFAULT_VERSION = "(unknown or development build)"; + private final Set middlewareFactories = new HashSet<>(); private final Set options = new HashSet<>(); private String host; @@ -597,6 +601,8 @@ public static final class Builder { @VisibleForTesting ClientCookieMiddleware.Factory cookieFactory = new ClientCookieMiddleware.Factory(); + DriverVersion driverVersion; + public Builder() {} /** @@ -631,6 +637,8 @@ public Builder() {} if (original.retainAuth) { this.authFactory = original.authFactory; } + + this.driverVersion = original.driverVersion; } /** @@ -879,6 +887,17 @@ public Builder withConnectTimeout(Duration connectTimeout) { return this; } + /** + * Sets the driver version for this handler. + * + * @param driverVersion the driver version to set + * @return this builder instance + */ + public Builder withDriverVersion(DriverVersion driverVersion) { + this.driverVersion = driverVersion; + return this; + } + public String getCacheKey() { return getLocation().toString(); } @@ -914,6 +933,11 @@ public ArrowFlightSqlClientHandler build() throws SQLException { final NettyClientBuilder clientBuilder = new NettyClientBuilder(); clientBuilder.allocator(allocator); + String userAgent = String.format(USER_AGENT_TEMPLATE, DEFAULT_VERSION); + if (driverVersion != null && driverVersion.versionString != null) { + userAgent = String.format(USER_AGENT_TEMPLATE, driverVersion.versionString); + } + buildTimeMiddlewareFactories.add(new ClientCookieMiddleware.Factory()); buildTimeMiddlewareFactories.forEach(clientBuilder::intercept); if (useEncryption) { @@ -948,6 +972,9 @@ public ArrowFlightSqlClientHandler build() throws SQLException { } NettyChannelBuilder channelBuilder = clientBuilder.build(); + + channelBuilder.userAgent(userAgent); + if (connectTimeout != null) { channelBuilder.withOption( ChannelOption.CONNECT_TIMEOUT_MILLIS, (int) connectTimeout.toMillis()); diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcConnectionCookieTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcConnectionCookieTest.java index 1977b61392..7127c7fc32 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcConnectionCookieTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcConnectionCookieTest.java @@ -39,11 +39,11 @@ public void testCookies() throws SQLException { Statement statement = connection.createStatement()) { // Expect client didn't receive cookies before any operation - assertNull(FLIGHT_SERVER_TEST_EXTENSION.getMiddlewareCookieFactory().getCookie()); + assertNull(FLIGHT_SERVER_TEST_EXTENSION.getInterceptorFactory().getCookie()); // Run another action for check if the cookies was sent by the server. statement.execute(CoreMockedSqlProducers.LEGACY_REGULAR_SQL_CMD); - assertEquals("k=v", FLIGHT_SERVER_TEST_EXTENSION.getMiddlewareCookieFactory().getCookie()); + assertEquals("k=v", FLIGHT_SERVER_TEST_EXTENSION.getInterceptorFactory().getCookie()); } } } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java index 8e872a1167..72e4b222a3 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java @@ -31,6 +31,7 @@ import org.apache.arrow.driver.jdbc.client.ArrowFlightSqlClientHandler; import org.apache.arrow.driver.jdbc.utils.ArrowFlightConnectionConfigImpl.ArrowFlightConnectionProperty; import org.apache.arrow.driver.jdbc.utils.MockFlightSqlProducer; +import org.apache.arrow.flight.FlightMethod; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.util.AutoCloseables; @@ -576,4 +577,49 @@ public void testPasswordConnectionPropertyIntegerCorrectCastUrlWithDriverManager assertTrue(connection.isValid(0)); } } + + /** + * Test that the JDBC driver properly integrates driver version into client handler. + * + * @throws Exception on error. + */ + @Test + public void testJdbcDriverVersionIntegration() throws Exception { + final Properties properties = new Properties(); + properties.put( + ArrowFlightConnectionProperty.HOST.camelName(), FLIGHT_SERVER_TEST_EXTENSION.getHost()); + properties.put( + ArrowFlightConnectionProperty.PORT.camelName(), FLIGHT_SERVER_TEST_EXTENSION.getPort()); + properties.put(ArrowFlightConnectionProperty.USER.camelName(), userTest); + properties.put(ArrowFlightConnectionProperty.PASSWORD.camelName(), passTest); + properties.put(ArrowFlightConnectionProperty.USE_ENCRYPTION.camelName(), false); + + // Create a driver instance and connect + ArrowFlightJdbcDriver driverVersion = new ArrowFlightJdbcDriver(); + + try (Connection connection = + ArrowFlightConnection.createNewConnection( + driverVersion, + new ArrowFlightJdbcFactory(), + "jdbc:arrow-flight-sql://localhost:" + FLIGHT_SERVER_TEST_EXTENSION.getPort(), + properties, + allocator)) { + + assertTrue(connection.isValid(0)); + + var actualUserAgent = + FLIGHT_SERVER_TEST_EXTENSION + .getInterceptorFactory() + .getHeader(FlightMethod.HANDSHAKE, "user-agent"); + + var expectedUserAgent = + "JDBC Flight SQL Driver " + driverVersion.getDriverVersion().versionString; + // Driver appends version to grpc user-agent header. Assert the header starts with the + // expected + // value and ignored grpc version. + assertTrue( + actualUserAgent.startsWith(expectedUserAgent), + "Expected: " + expectedUserAgent + " but found: " + actualUserAgent); + } + } } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/FlightServerTestExtension.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/FlightServerTestExtension.java index aa586651f5..db0438059f 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/FlightServerTestExtension.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/FlightServerTestExtension.java @@ -25,6 +25,8 @@ import java.sql.SQLException; import java.util.ArrayDeque; import java.util.Deque; +import java.util.HashMap; +import java.util.Map; import java.util.Properties; import org.apache.arrow.driver.jdbc.authentication.Authentication; import org.apache.arrow.driver.jdbc.authentication.TokenAuthentication; @@ -33,6 +35,7 @@ import org.apache.arrow.flight.CallHeaders; import org.apache.arrow.flight.CallInfo; import org.apache.arrow.flight.CallStatus; +import org.apache.arrow.flight.FlightMethod; import org.apache.arrow.flight.FlightServer; import org.apache.arrow.flight.FlightServerMiddleware; import org.apache.arrow.flight.Location; @@ -67,7 +70,8 @@ public class FlightServerTestExtension private final CertKeyPair certKeyPair; private final File mTlsCACert; - private final MiddlewareCookie.Factory middlewareCookieFactory = new MiddlewareCookie.Factory(); + private final InterceptorMiddleware.Factory interceptorFactory = + new InterceptorMiddleware.Factory(); private FlightServerTestExtension( final Properties properties, @@ -130,8 +134,8 @@ private void setUseEncryption(boolean useEncryption) { properties.put("useEncryption", useEncryption); } - public MiddlewareCookie.Factory getMiddlewareCookieFactory() { - return middlewareCookieFactory; + public InterceptorMiddleware.Factory getInterceptorFactory() { + return interceptorFactory; } @FunctionalInterface @@ -143,7 +147,7 @@ private FlightServer initiateServer(Location location) throws IOException { FlightServer.Builder builder = FlightServer.builder(allocator, location, producer) .headerAuthenticator(authentication.authenticate()) - .middleware(FlightServerMiddleware.Key.of("KEY"), middlewareCookieFactory); + .middleware(FlightServerMiddleware.Key.of("KEY"), interceptorFactory); if (certKeyPair != null) { builder.useTls(certKeyPair.cert, certKeyPair.key); } @@ -301,11 +305,11 @@ public FlightServerTestExtension build() { * A middleware to handle with the cookies in the server. It is used to test if cookies are being * sent properly. */ - static class MiddlewareCookie implements FlightServerMiddleware { + static class InterceptorMiddleware implements FlightServerMiddleware { private final Factory factory; - public MiddlewareCookie(Factory factory) { + public InterceptorMiddleware(Factory factory) { this.factory = factory; } @@ -323,22 +327,33 @@ public void onCallCompleted(CallStatus callStatus) {} public void onCallErrored(Throwable throwable) {} /** A factory for the MiddlewareCookie. */ - static class Factory implements FlightServerMiddleware.Factory { + static class Factory implements FlightServerMiddleware.Factory { + private final Map receivedCallHeaders = new HashMap<>(); private boolean receivedCookieHeader = false; private String cookie; @Override - public MiddlewareCookie onCallStarted( + public InterceptorMiddleware onCallStarted( CallInfo callInfo, CallHeaders callHeaders, RequestContext requestContext) { cookie = callHeaders.get("Cookie"); receivedCookieHeader = null != cookie; - return new MiddlewareCookie(this); + + receivedCallHeaders.put(callInfo.method(), callHeaders); + return new InterceptorMiddleware(this); } public String getCookie() { return cookie; } + + public String getHeader(FlightMethod method, String key) { + CallHeaders headers = receivedCallHeaders.get(method); + if (headers == null) { + return null; + } + return headers.get(key); + } } } } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerBuilderTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerBuilderTest.java index 6524eaf39a..a60a71f23d 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerBuilderTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerBuilderTest.java @@ -149,6 +149,7 @@ public void testDefaults() { assertEquals(Optional.empty(), builder.catalog); assertNull(builder.flightClientCache); assertNull(builder.connectTimeout); + assertNull(builder.driverVersion); } @Test From 199e47bc2649538ca091c17ffb4c77b8dfd5966e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 05:53:05 +0900 Subject: [PATCH 191/408] MINOR: [CI] Bump actions/cache from 4.2.3 to 4.2.4 (#813) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/cache](https://github.com/actions/cache) from 4.2.3 to 4.2.4.
Release notes

Sourced from actions/cache's releases.

v4.2.4

What's Changed

New Contributors

Full Changelog: https://github.com/actions/cache/compare/v4...v4.2.4

Changelog

Sourced from actions/cache's changelog.

Releases

4.2.4

  • Bump @actions/cache to v4.0.5

4.2.3

  • Bump @actions/cache to v4.0.3 (obfuscates SAS token in debug logs for cache entries)

4.2.2

  • Bump @actions/cache to v4.0.2

4.2.1

  • Bump @actions/cache to v4.0.1

4.2.0

TLDR; The cache backend service has been rewritten from the ground up for improved performance and reliability. actions/cache now integrates with the new cache service (v2) APIs.

The new service will gradually roll out as of February 1st, 2025. The legacy service will also be sunset on the same date. Changes in these release are fully backward compatible.

We are deprecating some versions of this action. We recommend upgrading to version v4 or v3 as soon as possible before February 1st, 2025. (Upgrade instructions below).

If you are using pinned SHAs, please use the SHAs of versions v4.2.0 or v3.4.0

If you do not upgrade, all workflow runs using any of the deprecated actions/cache will fail.

Upgrading to the recommended versions will not break your workflows.

4.1.2

  • Add GitHub Enterprise Cloud instances hostname filters to inform API endpoint choices - #1474
  • Security fix: Bump braces from 3.0.2 to 3.0.3 - #1475

4.1.1

  • Restore original behavior of cache-hit output - #1467

4.1.0

  • Ensure cache-hit output is set when a cache is missed - #1404
  • Deprecate save-always input - #1452

4.0.2

  • Fixed restore fail-on-cache-miss not working.

... (truncated)

Commits
  • 0400d5f Merge pull request #1636 from actions/Link-/release-4.2.4
  • 374a27f Prepare release 4.2.4
  • 358a730 Merge pull request #1634 from actions/Link-/optimise-deps
  • 2ee706e Fix with another approach
  • 94f7b5d Fix bundle exec
  • c36116c Fix the workflow to use licensed from source
  • 320fe7d Update the licensed workflow to use the latest version
  • d81cc47 Add licensed output
  • de24398 Add licensed output
  • e7b6a9c @​protobuf-ts/plugin to dev dependencies
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/cache&package-manager=github_actions&previous-version=4.2.3&new-version=4.2.4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/dev.yml | 2 +- .github/workflows/rc.yml | 8 ++++---- .github/workflows/test.yml | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 37320898a6..4242af55c3 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -42,7 +42,7 @@ jobs: with: python-version: '3.x' - name: pre-commit (cache) - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 with: path: ~/.cache/pre-commit key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 72456fa556..8836537b7f 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -139,7 +139,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Cache - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 with: path: .docker key: jni-linux-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} @@ -270,7 +270,7 @@ jobs: run: | echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} - name: Cache ccache - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 with: path: ccache key: jni-macos-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} @@ -346,7 +346,7 @@ jobs: run: | echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} - name: Cache ccache - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 with: path: ccache key: jni-windows-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} @@ -420,7 +420,7 @@ jobs: repository: apache/arrow-testing path: testing - name: Cache ~/.m2 - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 with: path: ~/.m2 key: binaries-build-${{ hashFiles('**/*.java', '**/pom.xml') }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f1028ce029..c461b3c19e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,7 +63,7 @@ jobs: fetch-depth: 0 submodules: recursive - name: Cache Docker Volumes - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 with: path: .docker key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('compose.yaml', '**/pom.xml', '**/*.java') }} @@ -185,7 +185,7 @@ jobs: run: | ci/scripts/util_free_space.sh - name: Cache Docker Volumes - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 with: path: .docker key: integration-conda-${{ hashFiles('cpp/**') }} From edf64236b1ee58ca1ff081c7dca31384e6b0e280 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 05:53:32 +0900 Subject: [PATCH 192/408] MINOR: [CI] Bump docker/login-action from 3.4.0 to 3.5.0 (#814) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [docker/login-action](https://github.com/docker/login-action) from 3.4.0 to 3.5.0.
Release notes

Sourced from docker/login-action's releases.

v3.5.0

Full Changelog: https://github.com/docker/login-action/compare/v3.4.0...v3.5.0

Commits
  • 184bdaa Merge pull request #878 from docker/dependabot/npm_and_yarn/aws-sdk-dependenc...
  • 5c6bc94 chore: update generated content
  • caf4058 build(deps): bump the aws-sdk-dependencies group with 2 updates
  • ef38ec3 Merge pull request #860 from docker/dependabot/npm_and_yarn/aws-sdk-dependenc...
  • d52e8ef chore: update generated content
  • 9644ab7 build(deps): bump the aws-sdk-dependencies group with 2 updates
  • 7abd1d5 Merge pull request #875 from docker/dependabot/npm_and_yarn/form-data-2.5.5
  • 1a81202 Merge pull request #876 from crazy-max/aws-public-dual-stack
  • d1ab30d chore: update generated content
  • f25ff28 support dual-stack for aws public ecr
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=docker/login-action&package-manager=github_actions&previous-version=3.4.0&new-version=3.5.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 8836537b7f..98778dd0d0 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -133,7 +133,7 @@ jobs: with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing - - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + - uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 with: registry: ghcr.io username: ${{ github.actor }} From 65caf509bdb33b96e576ce15cb734fe1689bdb37 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 05:54:58 +0900 Subject: [PATCH 193/408] MINOR: [CI] Bump actions/download-artifact from 4.3.0 to 5.0.0 (#815) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 4.3.0 to 5.0.0.
Release notes

Sourced from actions/download-artifact's releases.

v5.0.0

What's Changed

v5.0.0

🚨 Breaking Change

This release fixes an inconsistency in path behavior for single artifact downloads by ID. If you're downloading single artifacts by ID, the output path may change.

What Changed

Previously, single artifact downloads behaved differently depending on how you specified the artifact:

  • By name: name: my-artifact → extracted to path/ (direct)
  • By ID: artifact-ids: 12345 → extracted to path/my-artifact/ (nested)

Now both methods are consistent:

  • By name: name: my-artifact → extracted to path/ (unchanged)
  • By ID: artifact-ids: 12345 → extracted to path/ (fixed - now direct)

Migration Guide

✅ No Action Needed If:
  • You download artifacts by name
  • You download multiple artifacts by ID
  • You already use merge-multiple: true as a workaround
⚠️ Action Required If:

You download single artifacts by ID and your workflows expect the nested directory structure.

Before v5 (nested structure):

- uses: actions/download-artifact@v4
  with:
    artifact-ids: 12345
    path: dist
# Files were in: dist/my-artifact/

Where my-artifact is the name of the artifact you previously uploaded

To maintain old behavior (if needed):

</tr></table>

... (truncated)

Commits
  • 634f93c Merge pull request #416 from actions/single-artifact-id-download-path
  • b19ff43 refactor: resolve download path correctly in artifact download tests (mainly ...
  • e262cbe bundle dist
  • bff23f9 update docs
  • fff8c14 fix download path logic when downloading a single artifact by id
  • 448e3f8 Merge pull request #407 from actions/nebuk89-patch-1
  • 47225c4 Update README.md
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/download-artifact&package-manager=github_actions&previous-version=4.3.0&new-version=5.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 98778dd0d0..9a8d7d9563 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -101,7 +101,7 @@ jobs: packages: write steps: - name: Download source archive - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: name: release-source - name: Extract source archive @@ -174,7 +174,7 @@ jobs: MACOSX_DEPLOYMENT_TARGET: "14.0" steps: - name: Download source archive - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: name: release-source - name: Extract source archive @@ -302,7 +302,7 @@ jobs: arch: "x86_64" steps: - name: Download source archive - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: name: release-source - name: Extract source archive @@ -375,7 +375,7 @@ jobs: - jni-windows steps: - name: Download artifacts - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: path: artifacts - name: Decompress artifacts @@ -456,11 +456,11 @@ jobs: with: cache: 'pip' - name: Download source archive - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: name: release-source - name: Download Javadocs - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: name: reference - name: Extract source archive @@ -525,7 +525,7 @@ jobs: cp ../.asf.yaml ./ git add .nojekyll .asf.yaml - name: Download - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: name: release-html - name: Extract @@ -561,7 +561,7 @@ jobs: - ubuntu-latest steps: - name: Download release artifacts - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: pattern: release-* - name: Verify @@ -595,7 +595,7 @@ jobs: contents: write steps: - name: Download release artifacts - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: pattern: release-* path: artifacts From 52f7a86b0f315df8848dd68cb7457e13236b5fd5 Mon Sep 17 00:00:00 2001 From: David Schlosnagle Date: Wed, 13 Aug 2025 19:39:22 -0400 Subject: [PATCH 194/408] GH-816: Presize JsonStringArrayList vector results (#817) ## What's Changed Presize `JsonStringArrayList`s when constructing them as part of `ValueVector#getObject` conversions. `FixedSizeListVector#getObject` already performs this optimization; however, `ListVector`, `ListViewVector`, `LargeListVector`, and `LargeListViewVector` do not yet presize the result `JsonStringArrayList` requiring dynamic reallocations as elements are converted & added. This can become a scalability bottleneck when using these types. Closes #816. --- .../apache/arrow/adapter/avro/AvroToArrowUtils.java | 4 ++-- .../arrow/adapter/avro/AvroToArrowVectorIterator.java | 11 +++++++---- .../apache/arrow/vector/complex/LargeListVector.java | 4 +++- .../arrow/vector/complex/LargeListViewVector.java | 2 +- .../org/apache/arrow/vector/complex/ListVector.java | 2 +- .../apache/arrow/vector/complex/ListViewVector.java | 2 +- 6 files changed, 15 insertions(+), 10 deletions(-) diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java index aedef7732e..a6e77e4050 100644 --- a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java @@ -1071,8 +1071,8 @@ private static FieldType createFieldType( } private static String convertAliases(Set aliases) { - JsonStringArrayList jsonList = new JsonStringArrayList(); - aliases.stream().forEach(a -> jsonList.add(a)); + JsonStringArrayList jsonList = new JsonStringArrayList(aliases.size()); + jsonList.addAll(aliases); return jsonList.toString(); } } diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowVectorIterator.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowVectorIterator.java index 4123370061..e82fdc36fb 100644 --- a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowVectorIterator.java +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowVectorIterator.java @@ -17,13 +17,14 @@ package org.apache.arrow.adapter.avro; import java.io.EOFException; -import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.stream.Collectors; import org.apache.arrow.adapter.avro.consumers.CompositeAvroConsumer; +import org.apache.arrow.adapter.avro.consumers.Consumer; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.util.ValueVectorUtility; @@ -75,9 +76,11 @@ public static AvroToArrowVectorIterator create( private void initialize() { // create consumers compositeConsumer = AvroToArrowUtils.createCompositeConsumer(schema, config); - List vectors = new ArrayList<>(); - compositeConsumer.getConsumers().forEach(c -> vectors.add(c.getVector())); - List fields = vectors.stream().map(t -> t.getField()).collect(Collectors.toList()); + List vectors = + compositeConsumer.getConsumers().stream() + .map(Consumer::getVector) + .collect(Collectors.toList()); + List fields = vectors.stream().map(ValueVector::getField).collect(Collectors.toList()); VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, 0); rootSchema = root.getSchema(); diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java index 835d3468f3..997b5a8b78 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java @@ -31,6 +31,7 @@ import org.apache.arrow.memory.util.ArrowBufPointer; import org.apache.arrow.memory.util.ByteFunctionHelpers; import org.apache.arrow.memory.util.CommonUtil; +import org.apache.arrow.memory.util.LargeMemoryUtil; import org.apache.arrow.memory.util.hash.ArrowBufHasher; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.AddOrGetResult; @@ -861,10 +862,11 @@ public List getObject(int index) { if (isSet(index) == 0) { return null; } - final List vals = new JsonStringArrayList<>(); final long start = offsetBuffer.getLong((long) index * OFFSET_WIDTH); final long end = offsetBuffer.getLong(((long) index + 1L) * OFFSET_WIDTH); final ValueVector vv = getDataVector(); + final List vals = + new JsonStringArrayList<>(LargeMemoryUtil.checkedCastToInt(end - start)); for (long i = start; i < end; i++) { vals.add(vv.getObject(checkedCastToInt(i))); } diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java index 394c3c67bb..2da7eb057e 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java @@ -672,10 +672,10 @@ public List getObject(int index) { if (isSet(index) == 0) { return null; } - final List vals = new JsonStringArrayList<>(); final int start = offsetBuffer.getInt(index * OFFSET_WIDTH); final int end = start + sizeBuffer.getInt((index) * SIZE_WIDTH); final ValueVector vv = getDataVector(); + final List vals = new JsonStringArrayList<>(end - start); for (int i = start; i < end; i++) { vals.add(vv.getObject(checkedCastToInt(i))); } diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 2b2817515f..93a313ef4f 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -719,10 +719,10 @@ public List getObject(int index) { if (isSet(index) == 0) { return null; } - final List vals = new JsonStringArrayList<>(); final int start = offsetBuffer.getInt(index * OFFSET_WIDTH); final int end = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH); final ValueVector vv = getDataVector(); + final List vals = new JsonStringArrayList<>(end - start); for (int i = start; i < end; i++) { vals.add(vv.getObject(i)); } diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java index 2b80101926..8711db5e0f 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java @@ -678,10 +678,10 @@ public List getObject(int index) { if (isSet(index) == 0) { return null; } - final List vals = new JsonStringArrayList<>(); final int start = offsetBuffer.getInt(index * OFFSET_WIDTH); final int end = start + sizeBuffer.getInt((index) * SIZE_WIDTH); final ValueVector vv = getDataVector(); + final List vals = new JsonStringArrayList<>(end - start); for (int i = start; i < end; i++) { vals.add(vv.getObject(i)); } From 156b465f7836dec3c3a109dbc101ee7270009ff7 Mon Sep 17 00:00:00 2001 From: Pedro Matias Date: Wed, 20 Aug 2025 11:36:12 +0100 Subject: [PATCH 195/408] GH-797: [JDBC] Fix PreparedStatement#execute for DML/DDL (#811) ## What's Changed Instead of always obtaining a result set for queries issued via PreparedStatement.execute(), we now inspect the dataset_schema returned in ActionCreatePreparedStatementResult. If the schema has no fields, we retrieve the update count instead. This aligns the return value with the expectations of the JDBC API. For such cases, the Arrow Flight SQL path now uses CommandPreparedStatementUpdate instead of CommandPreparedStatementQuery. This change mirrors the existing approach in Statement.execute() and Statement.executeUpdate(). ### Are these changes tested? Yes Closes #797. --- .../driver/jdbc/ArrowFlightMetaImpl.java | 10 +++++-- .../ArrowFlightPreparedStatementTest.java | 29 +++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java index 9c7112f1c3..21cc3e431f 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java @@ -62,14 +62,17 @@ static Signature newSignature(final String sql, Schema resultSetSchema, Schema p parameterSchema == null ? new ArrayList<>() : ConvertUtils.convertArrowFieldsToAvaticaParameters(parameterSchema.getFields()); - + StatementType statementType = + resultSetSchema == null || resultSetSchema.getFields().isEmpty() + ? StatementType.IS_DML + : StatementType.SELECT; return new Signature( columnMetaData, sql, parameters, Collections.emptyMap(), null, // unnecessary, as SQL requests use ArrowFlightJdbcCursor - StatementType.SELECT); + statementType); } @Override @@ -105,7 +108,8 @@ public ExecuteResult execute( preparedStatement, ((ArrowFlightConnection) connection).getBufferAllocator()) .bind(typedValues); - if (statementHandle.signature == null) { + if (statementHandle.signature == null + || statementHandle.signature.statementType == StatementType.IS_DML) { // Update query long updatedCount = preparedStatement.executeUpdate(); return new ExecuteResult( diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightPreparedStatementTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightPreparedStatementTest.java index 774ad0081e..0369c3a162 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightPreparedStatementTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightPreparedStatementTest.java @@ -20,6 +20,8 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.nio.charset.StandardCharsets; import java.sql.Connection; @@ -83,6 +85,19 @@ public void testSimpleQueryNoParameterBinding() throws SQLException { } } + @Test + public void testSimpleQueryNoParameterBindingWithExecute() throws SQLException { + final String query = CoreMockedSqlProducers.LEGACY_REGULAR_SQL_CMD; + try (final PreparedStatement preparedStatement = connection.prepareStatement(query)) { + boolean isResultSet = preparedStatement.execute(); + assertTrue(isResultSet); + final ResultSet resultSet = preparedStatement.getResultSet(); + CoreMockedSqlProducers.assertLegacyRegularSqlResultSet(resultSet); + assertFalse(preparedStatement.getMoreResults()); + assertEquals(-1, preparedStatement.getUpdateCount()); + } + } + @Test public void testQueryWithParameterBinding() throws SQLException { final String query = "Fake query with parameters"; @@ -174,6 +189,20 @@ public void testUpdateQuery() throws SQLException { } } + @Test + public void testUpdateQueryWithExecute() throws SQLException { + String query = "Fake update with execute"; + PRODUCER.addUpdateQuery(query, /*updatedRows*/ 42); + try (final PreparedStatement stmt = connection.prepareStatement(query)) { + boolean isResultSet = stmt.execute(); + assertFalse(isResultSet); + int updated = stmt.getUpdateCount(); + assertEquals(42, updated); + assertFalse(stmt.getMoreResults()); + assertEquals(-1, stmt.getUpdateCount()); + } + } + @Test public void testUpdateQueryWithParameters() throws SQLException { String query = "Fake update with parameters"; From c21b8a731d2d700a7edd28f8d7dba2983cbd4dae Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 8 Sep 2025 15:50:11 +0900 Subject: [PATCH 196/408] GH-841: Use apache/arrow-dotnet for integration test (#842) ## What's Changed Use apache/arrow-dotnet for the .NET implementation. Use `@vX` for `actions/*` because Dependabot sometimes doesn't work well for `@SHA512`. See also: https://github.com/apache/arrow-java/pull/820#discussion_r2283530810 Closes #841. --- .github/workflows/test.yml | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c461b3c19e..5d420d1f91 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,12 +58,12 @@ jobs: MAVEN: ${{ matrix.maven }} steps: - name: Checkout Arrow - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + uses: actions/checkout@v5 with: fetch-depth: 0 submodules: recursive - name: Cache Docker Volumes - uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + uses: actions/cache@v4 with: path: .docker key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('compose.yaml', '**/pom.xml', '**/*.java') }} @@ -95,12 +95,12 @@ jobs: macos: latest steps: - name: Set up Java - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: 'temurin' java-version: ${{ matrix.jdk }} - name: Checkout Arrow - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: fetch-depth: 0 submodules: recursive @@ -126,12 +126,12 @@ jobs: jdk: [11] steps: - name: Set up Java - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: ${{ matrix.jdk }} distribution: 'temurin' - name: Checkout Arrow - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: fetch-depth: 0 submodules: recursive @@ -152,32 +152,37 @@ jobs: timeout-minutes: 60 steps: - name: Checkout Arrow - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: fetch-depth: 0 repository: apache/arrow submodules: recursive - name: Checkout Arrow Rust - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: repository: apache/arrow-rs path: rust - name: Checkout Arrow nanoarrow - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: repository: apache/arrow-nanoarrow path: nanoarrow + - name: Checkout Arrow .NET + uses: actions/checkout@v5 + with: + repository: apache/arrow-dotnet + path: dotnet - name: Checkout Arrow Go - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: repository: apache/arrow-go path: go - name: Checkout Arrow Java - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: path: java - name: Checkout Arrow JavaScript - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: repository: apache/arrow-js path: js @@ -185,13 +190,13 @@ jobs: run: | ci/scripts/util_free_space.sh - name: Cache Docker Volumes - uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + uses: actions/cache@v4 with: path: .docker key: integration-conda-${{ hashFiles('cpp/**') }} restore-keys: integration-conda- - name: Setup Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + uses: actions/setup-python@v5 with: python-version: 3.12 - name: Setup Archery @@ -202,6 +207,7 @@ jobs: archery docker run \ -e ARCHERY_DEFAULT_BRANCH=main \ -e ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS=java \ + -e ARCHERY_INTEGRATION_WITH_DOTNET=1 \ -e ARCHERY_INTEGRATION_WITH_GO=1 \ -e ARCHERY_INTEGRATION_WITH_JAVA=1 \ -e ARCHERY_INTEGRATION_WITH_JS=1 \ From d51f300df79acf408c29dae68c6184835ecf91a0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Sep 2025 10:08:02 +0900 Subject: [PATCH 197/408] MINOR: [CI] Bump actions/setup-java from 4.6.0 to 5.0.0 (#847) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/setup-java](https://github.com/actions/setup-java) from 4.6.0 to 5.0.0.
Release notes

Sourced from actions/setup-java's releases.

v5.0.0

What's Changed

Breaking Changes

Make sure your runner is updated to this version or newer to use this release. v2.327.1 Release Notes

Dependency Upgrades

Bug Fixes

New Contributors

Full Changelog: https://github.com/actions/setup-java/compare/v4...v5.0.0

v4.7.1

What's Changed

Documentation changes

Dependency updates:

Full Changelog: https://github.com/actions/setup-java/compare/v4...v4.7.1

v4.7.0

What's Changed

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/setup-java&package-manager=github_actions&previous-version=4.6.0&new-version=5.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sutou Kouhei --- .github/workflows/rc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 9a8d7d9563..25c31da84a 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -321,7 +321,7 @@ jobs: repository: apache/arrow path: arrow - name: Set up Java - uses: actions/setup-java@7a6d8a8234af8eb26422e24e3006232cccaa061b # v4.6.0 + uses: actions/setup-java@v5 with: java-version: '11' distribution: 'temurin' From 8dd5ed8ab1b9655dbbb4fe315291e9d688f10b5b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Sep 2025 10:08:53 +0900 Subject: [PATCH 198/408] MINOR: [CI] Bump actions/setup-python from 5 to 6 (#843) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5 to 6.
Release notes

Sourced from actions/setup-python's releases.

v6.0.0

What's Changed

Breaking Changes

Make sure your runner is on version v2.327.1 or later to ensure compatibility with this release. See Release Notes

Enhancements:

Bug fixes:

Dependency updates:

New Contributors

Full Changelog: https://github.com/actions/setup-python/compare/v5...v6.0.0

v5.6.0

What's Changed

Full Changelog: https://github.com/actions/setup-python/compare/v5...v5.6.0

v5.5.0

What's Changed

Enhancements:

Bug fixes:

... (truncated)

Commits
  • e797f83 Upgrade to node 24 (#1164)
  • 3d1e2d2 Revert "Enhance cache-dependency-path handling to support files outside the w...
  • 65b0712 Clarify pythonLocation behavior for PyPy and GraalPy in environment variables...
  • 5b668cf Bump actions/checkout from 4 to 5 (#1181)
  • f62a0e2 Change missing cache directory error to warning (#1182)
  • 9322b3c Upgrade setuptools to 78.1.1 to fix path traversal vulnerability in PackageIn...
  • fbeb884 Bump form-data to fix critical vulnerabilities #182 & #183 (#1163)
  • 03bb615 Bump idna from 2.9 to 3.7 in /tests/data (#843)
  • 36da51d Add version parsing from Pipfile (#1067)
  • 3c6f142 update documentation (#1156)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/setup-python&package-manager=github_actions&previous-version=5&new-version=6)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sutou Kouhei --- .github/workflows/dev.yml | 2 +- .github/workflows/rc.yml | 4 ++-- .github/workflows/test.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 4242af55c3..12968bbcca 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -38,7 +38,7 @@ jobs: fetch-depth: 0 persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: '3.x' - name: pre-commit (cache) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 25c31da84a..eec3344097 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -201,7 +201,7 @@ jobs: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing - name: Set up Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + uses: actions/setup-python@v6 with: cache: 'pip' python-version: 3.12 @@ -452,7 +452,7 @@ jobs: contents: read packages: write steps: - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + - uses: actions/setup-python@v6 with: cache: 'pip' - name: Download source archive diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5d420d1f91..65fbc262fc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -196,7 +196,7 @@ jobs: key: integration-conda-${{ hashFiles('cpp/**') }} restore-keys: integration-conda- - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: 3.12 - name: Setup Archery From e026f3c29832577ee4f55dff8f942d7cab8301ae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Sep 2025 10:09:27 +0900 Subject: [PATCH 199/408] MINOR: [CI] Bump actions/github-script from 7.0.1 to 8.0.0 (#844) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/github-script](https://github.com/actions/github-script) from 7.0.1 to 8.0.0.
Release notes

Sourced from actions/github-script's releases.

v8.0.0

What's Changed

⚠️ Minimum Compatible Runner Version

v2.327.1
Release Notes

Make sure your runner is updated to this version or newer to use this release.

New Contributors

Full Changelog: https://github.com/actions/github-script/compare/v7.1.0...v8.0.0

v7.1.0

What's Changed

New Contributors

Full Changelog: https://github.com/actions/github-script/compare/v7...v7.1.0

Commits
  • ed59741 Merge pull request #653 from actions/sneha-krip/readme-for-v8
  • 2dc352e Bold minimum Actions Runner version in README
  • 01e118c Update README for Node 24 runtime requirements
  • 8b222ac Apply suggestion from @​salmanmkc
  • adc0eea README for updating actions/github-script from v7 to v8
  • 20fe497 Merge pull request #637 from actions/node24
  • e7b7f22 update licenses
  • 2c81ba0 Update Node.js version support to 24.x
  • f28e40c Merge pull request #610 from actions/nebuk89-patch-1
  • 1ae9958 Update README.md
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/github-script&package-manager=github_actions&previous-version=7.0.1&new-version=8.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sutou Kouhei --- .github/workflows/comment_bot.yml | 2 +- .github/workflows/dev_pr.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml index 5fbc858cc6..b4dbc92dfb 100644 --- a/.github/workflows/comment_bot.yml +++ b/.github/workflows/comment_bot.yml @@ -30,7 +30,7 @@ jobs: if: github.event.comment.body == 'take' runs-on: ubuntu-latest steps: - - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + - uses: actions/github-script@v8 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: |- diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml index 7352137b09..58f5f3d30a 100644 --- a/.github/workflows/dev_pr.yml +++ b/.github/workflows/dev_pr.yml @@ -49,28 +49,28 @@ jobs: - name: Ensure PR title format id: title-format - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + uses: actions/github-script@v8 with: script: | const scripts = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr.js`); return scripts.check_title_format({core, github, context}); - name: Label PR - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + uses: actions/github-script@v8 with: script: | const scripts = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr.js`); await scripts.apply_labels({core, github, context}); - name: Ensure PR is labeled - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + uses: actions/github-script@v8 with: script: | const scripts = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr.js`); await scripts.check_labels({core, github, context}); - name: Ensure PR is linked to an issue - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + uses: actions/github-script@v8 with: script: | const scripts = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr.js`); From ec3a424736ce00e7469cb6d983061c335bf4853a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Giraldo?= Date: Thu, 25 Sep 2025 01:09:42 -0600 Subject: [PATCH 200/408] GH-858: Fix error handling in CompositeJdbcConsumer (#857) ## What's Changed Turns out not all MinorTypes have a corresponding ArrowType, which can cause the following exception while handling the original exception: ``` Caused by: java.lang.UnsupportedOperationException: Cannot get simple type for type DECIMAL at org.apache.arrow.vector.types.Types$MinorType.getType(Types.java:815) at org.apache.arrow.adapter.jdbc.consumer.CompositeJdbcConsumer.consume(CompositeJdbcConsumer.java:49) ``` This PR changes the value we store in the exception to be the `MinorType` instead so we can still get useful info about the type but avoiding this possible exception. Closes #858. --- .../jdbc/consumer/CompositeJdbcConsumer.java | 5 ++--- .../consumer/exceptions/JdbcConsumerException.java | 14 +++++++------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/CompositeJdbcConsumer.java b/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/CompositeJdbcConsumer.java index 2366116fd0..b8389ee27c 100644 --- a/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/CompositeJdbcConsumer.java +++ b/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/CompositeJdbcConsumer.java @@ -24,7 +24,6 @@ import org.apache.arrow.util.AutoCloseables; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.types.pojo.ArrowType; /** Composite consumer which hold all consumers. It manages the consume and cleanup process. */ public class CompositeJdbcConsumer implements JdbcConsumer { @@ -46,9 +45,9 @@ public void consume(ResultSet rs) throws SQLException, IOException { BaseConsumer consumer = (BaseConsumer) consumers[i]; JdbcFieldInfo fieldInfo = new JdbcFieldInfo(rs.getMetaData(), consumer.columnIndexInResultSet); - ArrowType arrowType = consumer.vector.getMinorType().getType(); + throw new JdbcConsumerException( - "Exception while consuming JDBC value", e, fieldInfo, arrowType); + "Exception while consuming JDBC value", e, fieldInfo, consumer.vector.getField()); } else { throw e; } diff --git a/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/exceptions/JdbcConsumerException.java b/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/exceptions/JdbcConsumerException.java index 04e26d640c..98927f416c 100644 --- a/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/exceptions/JdbcConsumerException.java +++ b/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/exceptions/JdbcConsumerException.java @@ -17,7 +17,7 @@ package org.apache.arrow.adapter.jdbc.consumer.exceptions; import org.apache.arrow.adapter.jdbc.JdbcFieldInfo; -import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; /** * Exception while consuming JDBC data. This exception stores the JdbcFieldInfo for the column and @@ -25,7 +25,7 @@ */ public class JdbcConsumerException extends RuntimeException { final JdbcFieldInfo fieldInfo; - final ArrowType arrowType; + final Field field; /** * Construct JdbcConsumerException with all fields. @@ -33,17 +33,17 @@ public class JdbcConsumerException extends RuntimeException { * @param message error message * @param cause original exception * @param fieldInfo JdbcFieldInfo for the column - * @param arrowType ArrowType for the corresponding vector + * @param field ArrowType for the corresponding vector */ public JdbcConsumerException( - String message, Throwable cause, JdbcFieldInfo fieldInfo, ArrowType arrowType) { + String message, Throwable cause, JdbcFieldInfo fieldInfo, Field field) { super(message, cause); this.fieldInfo = fieldInfo; - this.arrowType = arrowType; + this.field = field; } - public ArrowType getArrowType() { - return this.arrowType; + public Field getField() { + return this.field; } public JdbcFieldInfo getFieldInfo() { From f38e72f5a46e30f44c28ff407eb522a5e5c266c0 Mon Sep 17 00:00:00 2001 From: Zhen Wang <643348094@qq.com> Date: Fri, 26 Sep 2025 09:21:33 +0800 Subject: [PATCH 201/408] GH-859: Fix ARROW_STRUCT_CONFLICT_POLICY env var (#860) ## What's Changed Do not specify default value when getting the `arrow.struct.conflict.policy` property Closes #859 --- .../arrow/vector/complex/AbstractStructVector.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/AbstractStructVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/AbstractStructVector.java index 2921e43cb6..a57fbe473f 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/AbstractStructVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/AbstractStructVector.java @@ -46,11 +46,13 @@ public abstract class AbstractStructVector extends AbstractContainerVector { private ConflictPolicy conflictPolicy; static { - String conflictPolicyStr = - System.getProperty(STRUCT_CONFLICT_POLICY_JVM, ConflictPolicy.CONFLICT_REPLACE.toString()); + String conflictPolicyStr = System.getProperty(STRUCT_CONFLICT_POLICY_JVM); if (conflictPolicyStr == null) { conflictPolicyStr = System.getenv(STRUCT_CONFLICT_POLICY_ENV); } + if (conflictPolicyStr == null) { + conflictPolicyStr = ConflictPolicy.CONFLICT_REPLACE.toString(); + } ConflictPolicy conflictPolicy; try { conflictPolicy = ConflictPolicy.valueOf(conflictPolicyStr.toUpperCase(Locale.ROOT)); @@ -62,11 +64,11 @@ public abstract class AbstractStructVector extends AbstractContainerVector { /** Policy to determine how to react when duplicate columns are encountered. */ public enum ConflictPolicy { - // Ignore the conflict and append the field. This is the default behaviour + // Ignore the conflict and append the field. CONFLICT_APPEND, // Keep the existing field and ignore the newer one. CONFLICT_IGNORE, - // Replace the existing field with the newer one. + // Replace the existing field with the newer one. This is the default behaviour CONFLICT_REPLACE, // Refuse the new field and error out. CONFLICT_ERROR From 34060eb491a870f5ede5d30e007060b8310dc64f Mon Sep 17 00:00:00 2001 From: Ivan Chesnov Date: Wed, 1 Oct 2025 15:43:26 +0300 Subject: [PATCH 202/408] GH-836: Added support of ExtensionType for ComplexCopier (#837) ## What's Changed Updated ComplexCopier to support ExtensionType - it contains two **copy** methods ``` public static void copy(FieldReader input, FieldWriter output) //for not breaking existing logic public static void copy(FieldReader input, FieldWriter output, ExtensionTypeWriterFactory extensionTypeWriterFactory) ``` Also updated ComplexCopier tests. Closes #836. --- .../src/main/codegen/includes/vv_imports.ftl | 1 + .../templates/AbstractFieldReader.java | 4 + .../main/codegen/templates/BaseReader.java | 3 + .../main/codegen/templates/ComplexCopier.java | 39 +++++- .../main/codegen/templates/NullReader.java | 1 + .../apache/arrow/vector/BaseValueVector.java | 13 ++ .../org/apache/arrow/vector/NullVector.java | 13 ++ .../org/apache/arrow/vector/ValueVector.java | 25 ++++ .../complex/AbstractContainerVector.java | 13 ++ .../arrow/vector/complex/LargeListVector.java | 33 ++++- .../vector/complex/LargeListViewVector.java | 15 +++ .../arrow/vector/complex/ListVector.java | 33 ++++- .../arrow/vector/complex/ListViewVector.java | 15 ++- .../complex/impl/AbstractBaseReader.java | 10 ++ .../complex/impl/UnionExtensionWriter.java | 5 + .../complex/impl/UnionLargeListReader.java | 4 + .../apache/arrow/vector/TestListVector.java | 43 +++++++ .../apache/arrow/vector/TestMapVector.java | 96 +++++++++++++++ .../complex/impl/TestComplexCopier.java | 114 ++++++++++++++++++ .../vector/complex/impl/UuidReaderImpl.java | 5 + 20 files changed, 476 insertions(+), 9 deletions(-) diff --git a/vector/src/main/codegen/includes/vv_imports.ftl b/vector/src/main/codegen/includes/vv_imports.ftl index 7f216a7b43..2bbcecc856 100644 --- a/vector/src/main/codegen/includes/vv_imports.ftl +++ b/vector/src/main/codegen/includes/vv_imports.ftl @@ -34,6 +34,7 @@ import org.apache.arrow.vector.complex.*; import org.apache.arrow.vector.complex.reader.*; import org.apache.arrow.vector.complex.impl.*; import org.apache.arrow.vector.complex.writer.*; +import org.apache.arrow.vector.complex.writer.BaseWriter.ExtensionWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.StructWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; diff --git a/vector/src/main/codegen/templates/AbstractFieldReader.java b/vector/src/main/codegen/templates/AbstractFieldReader.java index 7e84323b64..c7c5b4d78d 100644 --- a/vector/src/main/codegen/templates/AbstractFieldReader.java +++ b/vector/src/main/codegen/templates/AbstractFieldReader.java @@ -109,6 +109,10 @@ public void copyAsField(String name, ${name}Writer writer) { + public void copyAsValue(StructWriter writer, ExtensionTypeWriterFactory writerFactory) { + fail("CopyAsValue StructWriter"); + } + public void read(ExtensionHolder holder) { fail("Extension"); } diff --git a/vector/src/main/codegen/templates/BaseReader.java b/vector/src/main/codegen/templates/BaseReader.java index c52345af21..4c6f49ab9b 100644 --- a/vector/src/main/codegen/templates/BaseReader.java +++ b/vector/src/main/codegen/templates/BaseReader.java @@ -49,6 +49,7 @@ public interface RepeatedStructReader extends StructReader{ boolean next(); int size(); void copyAsValue(StructWriter writer); + void copyAsValue(StructWriter writer, ExtensionTypeWriterFactory writerFactory); } public interface ListReader extends BaseReader{ @@ -59,6 +60,7 @@ public interface RepeatedListReader extends ListReader{ boolean next(); int size(); void copyAsValue(ListWriter writer); + void copyAsValue(ListWriter writer, ExtensionTypeWriterFactory writerFactory); } public interface MapReader extends BaseReader{ @@ -69,6 +71,7 @@ public interface RepeatedMapReader extends MapReader{ boolean next(); int size(); void copyAsValue(MapWriter writer); + void copyAsValue(MapWriter writer, ExtensionTypeWriterFactory writerFactory); } public interface ScalarReader extends diff --git a/vector/src/main/codegen/templates/ComplexCopier.java b/vector/src/main/codegen/templates/ComplexCopier.java index 4fff7059a7..4df5478f48 100644 --- a/vector/src/main/codegen/templates/ComplexCopier.java +++ b/vector/src/main/codegen/templates/ComplexCopier.java @@ -42,10 +42,14 @@ public class ComplexCopier { * @param output field to write to */ public static void copy(FieldReader input, FieldWriter output) { - writeValue(input, output); + writeValue(input, output, null); } - private static void writeValue(FieldReader reader, FieldWriter writer) { + public static void copy(FieldReader input, FieldWriter output, ExtensionTypeWriterFactory extensionTypeWriterFactory) { + writeValue(input, output, extensionTypeWriterFactory); + } + + private static void writeValue(FieldReader reader, FieldWriter writer, ExtensionTypeWriterFactory extensionTypeWriterFactory) { final MinorType mt = reader.getMinorType(); switch (mt) { @@ -61,7 +65,7 @@ private static void writeValue(FieldReader reader, FieldWriter writer) { FieldReader childReader = reader.reader(); FieldWriter childWriter = getListWriterForReader(childReader, writer); if (childReader.isSet()) { - writeValue(childReader, childWriter); + writeValue(childReader, childWriter, extensionTypeWriterFactory); } else { childWriter.writeNull(); } @@ -79,8 +83,8 @@ private static void writeValue(FieldReader reader, FieldWriter writer) { FieldReader structReader = reader.reader(); if (structReader.isSet()) { writer.startEntry(); - writeValue(mapReader.key(), getMapWriterForReader(mapReader.key(), writer.key())); - writeValue(mapReader.value(), getMapWriterForReader(mapReader.value(), writer.value())); + writeValue(mapReader.key(), getMapWriterForReader(mapReader.key(), writer.key()), extensionTypeWriterFactory); + writeValue(mapReader.value(), getMapWriterForReader(mapReader.value(), writer.value()), extensionTypeWriterFactory); writer.endEntry(); } else { writer.writeNull(); @@ -99,7 +103,7 @@ private static void writeValue(FieldReader reader, FieldWriter writer) { if (childReader.getMinorType() != Types.MinorType.NULL) { FieldWriter childWriter = getStructWriterForReader(childReader, writer, name); if (childReader.isSet()) { - writeValue(childReader, childWriter); + writeValue(childReader, childWriter, extensionTypeWriterFactory); } else { childWriter.writeNull(); } @@ -110,6 +114,20 @@ private static void writeValue(FieldReader reader, FieldWriter writer) { writer.writeNull(); } break; + case EXTENSIONTYPE: + if (extensionTypeWriterFactory == null) { + throw new IllegalArgumentException("Must provide ExtensionTypeWriterFactory"); + } + if (reader.isSet()) { + Object value = reader.readObject(); + if (value != null) { + writer.addExtensionTypeWriterFactory(extensionTypeWriterFactory); + writer.writeExtension(value); + } + } else { + writer.writeNull(); + } + break; <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> @@ -162,6 +180,9 @@ private static FieldWriter getStructWriterForReader(FieldReader reader, StructWr return (FieldWriter) writer.map(name); case LISTVIEW: return (FieldWriter) writer.listView(name); + case EXTENSIONTYPE: + ExtensionWriter extensionWriter = writer.extension(name, reader.getField().getType()); + return (FieldWriter) extensionWriter; default: throw new UnsupportedOperationException(reader.getMinorType().toString()); } @@ -186,6 +207,9 @@ private static FieldWriter getListWriterForReader(FieldReader reader, ListWriter return (FieldWriter) writer.list(); case LISTVIEW: return (FieldWriter) writer.listView(); + case EXTENSIONTYPE: + ExtensionWriter extensionWriter = writer.extension(reader.getField().getType()); + return (FieldWriter) extensionWriter; default: throw new UnsupportedOperationException(reader.getMinorType().toString()); } @@ -211,6 +235,9 @@ private static FieldWriter getMapWriterForReader(FieldReader reader, MapWriter w return (FieldWriter) writer.listView(); case MAP: return (FieldWriter) writer.map(false); + case EXTENSIONTYPE: + ExtensionWriter extensionWriter = writer.extension(reader.getField().getType()); + return (FieldWriter) extensionWriter; default: throw new UnsupportedOperationException(reader.getMinorType().toString()); } diff --git a/vector/src/main/codegen/templates/NullReader.java b/vector/src/main/codegen/templates/NullReader.java index 88e6ea98ea..0529633478 100644 --- a/vector/src/main/codegen/templates/NullReader.java +++ b/vector/src/main/codegen/templates/NullReader.java @@ -86,6 +86,7 @@ public void read(int arrayIndex, Nullable${name}Holder holder){ } + public void copyAsValue(StructWriter writer, ExtensionTypeWriterFactory writerFactory){} public void read(ExtensionHolder holder) { holder.isSet = 0; } diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java index 37dfa20616..cc57cde29e 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java @@ -22,6 +22,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.ReferenceManager; import org.apache.arrow.util.Preconditions; +import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.util.DataSizeRoundingUtil; import org.apache.arrow.vector.util.TransferPair; @@ -260,6 +261,18 @@ public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { throw new UnsupportedOperationException(); } + @Override + public void copyFrom( + int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { + throw new UnsupportedOperationException(); + } + + @Override + public void copyFromSafe( + int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { + throw new UnsupportedOperationException(); + } + /** * Transfer the validity buffer from `validityBuffer` to the target vector's `validityBuffer`. * Start at `startIndex` and copy `length` number of elements. If the starting index is 8 byte diff --git a/vector/src/main/java/org/apache/arrow/vector/NullVector.java b/vector/src/main/java/org/apache/arrow/vector/NullVector.java index 6bfe540d23..0d6dab2837 100644 --- a/vector/src/main/java/org/apache/arrow/vector/NullVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/NullVector.java @@ -27,6 +27,7 @@ import org.apache.arrow.memory.util.hash.ArrowBufHasher; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.compare.VectorVisitor; +import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.impl.NullReader; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.ipc.message.ArrowFieldNode; @@ -329,6 +330,18 @@ public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { throw new UnsupportedOperationException(); } + @Override + public void copyFrom( + int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { + throw new UnsupportedOperationException(); + } + + @Override + public void copyFromSafe( + int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { + throw new UnsupportedOperationException(); + } + @Override public String getName() { return this.getField().getName(); diff --git a/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/vector/src/main/java/org/apache/arrow/vector/ValueVector.java index 3a5058256c..e0628c2ee1 100644 --- a/vector/src/main/java/org/apache/arrow/vector/ValueVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/ValueVector.java @@ -22,6 +22,7 @@ import org.apache.arrow.memory.OutOfMemoryException; import org.apache.arrow.memory.util.hash.ArrowBufHasher; import org.apache.arrow.vector.compare.VectorVisitor; +import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.Field; @@ -309,6 +310,30 @@ public interface ValueVector extends Closeable, Iterable { */ void copyFromSafe(int fromIndex, int thisIndex, ValueVector from); + /** + * Copy a cell value from a particular index in source vector to a particular position in this + * vector. + * + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + * @param writerFactory the extension type writer factory to use for copying extension type values + */ + void copyFrom( + int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory); + + /** + * Same as {@link #copyFrom(int, int, ValueVector)} except that it handles the case when the + * capacity of the vector needs to be expanded before copy. + * + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + * @param writerFactory the extension type writer factory to use for copying extension type values + */ + void copyFromSafe( + int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory); + /** * Accept a generic {@link VectorVisitor} and return the result. * diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java index a6a71cf1a4..429f9884bb 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java @@ -21,6 +21,7 @@ import org.apache.arrow.vector.DensityAwareVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeList; @@ -151,6 +152,18 @@ public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { throw new UnsupportedOperationException(); } + @Override + public void copyFrom( + int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { + throw new UnsupportedOperationException(); + } + + @Override + public void copyFromSafe( + int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { + throw new UnsupportedOperationException(); + } + @Override public String getName() { return name; diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java index 997b5a8b78..48c8127e23 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java @@ -49,6 +49,7 @@ import org.apache.arrow.vector.ZeroVector; import org.apache.arrow.vector.compare.VectorVisitor; import org.apache.arrow.vector.complex.impl.ComplexCopier; +import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.impl.UnionLargeListReader; import org.apache.arrow.vector.complex.impl.UnionLargeListWriter; import org.apache.arrow.vector.complex.reader.FieldReader; @@ -482,12 +483,42 @@ public void copyFromSafe(int inIndex, int outIndex, ValueVector from) { */ @Override public void copyFrom(int inIndex, int outIndex, ValueVector from) { + copyFrom(inIndex, outIndex, from, null); + } + + /** + * Copy a cell value from a particular index in source vector to a particular position in this + * vector. + * + * @param inIndex position to copy from in source vector + * @param outIndex position to copy to in this vector + * @param from source vector + * @param writerFactory the extension type writer factory to use for copying extension type values + */ + @Override + public void copyFrom( + int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { Preconditions.checkArgument(this.getMinorType() == from.getMinorType()); FieldReader in = from.getReader(); in.setPosition(inIndex); UnionLargeListWriter out = getWriter(); out.setPosition(outIndex); - ComplexCopier.copy(in, out); + ComplexCopier.copy(in, out, writerFactory); + } + + /** + * Same as {@link #copyFrom(int, int, ValueVector)} except that it handles the case when the + * capacity of the vector needs to be expanded before copy. + * + * @param inIndex position to copy from in source vector + * @param outIndex position to copy to in this vector + * @param from source vector + * @param writerFactory the extension type writer factory to use for copying extension type values + */ + @Override + public void copyFromSafe( + int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { + copyFrom(inIndex, outIndex, from, writerFactory); } /** diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java index 2da7eb057e..992a664449 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java @@ -41,6 +41,7 @@ import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.ZeroVector; import org.apache.arrow.vector.compare.VectorVisitor; +import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.impl.UnionLargeListViewReader; import org.apache.arrow.vector.complex.impl.UnionLargeListViewWriter; import org.apache.arrow.vector.complex.impl.UnionListReader; @@ -346,6 +347,20 @@ public void copyFrom(int inIndex, int outIndex, ValueVector from) { "LargeListViewVector does not support copyFrom operation yet."); } + @Override + public void copyFromSafe( + int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { + throw new UnsupportedOperationException( + "LargeListViewVector does not support copyFromSafe operation yet."); + } + + @Override + public void copyFrom( + int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { + throw new UnsupportedOperationException( + "LargeListViewVector does not support copyFrom operation yet."); + } + @Override public FieldVector getDataVector() { return vector; diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 93a313ef4f..89549257c4 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -42,6 +42,7 @@ import org.apache.arrow.vector.ZeroVector; import org.apache.arrow.vector.compare.VectorVisitor; import org.apache.arrow.vector.complex.impl.ComplexCopier; +import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.impl.UnionListReader; import org.apache.arrow.vector.complex.impl.UnionListWriter; import org.apache.arrow.vector.complex.reader.FieldReader; @@ -400,12 +401,42 @@ public void copyFromSafe(int inIndex, int outIndex, ValueVector from) { */ @Override public void copyFrom(int inIndex, int outIndex, ValueVector from) { + copyFrom(inIndex, outIndex, from, null); + } + + /** + * Same as {@link #copyFrom(int, int, ValueVector)} except that it handles the case when the + * capacity of the vector needs to be expanded before copy. + * + * @param inIndex position to copy from in source vector + * @param outIndex position to copy to in this vector + * @param from source vector + * @param writerFactory the extension type writer factory to use for copying extension type values + */ + @Override + public void copyFromSafe( + int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { + copyFrom(inIndex, outIndex, from, writerFactory); + } + + /** + * Copy a cell value from a particular index in source vector to a particular position in this + * vector. + * + * @param inIndex position to copy from in source vector + * @param outIndex position to copy to in this vector + * @param from source vector + * @param writerFactory the extension type writer factory to use for copying extension type values + */ + @Override + public void copyFrom( + int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { Preconditions.checkArgument(this.getMinorType() == from.getMinorType()); FieldReader in = from.getReader(); in.setPosition(inIndex); FieldWriter out = getWriter(); out.setPosition(outIndex); - ComplexCopier.copy(in, out); + ComplexCopier.copy(in, out, writerFactory); } /** diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java index 8711db5e0f..2784240429 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java @@ -42,6 +42,7 @@ import org.apache.arrow.vector.ZeroVector; import org.apache.arrow.vector.compare.VectorVisitor; import org.apache.arrow.vector.complex.impl.ComplexCopier; +import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.impl.UnionListViewReader; import org.apache.arrow.vector.complex.impl.UnionListViewWriter; import org.apache.arrow.vector.complex.reader.FieldReader; @@ -338,6 +339,12 @@ public void copyFromSafe(int inIndex, int outIndex, ValueVector from) { copyFrom(inIndex, outIndex, from); } + @Override + public void copyFromSafe( + int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { + copyFrom(inIndex, outIndex, from, writerFactory); + } + @Override public OUT accept(VectorVisitor visitor, IN value) { return visitor.visit(this, value); @@ -345,12 +352,18 @@ public OUT accept(VectorVisitor visitor, IN value) { @Override public void copyFrom(int inIndex, int outIndex, ValueVector from) { + copyFrom(inIndex, outIndex, from, null); + } + + @Override + public void copyFrom( + int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { Preconditions.checkArgument(this.getMinorType() == from.getMinorType()); FieldReader in = from.getReader(); in.setPosition(inIndex); FieldWriter out = getWriter(); out.setPosition(outIndex); - ComplexCopier.copy(in, out); + ComplexCopier.copy(in, out, writerFactory); } @Override diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java index b2e95663f7..bf074ecb90 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java @@ -115,4 +115,14 @@ public void copyAsValue(ListWriter writer) { public void copyAsValue(MapWriter writer) { ComplexCopier.copy(this, (FieldWriter) writer); } + + @Override + public void copyAsValue(ListWriter writer, ExtensionTypeWriterFactory writerFactory) { + ComplexCopier.copy(this, (FieldWriter) writer, writerFactory); + } + + @Override + public void copyAsValue(MapWriter writer, ExtensionTypeWriterFactory writerFactory) { + ComplexCopier.copy(this, (FieldWriter) writer, writerFactory); + } } diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionExtensionWriter.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionExtensionWriter.java index d341384bd9..4219069cba 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionExtensionWriter.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionExtensionWriter.java @@ -76,4 +76,9 @@ public void setPosition(int index) { this.writer.setPosition(index); } } + + @Override + public void writeNull() { + this.writer.writeNull(); + } } diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionLargeListReader.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionLargeListReader.java index be236c3166..a9104cb0d2 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionLargeListReader.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionLargeListReader.java @@ -105,4 +105,8 @@ public boolean next() { public void copyAsValue(UnionLargeListWriter writer) { ComplexCopier.copy(this, (FieldWriter) writer); } + + public void copyAsValue(UnionLargeListWriter writer, ExtensionTypeWriterFactory writerFactory) { + ComplexCopier.copy(this, (FieldWriter) writer, writerFactory); + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index d58b7cc941..c6c7c5c862 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -1271,6 +1271,49 @@ public void testListVectorReaderForExtensionType() throws Exception { } } + @Test + public void testCopyFromForExtensionType() throws Exception { + try (ListVector inVector = ListVector.empty("input", allocator); + ListVector outVector = ListVector.empty("output", allocator)) { + UnionListWriter writer = inVector.getWriter(); + writer.allocate(); + writer.setPosition(0); + UUID u1 = UUID.randomUUID(); + UUID u2 = UUID.randomUUID(); + writer.startList(); + ExtensionWriter extensionWriter = writer.extension(new UuidType()); + extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); + extensionWriter.writeExtension(u1); + extensionWriter.writeExtension(u2); + extensionWriter.writeNull(); + writer.endList(); + + writer.setValueCount(1); + + // copy values from input to output + outVector.allocateNew(); + outVector.copyFrom(0, 0, inVector, new UuidWriterFactory()); + outVector.setValueCount(1); + + UnionListReader reader = outVector.getReader(); + assertTrue(reader.isSet(), "shouldn't be null"); + reader.setPosition(0); + reader.next(); + FieldReader uuidReader = reader.reader(); + UuidHolder holder = new UuidHolder(); + uuidReader.read(holder); + ByteBuffer bb = ByteBuffer.wrap(holder.value); + UUID actualUuid = new UUID(bb.getLong(), bb.getLong()); + assertEquals(u1, actualUuid); + reader.next(); + uuidReader = reader.reader(); + uuidReader.read(holder); + bb = ByteBuffer.wrap(holder.value); + actualUuid = new UUID(bb.getLong(), bb.getLong()); + assertEquals(u2, actualUuid); + } + } + private void writeIntValues(UnionListWriter writer, int[] values) { writer.startList(); for (int v : values) { diff --git a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java index 313d83ec91..1a1810d0f7 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java @@ -22,24 +22,30 @@ import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.UUID; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.complex.impl.UnionMapReader; import org.apache.arrow.vector.complex.impl.UnionMapWriter; +import org.apache.arrow.vector.complex.impl.UuidWriterFactory; import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.complex.writer.BaseWriter.ExtensionWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; import org.apache.arrow.vector.complex.writer.FieldWriter; +import org.apache.arrow.vector.holder.UuidHolder; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.UuidType; import org.apache.arrow.vector.util.JsonStringArrayList; import org.apache.arrow.vector.util.TransferPair; import org.junit.jupiter.api.AfterEach; @@ -1263,4 +1269,94 @@ public void testMapTypeReturnsSupportedMapWriter() { assertEquals(11, getResultValue(resultStruct)); } } + + @Test + public void testMapVectorWithExtensionType() throws Exception { + try (final MapVector inVector = MapVector.empty("map", allocator, false)) { + inVector.allocateNew(); + UnionMapWriter writer = inVector.getWriter(); + writer.setPosition(0); + UUID u1 = UUID.randomUUID(); + UUID u2 = UUID.randomUUID(); + writer.startMap(); + writer.startEntry(); + writer.key().bigInt().writeBigInt(0); + ExtensionWriter extensionWriter = writer.value().extension(new UuidType()); + extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); + extensionWriter.writeExtension(u1); + writer.endEntry(); + writer.startEntry(); + writer.key().bigInt().writeBigInt(1); + extensionWriter = writer.value().extension(new UuidType()); + extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); + extensionWriter.writeExtension(u2); + writer.endEntry(); + writer.endMap(); + + writer.setValueCount(1); + + UnionMapReader mapReader = inVector.getReader(); + mapReader.setPosition(0); + mapReader.next(); + FieldReader uuidReader = mapReader.value(); + UuidHolder holder = new UuidHolder(); + uuidReader.read(holder); + ByteBuffer bb = ByteBuffer.wrap(holder.value); + UUID actualUuid = new UUID(bb.getLong(), bb.getLong()); + assertEquals(u1, actualUuid); + mapReader.next(); + uuidReader = mapReader.value(); + uuidReader.read(holder); + bb = ByteBuffer.wrap(holder.value); + actualUuid = new UUID(bb.getLong(), bb.getLong()); + assertEquals(u2, actualUuid); + } + } + + @Test + public void testCopyFromForExtensionType() throws Exception { + try (final MapVector inVector = MapVector.empty("in", allocator, false); + final MapVector outVector = MapVector.empty("out", allocator, false)) { + inVector.allocateNew(); + UnionMapWriter writer = inVector.getWriter(); + writer.setPosition(0); + UUID u1 = UUID.randomUUID(); + UUID u2 = UUID.randomUUID(); + writer.startMap(); + writer.startEntry(); + writer.key().bigInt().writeBigInt(0); + ExtensionWriter extensionWriter = writer.value().extension(new UuidType()); + extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); + extensionWriter.writeExtension(u1); + writer.endEntry(); + writer.startEntry(); + writer.key().bigInt().writeBigInt(1); + extensionWriter = writer.value().extension(new UuidType()); + extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); + extensionWriter.writeExtension(u2); + writer.endEntry(); + writer.endMap(); + + writer.setValueCount(1); + outVector.allocateNew(); + outVector.copyFrom(0, 0, inVector, new UuidWriterFactory()); + outVector.setValueCount(1); + + UnionMapReader mapReader = outVector.getReader(); + mapReader.setPosition(0); + mapReader.next(); + FieldReader uuidReader = mapReader.value(); + UuidHolder holder = new UuidHolder(); + uuidReader.read(holder); + ByteBuffer bb = ByteBuffer.wrap(holder.value); + UUID actualUuid = new UUID(bb.getLong(), bb.getLong()); + assertEquals(u1, actualUuid); + mapReader.next(); + uuidReader = mapReader.value(); + uuidReader.read(holder); + bb = ByteBuffer.wrap(holder.value); + actualUuid = new UUID(bb.getLong(), bb.getLong()); + assertEquals(u2, actualUuid); + } + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java index 3bc02c6029..738e8905e3 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java @@ -20,6 +20,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.math.BigDecimal; +import java.util.UUID; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.DecimalVector; @@ -30,12 +31,14 @@ import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.complex.writer.BaseWriter; +import org.apache.arrow.vector.complex.writer.BaseWriter.ExtensionWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.StructWriter; import org.apache.arrow.vector.complex.writer.FieldWriter; import org.apache.arrow.vector.holders.DecimalHolder; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.UuidType; import org.apache.arrow.vector.util.DecimalUtility; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -845,4 +848,115 @@ public void testCopyMapVectorWithMapValue() { assertTrue(VectorEqualsVisitor.vectorEquals(from, to)); } } + + @Test + public void testCopyListVectorWithExtensionType() { + try (ListVector from = ListVector.empty("v", allocator); + ListVector to = ListVector.empty("v", allocator)) { + + UnionListWriter listWriter = from.getWriter(); + listWriter.allocate(); + + for (int i = 0; i < COUNT; i++) { + listWriter.setPosition(i); + listWriter.startList(); + ExtensionWriter extensionWriter = listWriter.extension(new UuidType()); + extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); + extensionWriter.writeExtension(UUID.randomUUID()); + extensionWriter.writeExtension(UUID.randomUUID()); + listWriter.endList(); + } + from.setValueCount(COUNT); + + // copy values + FieldReader in = from.getReader(); + FieldWriter out = to.getWriter(); + for (int i = 0; i < COUNT; i++) { + in.setPosition(i); + out.setPosition(i); + ComplexCopier.copy(in, out, new UuidWriterFactory()); + } + + to.setValueCount(COUNT); + + // validate equals + assertTrue(VectorEqualsVisitor.vectorEquals(from, to)); + } + } + + @Test + public void testCopyMapVectorWithExtensionType() { + try (final MapVector from = MapVector.empty("v", allocator, false); + final MapVector to = MapVector.empty("v", allocator, false)) { + + from.allocateNew(); + + UnionMapWriter mapWriter = from.getWriter(); + for (int i = 0; i < COUNT; i++) { + mapWriter.setPosition(i); + mapWriter.startMap(); + mapWriter.startEntry(); + ExtensionWriter extensionKeyWriter = mapWriter.key().extension(new UuidType()); + extensionKeyWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); + extensionKeyWriter.writeExtension(UUID.randomUUID()); + ExtensionWriter extensionValueWriter = mapWriter.value().extension(new UuidType()); + extensionValueWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); + extensionValueWriter.writeExtension(UUID.randomUUID()); + mapWriter.endEntry(); + mapWriter.endMap(); + } + + from.setValueCount(COUNT); + + // copy values + FieldReader in = from.getReader(); + FieldWriter out = to.getWriter(); + for (int i = 0; i < COUNT; i++) { + in.setPosition(i); + out.setPosition(i); + ComplexCopier.copy(in, out, new UuidWriterFactory()); + } + to.setValueCount(COUNT); + + // validate equals + assertTrue(VectorEqualsVisitor.vectorEquals(from, to)); + } + } + + @Test + public void testCopyStructVectorWithExtensionType() { + try (final StructVector from = StructVector.empty("v", allocator); + final StructVector to = StructVector.empty("v", allocator)) { + + from.allocateNewSafe(); + + NullableStructWriter structWriter = from.getWriter(); + for (int i = 0; i < COUNT; i++) { + structWriter.setPosition(i); + structWriter.start(); + ExtensionWriter extensionWriter1 = structWriter.extension("timestamp1", new UuidType()); + extensionWriter1.addExtensionTypeWriterFactory(new UuidWriterFactory()); + extensionWriter1.writeExtension(UUID.randomUUID()); + ExtensionWriter extensionWriter2 = structWriter.extension("timestamp2", new UuidType()); + extensionWriter2.addExtensionTypeWriterFactory(new UuidWriterFactory()); + extensionWriter2.writeExtension(UUID.randomUUID()); + structWriter.end(); + } + + from.setValueCount(COUNT); + + // copy values + FieldReader in = from.getReader(); + FieldWriter out = to.getWriter(); + for (int i = 0; i < COUNT; i++) { + in.setPosition(i); + out.setPosition(i); + ComplexCopier.copy(in, out, new UuidWriterFactory()); + } + to.setValueCount(COUNT); + + // validate equals + assertTrue(VectorEqualsVisitor.vectorEquals(from, to)); + } + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java index 16dd734de8..6b98d3b340 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java @@ -61,4 +61,9 @@ public void copyAsValue(AbstractExtensionTypeWriter writer) { UuidWriterImpl impl = (UuidWriterImpl) writer; impl.vector.copyFromSafe(idx(), impl.idx(), vector); } + + @Override + public Object readObject() { + return vector.getObject(idx()); + } } From 9cfa6ffc314e148afd196a89f2d5b12ec35c1bc2 Mon Sep 17 00:00:00 2001 From: XenoAmess Date: Fri, 3 Oct 2025 16:06:40 +0800 Subject: [PATCH 203/408] GH-848: TypedValue should be treated as Nullable in bind function in AvaticaParameterBinder (#849) ## What's Changed Closes #848. --- .../arrow/driver/jdbc/utils/AvaticaParameterBinder.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java index 4c2a9b865f..0fd99de539 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java @@ -44,6 +44,7 @@ import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.calcite.avatica.remote.TypedValue; +import org.checkerframework.checker.nullness.qual.Nullable; /** * Convert Avatica PreparedStatement parameters from a list of TypedValue to Arrow and bind them to @@ -108,9 +109,9 @@ public void bind(List typedValues, int index) { * @param typedValue TypedValue to bind to the vector. * @param index Vector index to bind the value at. */ - private void bind(FieldVector vector, TypedValue typedValue, int index) { + private void bind(FieldVector vector, @Nullable TypedValue typedValue, int index) { try { - if (typedValue.value == null) { + if (typedValue == null || typedValue.value == null) { if (vector.getField().isNullable()) { vector.setNull(index); } else { @@ -127,7 +128,7 @@ private void bind(FieldVector vector, TypedValue typedValue, int index) { throw new UnsupportedOperationException( String.format( "Binding value of type %s is not yet supported for expected Arrow type %s", - typedValue.type, vector.getField().getType())); + typedValue == null ? "null" : typedValue.type, vector.getField().getType())); } } From d529557087c601ccbb18d704081f8711f0df0cfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Milenkovi=C4=87?= Date: Fri, 3 Oct 2025 09:13:20 +0100 Subject: [PATCH 204/408] fix: issue with class names in arrow-c jni calls (#867) ## What's Changed Arrow C JNI code used class names which are not according to specification, making code unusable in latest graalvm 25. This PR changes class names according to JNI specification. More details at #866 Closes #866 --- c/src/main/cpp/jni_wrapper.cc | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/c/src/main/cpp/jni_wrapper.cc b/c/src/main/cpp/jni_wrapper.cc index 436cbdc806..3d7a194563 100644 --- a/c/src/main/cpp/jni_wrapper.cc +++ b/c/src/main/cpp/jni_wrapper.cc @@ -327,19 +327,20 @@ void ArrowArrayStreamRelease(ArrowArrayStream* stream) { jint JNI_OnLoad(JavaVM* vm, void* reserved) { JNIEnv* env; - if (vm->GetEnv(reinterpret_cast(&env), JNI_VERSION) != JNI_OK) { - return JNI_ERR; + const int err_code = vm->GetEnv(reinterpret_cast(&env), JNI_VERSION); + if (err_code != JNI_OK) { + return err_code; } JNI_METHOD_START - kObjectClass = CreateGlobalClassReference(env, "Ljava/lang/Object;"); + kObjectClass = CreateGlobalClassReference(env, "java/lang/Object"); kRuntimeExceptionClass = - CreateGlobalClassReference(env, "Ljava/lang/RuntimeException;"); + CreateGlobalClassReference(env, "java/lang/RuntimeException"); kPrivateDataClass = - CreateGlobalClassReference(env, "Lorg/apache/arrow/c/jni/PrivateData;"); + CreateGlobalClassReference(env, "org/apache/arrow/c/jni/PrivateData"); kCDataExceptionClass = - CreateGlobalClassReference(env, "Lorg/apache/arrow/c/jni/CDataJniException;"); + CreateGlobalClassReference(env, "org/apache/arrow/c/jni/CDataJniException"); kStreamPrivateDataClass = CreateGlobalClassReference( - env, "Lorg/apache/arrow/c/ArrayStreamExporter$ExportedArrayStreamPrivateData;"); + env, "org/apache/arrow/c/ArrayStreamExporter$ExportedArrayStreamPrivateData"); kPrivateDataLastErrorField = GetFieldID(env, kStreamPrivateDataClass, "lastError", "[B"); From d31d9b0cac6de22ba6649c38486281ffe6346294 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Giraldo?= Date: Mon, 6 Oct 2025 19:56:35 -0600 Subject: [PATCH 205/408] GH-839: Fix support for ResultSet.getObject for TIMESTAMP_WITH_TIMEZONE (#840) ## What's Changed Turns out AvaticaSite.get does not account for TIMESTAMP_WITH_TIMEZONE types so we add support on an override function. Closes #818. Closes #839. --- ...owFlightJdbcVectorSchemaRootResultSet.java | 31 ++++ .../jdbc/FlightServerTestExtension.java | 6 + .../driver/jdbc/TimestampResultSetTest.java | 164 ++++++++++++++++++ 3 files changed, 201 insertions(+) create mode 100644 flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/TimestampResultSetTest.java diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java index 0dc2b07c97..622e5fe7f6 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java @@ -19,6 +19,7 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; +import java.sql.Types; import java.util.HashSet; import java.util.List; import java.util.Objects; @@ -28,14 +29,17 @@ import org.apache.arrow.util.AutoCloseables; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.calcite.avatica.AvaticaConnection; import org.apache.calcite.avatica.AvaticaResultSet; import org.apache.calcite.avatica.AvaticaResultSetMetaData; +import org.apache.calcite.avatica.AvaticaSite; import org.apache.calcite.avatica.AvaticaStatement; import org.apache.calcite.avatica.ColumnMetaData; import org.apache.calcite.avatica.Meta; import org.apache.calcite.avatica.Meta.Frame; import org.apache.calcite.avatica.Meta.Signature; import org.apache.calcite.avatica.QueryState; +import org.apache.calcite.avatica.util.Cursor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -102,6 +106,33 @@ void populateData(final VectorSchemaRoot vectorSchemaRoot, final Schema schema) execute2(new ArrowFlightJdbcCursor(vectorSchemaRoot), this.signature.columns); } + /** + * The default method in AvaticaResultSet does not properly handle TIMESTASMP_WITH_TIMEZONE, so we + * override here to add support. + * + * @param columnIndex the first column is 1, the second is 2, ... + * @return Object + * @throws SQLException if there is an underlying exception + */ + @Override + public Object getObject(int columnIndex) throws SQLException { + this.checkOpen(); + + Cursor.Accessor accessor; + try { + accessor = accessorList.get(columnIndex - 1); + } catch (IndexOutOfBoundsException e) { + throw AvaticaConnection.HELPER.createException("invalid column ordinal: " + columnIndex); + } + + ColumnMetaData metaData = columnMetaDataList.get(columnIndex - 1); + if (metaData.type.id == Types.TIMESTAMP_WITH_TIMEZONE) { + return accessor.getTimestamp(localCalendar); + } else { + return AvaticaSite.get(accessor, metaData.type.id, localCalendar); + } + } + @Override protected void cancel() { signature.columns.clear(); diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/FlightServerTestExtension.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/FlightServerTestExtension.java index db0438059f..f71114e1b5 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/FlightServerTestExtension.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/FlightServerTestExtension.java @@ -130,6 +130,12 @@ public Connection getConnection(boolean useEncryption) throws SQLException { return this.createDataSource().getConnection(); } + public Connection getConnection(String timezone) throws SQLException { + setUseEncryption(false); + properties.put("timezone", timezone); + return this.createDataSource().getConnection(); + } + private void setUseEncryption(boolean useEncryption) { properties.put("useEncryption", useEncryption); } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/TimestampResultSetTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/TimestampResultSetTest.java new file mode 100644 index 0000000000..0921ae2d38 --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/TimestampResultSetTest.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc; + +import com.google.common.collect.ImmutableList; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.Calendar; +import java.util.Collections; +import java.util.TimeZone; +import org.apache.arrow.driver.jdbc.utils.MockFlightSqlProducer; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.TimeStampVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * Timestamps have a lot of nuances in JDBC. This class is here to test that timestamp behavior is + * correct for different types of Timestamp vectors as well as different methods of retrieving the + * timestamps in JDBC. + */ +public class TimestampResultSetTest { + private static final MockFlightSqlProducer FLIGHT_SQL_PRODUCER = new MockFlightSqlProducer(); + + @RegisterExtension public static FlightServerTestExtension FLIGHT_SERVER_TEST_EXTENSION; + + static { + FLIGHT_SERVER_TEST_EXTENSION = + FlightServerTestExtension.createStandardTestExtension(FLIGHT_SQL_PRODUCER); + } + + private static final String QUERY_STRING = "SELECT * FROM TIMESTAMPS"; + private static final Schema QUERY_SCHEMA = + new Schema( + ImmutableList.of( + Field.nullable("no_tz", new ArrowType.Timestamp(TimeUnit.MILLISECOND, null)), + Field.nullable("utc", new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC")), + Field.nullable("utc+1", new ArrowType.Timestamp(TimeUnit.MILLISECOND, "GMT+1")), + Field.nullable("utc-1", new ArrowType.Timestamp(TimeUnit.MILLISECOND, "GMT-1")))); + + @BeforeAll + public static void setup() throws SQLException { + Instant firstDay2025 = OffsetDateTime.of(2025, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant(); + + FLIGHT_SQL_PRODUCER.addSelectQuery( + QUERY_STRING, + QUERY_SCHEMA, + Collections.singletonList( + listener -> { + try (final BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + final VectorSchemaRoot root = VectorSchemaRoot.create(QUERY_SCHEMA, allocator)) { + listener.start(root); + root.getFieldVectors() + .forEach(v -> ((TimeStampVector) v).setSafe(0, firstDay2025.toEpochMilli())); + root.setRowCount(1); + listener.putNext(); + } catch (final Throwable throwable) { + listener.error(throwable); + } finally { + listener.completed(); + } + })); + } + + /** + * This test doesn't yet test anything other than ensuring all ResultSet methods to retrieve a + * timestamp succeed. + * + *

This is a good starting point to add more tests to ensure the values are correct when we + * change the "local calendar" either through changing the JVM default or through the connection + * property. + */ + @Test + public void test() { + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + try (Connection connection = FLIGHT_SERVER_TEST_EXTENSION.getConnection("UTC")) { + try (PreparedStatement s = connection.prepareStatement(QUERY_STRING)) { + try (ResultSet rs = s.executeQuery()) { + int numCols = rs.getMetaData().getColumnCount(); + try { + rs.next(); + for (int i = 1; i <= numCols; i++) { + int type = rs.getMetaData().getColumnType(i); + String name = rs.getMetaData().getColumnName(i); + System.out.println(name); + System.out.print("- getDate:\t\t\t\t\t\t\t"); + System.out.print(rs.getDate(i)); + System.out.println(); + System.out.print("- getTimestamp:\t\t\t\t\t\t"); + System.out.print(rs.getTimestamp(i)); + System.out.println(); + System.out.print("- getString:\t\t\t\t\t\t"); + System.out.print(rs.getString(i)); + System.out.println(); + System.out.print("- getObject:\t\t\t\t\t\t"); + System.out.print(rs.getObject(i)); + System.out.println(); + System.out.print("- getObject(Timestamp.class):\t\t"); + System.out.print(rs.getObject(i, Timestamp.class)); + System.out.println(); + System.out.print("- getTimestamp(default Calendar):\t"); + System.out.print(rs.getTimestamp(i, Calendar.getInstance())); + System.out.println(); + System.out.print("- getTimestamp(UTC Calendar):\t\t"); + System.out.print( + rs.getTimestamp(i, Calendar.getInstance(TimeZone.getTimeZone("UTC")))); + System.out.println(); + System.out.print("- getObject(LocalDateTime.class):\t"); + System.out.print(rs.getObject(i, LocalDateTime.class)); + System.out.println(); + if (type == Types.TIMESTAMP_WITH_TIMEZONE) { + System.out.print("- getObject(Instant.class):\t\t\t"); + System.out.print(rs.getObject(i, Instant.class)); + System.out.println(); + System.out.print("- getObject(OffsetDateTime.class):\t"); + System.out.print(rs.getObject(i, OffsetDateTime.class)); + System.out.println(); + System.out.print("- getObject(ZonedDateTime.class):\t"); + System.out.print(rs.getObject(i, ZonedDateTime.class)); + System.out.println(); + } + System.out.println(); + } + System.out.println(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + } +} From 1e533821c19516a19d7c6dec58c657735816c0e5 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 8 Oct 2025 12:49:54 +0900 Subject: [PATCH 206/408] GH-880: [CI] Fix syntax error in `dev_pr.yml` (#881) ## What's Changed We can't write `permissions` in a step. Closes #880. --- .github/workflows/dev_pr.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml index 58f5f3d30a..2f4a48d572 100644 --- a/.github/workflows/dev_pr.yml +++ b/.github/workflows/dev_pr.yml @@ -35,6 +35,7 @@ concurrency: permissions: contents: read + issues: write pull-requests: write jobs: @@ -80,9 +81,5 @@ jobs: if: '! github.event.pull_request.draft' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - permissions: - contents: read - issues: write - pull-requests: write run: | ./.github/workflows/dev_pr_milestone.sh "${GITHUB_REPOSITORY}" ${{ github.event.number }} From 37e9ba6b52d89c14cec5ed706f96d7986551eb8c Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 8 Oct 2025 12:51:17 +0900 Subject: [PATCH 207/408] GH-592: [Release] Use relative path in .sha* (#879) ## What's Changed If we use absolute path, users need to create the same directory structure. It's inconvenient. We should use relative path. Closes #592. --- ci/scripts/jni_full_build.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/scripts/jni_full_build.sh b/ci/scripts/jni_full_build.sh index e9ad0ddbda..5d0aee0555 100755 --- a/ci/scripts/jni_full_build.sh +++ b/ci/scripts/jni_full_build.sh @@ -97,8 +97,10 @@ find ~/.m2/repository/org/apache/arrow \ -exec echo "{}" ";" \ -exec cp "{}" "${dist_dir}" ";" -for artifact in "${dist_dir}"/*; do +pushd "${dist_dir}" +for artifact in *; do sha256sum "${artifact}" >"${artifact}.sha256" sha512sum "${artifact}" >"${artifact}.sha512" done +popd github_actions_group_end From aee8a1070132ada6a9cd5fde1efe1af06c2b72e0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Oct 2025 13:42:16 +0900 Subject: [PATCH 208/408] MINOR: [CI] Bump docker/login-action from 3.5.0 to 3.6.0 (#870) Bumps [docker/login-action](https://github.com/docker/login-action) from 3.5.0 to 3.6.0. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index eec3344097..60fa12f528 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -133,7 +133,7 @@ jobs: with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing - - uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 + - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: registry: ghcr.io username: ${{ github.actor }} From f71a02c043a311a17059df7420673bf538384826 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Mon, 27 Oct 2025 10:30:03 +0100 Subject: [PATCH 209/408] GH-898: Upgrade to Apache POM 35 and identify fixes needed to have CI happy (#865) Closes #898. --------- Co-authored-by: Sutou Kouhei Co-authored-by: Sutou Kouhei Co-authored-by: David Li --- .env | 5 +- .github/workflows/rc.yml | 26 +++++---- ci/docker/vcpkg-jni.dockerfile | 16 +---- ci/scripts/jni_build.sh | 2 +- ci/scripts/jni_macos_build.sh | 67 ++++----------------- ci/scripts/jni_manylinux_build.sh | 97 ++++--------------------------- ci/scripts/jni_windows_build.sh | 2 +- compose.yaml | 2 +- pom.xml | 5 +- 9 files changed, 48 insertions(+), 174 deletions(-) diff --git a/.env b/.env index d3e1c1d63a..a7783537d0 100644 --- a/.env +++ b/.env @@ -40,7 +40,7 @@ ARCH_SHORT=amd64 # Default repository to pull and push images from REPO=ghcr.io/apache/arrow-java-dev -ARROW_REPO=apache/arrow-dev +ARROW_REPO=ghcr.io/apache/arrow-dev # The setup attempts to generate coredumps by default, in order to disable the # coredump generation set it to 0 @@ -53,5 +53,4 @@ MAVEN=3.9.9 # Versions for various dependencies used to build artifacts # Keep in sync with apache/arrow ARROW_REPO_ROOT=./arrow -PYTHON=3.9 -VCPKG="f7423ee180c4b7f40d43402c2feb3859161ef625" # 2024.06.15 Release +VCPKG="4334d8b4c8916018600212ab4dd4bbdc343065d1" # 2025.09.17 Release diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 60fa12f528..f71f8aeed1 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -168,7 +168,7 @@ jobs: fail-fast: false matrix: platform: - - { runs_on: macos-13, arch: "x86_64"} + - { runs_on: macos-15-intel, arch: "x86_64"} - { runs_on: macos-14, arch: "aarch_64" } env: MACOSX_DEPLOYMENT_TARGET: "14.0" @@ -222,7 +222,7 @@ jobs: brew uninstall llvm || : # We can remove this when we drop support for - # macos-13. because macos-14 or later uses /opt/homebrew/ + # macos-15-intel. because macos-14 or later with arm64 uses /opt/homebrew/ # not /usr/local/. # # Ensure updating python@XXX with the "--overwrite" option. @@ -298,7 +298,7 @@ jobs: fail-fast: false matrix: platform: - - runs_on: windows-2019 + - runs_on: windows-2022 arch: "x86_64" steps: - name: Download source archive @@ -309,13 +309,19 @@ jobs: shell: bash run: | tar -xf apache-arrow-java-*.tar.gz --strip-components=1 - - name: Download the latest Apache Arrow C++ - if: github.event_name != 'schedule' - shell: bash - run: | - ci/scripts/download_cpp.sh + # We always use the main branch for apache/arrow for now. + # Because we want to use + # https://github.com/apache/arrow/pull/47749 in + # apache/arrow-java. We can revert this workaround once Apache + # Arrow 22.0.0 that includes the change released. + # + # - name: Download the latest Apache Arrow C++ + # if: github.event_name != 'schedule' + # shell: bash + # run: | + # ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ - if: github.event_name == 'schedule' + # if: github.event_name == 'schedule' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: apache/arrow @@ -354,7 +360,7 @@ jobs: - name: Build shell: cmd run: | - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 REM For ORC set TZDIR=/c/msys64/usr/share/zoneinfo bash -c "ci/scripts/jni_windows_build.sh . arrow build jni" diff --git a/ci/docker/vcpkg-jni.dockerfile b/ci/docker/vcpkg-jni.dockerfile index 55fa35e0d1..d6bd322a39 100644 --- a/ci/docker/vcpkg-jni.dockerfile +++ b/ci/docker/vcpkg-jni.dockerfile @@ -18,24 +18,10 @@ ARG base FROM ${base} -# Install the libraries required by Gandiva to run -# Use enable llvm[enable-rtti] in the vcpkg.json to avoid link problems in Gandiva -RUN vcpkg install \ - --clean-after-build \ - --x-install-root=${VCPKG_ROOT}/installed \ - --x-manifest-root=/arrow/ci/vcpkg \ - --x-feature=dev \ - --x-feature=flight \ - --x-feature=gcs \ - --x-feature=json \ - --x-feature=parquet \ - --x-feature=gandiva \ - --x-feature=s3 - # Install Java # We need Java for JNI headers, but we don't invoke Maven in this build. ARG java=11 -RUN yum install -y java-$java-openjdk-devel && yum clean all +RUN dnf install -y java-$java-openjdk-devel && dnf clean all # For ci/scripts/{cpp,java}_*.sh ENV ARROW_HOME=/tmp/local \ diff --git a/ci/scripts/jni_build.sh b/ci/scripts/jni_build.sh index aec6fc325c..c000837987 100755 --- a/ci/scripts/jni_build.sh +++ b/ci/scripts/jni_build.sh @@ -66,7 +66,7 @@ cmake \ -DProtobuf_USE_STATIC_LIBS=ON \ -GNinja \ "${EXTRA_CMAKE_OPTIONS[@]}" -cmake --build "${build_dir}" +cmake --build "${build_dir}" --verbose if [ "${ARROW_JAVA_BUILD_TESTS}" = "ON" ]; then ctest \ --output-on-failure \ diff --git a/ci/scripts/jni_macos_build.sh b/ci/scripts/jni_macos_build.sh index f7543b6f7a..13c0675d38 100755 --- a/ci/scripts/jni_macos_build.sh +++ b/ci/scripts/jni_macos_build.sh @@ -59,72 +59,24 @@ fi github_actions_group_begin "Building Arrow C++ libraries" install_dir="${build_dir}/cpp-install" -: "${ARROW_ACERO:=ON}" -export ARROW_ACERO -: "${ARROW_BUILD_TESTS:=OFF}" -export ARROW_BUILD_TESTS -: "${ARROW_DATASET:=ON}" -export ARROW_DATASET -: "${ARROW_GANDIVA:=ON}" -export ARROW_GANDIVA -: "${ARROW_ORC:=ON}" -export ARROW_ORC -: "${ARROW_PARQUET:=ON}" -: "${ARROW_S3:=ON}" -: "${CMAKE_BUILD_TYPE:=Release}" -: "${CMAKE_UNITY_BUILD:=ON}" -export ARROW_TEST_DATA="${arrow_dir}/testing/data" -export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data" +export ARROW_BUILD_TESTS=OFF + +export ARROW_DATASET=ON +export ARROW_GANDIVA=ON +export ARROW_ORC=ON +export ARROW_PARQUET=ON + export AWS_EC2_METADATA_DISABLED=TRUE cmake \ -S "${arrow_dir}/cpp" \ -B "${build_dir}/cpp" \ - -DARROW_ACERO="${ARROW_ACERO}" \ - -DARROW_BUILD_SHARED=OFF \ - -DARROW_BUILD_TESTS="${ARROW_BUILD_TESTS}" \ - -DARROW_CSV="${ARROW_DATASET}" \ - -DARROW_DATASET="${ARROW_DATASET}" \ - -DARROW_SUBSTRAIT="${ARROW_DATASET}" \ - -DARROW_DEPENDENCY_USE_SHARED=OFF \ - -DARROW_GANDIVA="${ARROW_GANDIVA}" \ - -DARROW_GANDIVA_STATIC_LIBSTDCPP=ON \ - -DARROW_JSON="${ARROW_DATASET}" \ - -DARROW_ORC="${ARROW_ORC}" \ - -DARROW_PARQUET="${ARROW_PARQUET}" \ - -DARROW_S3="${ARROW_S3}" \ - -DARROW_USE_CCACHE="${ARROW_USE_CCACHE}" \ - -DAWSSDK_SOURCE=BUNDLED \ - -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" \ - -DCMAKE_INSTALL_PREFIX="${install_dir}" \ - -DCMAKE_UNITY_BUILD="${CMAKE_UNITY_BUILD}" \ - -DGTest_SOURCE=BUNDLED \ - -DPARQUET_BUILD_EXAMPLES=OFF \ - -DPARQUET_BUILD_EXECUTABLES=OFF \ - -DPARQUET_REQUIRE_ENCRYPTION=OFF \ - -Dre2_SOURCE=BUNDLED \ - -GNinja + --preset=ninja-release-jni-macos \ + -DCMAKE_INSTALL_PREFIX="${install_dir}" cmake --build "${build_dir}/cpp" --target install github_actions_group_end -if [ "${ARROW_RUN_TESTS:-}" == "ON" ]; then - github_actions_group_begin "Running Arrow C++ libraries tests" - # MinIO is required - exclude_tests="arrow-s3fs-test" - # unstable - exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test" - exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test" - ctest \ - --exclude-regex "${exclude_tests}" \ - --label-regex unittest \ - --output-on-failure \ - --parallel "$(sysctl -n hw.ncpu)" \ - --test-dir "${build_dir}/cpp" \ - --timeout 300 - github_actions_group_end -fi - export JAVA_JNI_CMAKE_ARGS="-DProtobuf_ROOT=${build_dir}/cpp/protobuf_ep-install" "${source_dir}/ci/scripts/jni_build.sh" \ "${source_dir}" \ @@ -142,6 +94,7 @@ github_actions_group_begin "Checking shared dependencies for libraries" pushd "${dist_dir}" archery linking check-dependencies \ --allow CoreFoundation \ + --allow Network \ --allow Security \ --allow libSystem \ --allow libarrow_cdata_jni \ diff --git a/ci/scripts/jni_manylinux_build.sh b/ci/scripts/jni_manylinux_build.sh index a34ec0f420..3577c37ab3 100755 --- a/ci/scripts/jni_manylinux_build.sh +++ b/ci/scripts/jni_manylinux_build.sh @@ -53,33 +53,19 @@ if [ "${ARROW_USE_CCACHE}" == "ON" ]; then fi github_actions_group_begin "Building Arrow C++ libraries" -devtoolset_version="$(rpm -qa "devtoolset-*-gcc" --queryformat '%{VERSION}' | grep -o "^[0-9]*")" -devtoolset_include_cpp="/opt/rh/devtoolset-${devtoolset_version}/root/usr/include/c++/${devtoolset_version}" -: "${ARROW_ACERO:=ON}" -export ARROW_ACERO -: "${ARROW_BUILD_TESTS:=OFF}" -export ARROW_BUILD_TESTS -: "${ARROW_DATASET:=ON}" -export ARROW_DATASET -: "${ARROW_GANDIVA:=ON}" -export ARROW_GANDIVA -: "${ARROW_GCS:=ON}" -: "${ARROW_JEMALLOC:=OFF}" -: "${ARROW_MIMALLOC:=ON}" -: "${ARROW_RPATH_ORIGIN:=ON}" -: "${ARROW_ORC:=ON}" -export ARROW_ORC -: "${ARROW_PARQUET:=ON}" -: "${ARROW_S3:=ON}" -: "${CMAKE_BUILD_TYPE:=release}" -: "${CMAKE_UNITY_BUILD:=ON}" + : "${VCPKG_ROOT:=/opt/vcpkg}" : "${VCPKG_FEATURE_FLAGS:=-manifests}" -: "${VCPKG_TARGET_TRIPLET:=${VCPKG_DEFAULT_TRIPLET:-x64-linux-static-${CMAKE_BUILD_TYPE}}}" -: "${GANDIVA_CXX_FLAGS:=-isystem;${devtoolset_include_cpp};-isystem;${devtoolset_include_cpp}/x86_64-redhat-linux;-lpthread}" +: "${VCPKG_TARGET_TRIPLET:=${VCPKG_DEFAULT_TRIPLET:-x64-linux-static-release}}" +export VCPKG_TARGET_TRIPLET + +export ARROW_BUILD_TESTS=OFF + +export ARROW_DATASET=ON +export ARROW_GANDIVA=ON +export ARROW_ORC=ON +export ARROW_PARQUET=ON -export ARROW_TEST_DATA="${arrow_dir}/testing/data" -export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data" export AWS_EC2_METADATA_DISABLED=TRUE install_dir="${build_dir}/cpp-install" @@ -87,71 +73,12 @@ install_dir="${build_dir}/cpp-install" cmake \ -S "${arrow_dir}/cpp" \ -B "${build_dir}/cpp" \ - -DARROW_ACERO="${ARROW_ACERO}" \ - -DARROW_BUILD_SHARED=OFF \ - -DARROW_BUILD_TESTS="${ARROW_BUILD_TESTS}" \ - -DARROW_CSV="${ARROW_DATASET}" \ - -DARROW_DATASET="${ARROW_DATASET}" \ - -DARROW_SUBSTRAIT="${ARROW_DATASET}" \ - -DARROW_DEPENDENCY_SOURCE="VCPKG" \ - -DARROW_DEPENDENCY_USE_SHARED=OFF \ - -DARROW_GANDIVA_PC_CXX_FLAGS="${GANDIVA_CXX_FLAGS}" \ - -DARROW_GANDIVA="${ARROW_GANDIVA}" \ - -DARROW_GCS="${ARROW_GCS}" \ - -DARROW_JEMALLOC="${ARROW_JEMALLOC}" \ - -DARROW_JSON="${ARROW_DATASET}" \ - -DARROW_MIMALLOC="${ARROW_MIMALLOC}" \ - -DARROW_ORC="${ARROW_ORC}" \ - -DARROW_PARQUET="${ARROW_PARQUET}" \ - -DARROW_RPATH_ORIGIN="${ARROW_RPATH_ORIGIN}" \ - -DARROW_S3="${ARROW_S3}" \ - -DARROW_USE_CCACHE="${ARROW_USE_CCACHE}" \ - -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" \ - -DCMAKE_INSTALL_PREFIX="${install_dir}" \ - -DCMAKE_UNITY_BUILD="${CMAKE_UNITY_BUILD}" \ - -DGTest_SOURCE=BUNDLED \ - -DORC_SOURCE=BUNDLED \ - -DORC_PROTOBUF_EXECUTABLE="${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/tools/protobuf/protoc" \ - -DPARQUET_BUILD_EXAMPLES=OFF \ - -DPARQUET_BUILD_EXECUTABLES=OFF \ - -DPARQUET_REQUIRE_ENCRYPTION=OFF \ - -DVCPKG_MANIFEST_MODE=OFF \ - -DVCPKG_TARGET_TRIPLET="${VCPKG_TARGET_TRIPLET}" \ - -GNinja + --preset=ninja-release-jni-linux \ + -DCMAKE_INSTALL_PREFIX="${install_dir}" cmake --build "${build_dir}/cpp" cmake --install "${build_dir}/cpp" github_actions_group_end -if [ "${ARROW_RUN_TESTS:-OFF}" = "ON" ]; then - github_actions_group_begin "Running Arrow C++ libraries tests" - # MinIO is required - exclude_tests="arrow-s3fs-test" - case $(arch) in - aarch64) - # GCS testbench is crashed on aarch64: - # ImportError: ../grpc/_cython/cygrpc.cpython-38-aarch64-linux-gnu.so: - # undefined symbol: vtable for std::__cxx11::basic_ostringstream< - # char, std::char_traits, std::allocator > - exclude_tests="${exclude_tests}|arrow-gcsfs-test" - ;; - esac - # unstable - exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test" - exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test" - # external dependency - exclude_tests="${exclude_tests}|arrow-gcsfs-test" - # strptime - exclude_tests="${exclude_tests}|arrow-utility-test" - ctest \ - --exclude-regex "${exclude_tests}" \ - --label-regex unittest \ - --output-on-failure \ - --parallel "$(nproc)" \ - --test-dir "${build_dir}/cpp" \ - --timeout 300 - github_actions_group_end -fi - JAVA_JNI_CMAKE_ARGS="-DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" JAVA_JNI_CMAKE_ARGS="${JAVA_JNI_CMAKE_ARGS} -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET}" export JAVA_JNI_CMAKE_ARGS diff --git a/ci/scripts/jni_windows_build.sh b/ci/scripts/jni_windows_build.sh index d01ef45f5a..6503ac63e5 100755 --- a/ci/scripts/jni_windows_build.sh +++ b/ci/scripts/jni_windows_build.sh @@ -68,7 +68,7 @@ cmake \ -B "${build_dir}/cpp" \ -DARROW_ACERO="${ARROW_ACERO}" \ -DARROW_BUILD_SHARED=OFF \ - -DARROW_BUILD_TESTS=ON \ + -DARROW_BUILD_TESTS="${ARROW_BUILD_TESTS}" \ -DARROW_CSV="${ARROW_DATASET}" \ -DARROW_DATASET="${ARROW_DATASET}" \ -DARROW_SUBSTRAIT="${ARROW_DATASET}" \ diff --git a/compose.yaml b/compose.yaml index b125c3c983..f5082a22aa 100644 --- a/compose.yaml +++ b/compose.yaml @@ -99,7 +99,7 @@ services: cache_from: - ${REPO}:${ARCH}-vcpkg-jni-${VCPKG} args: - base: ${ARROW_REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2014-vcpkg-${VCPKG} + base: ${ARROW_REPO}:${ARCH}-cpp-jni-${VCPKG} volumes: - .:/arrow-java:delegated - ${ARROW_REPO_ROOT}:/arrow:delegated diff --git a/pom.xml b/pom.xml index 49e0c47c60..f6c9053db9 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache apache - 34 + 35 org.apache.arrow @@ -91,6 +91,7 @@ under the License. + 1695310533 ${project.build.directory}/generated-sources 1.9.0 5.12.2 @@ -123,6 +124,8 @@ under the License. 3.2.2 From ed81e5981a2bee40584b3a411ed755cb4cc5b91f Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Tue, 28 Oct 2025 00:57:13 +0100 Subject: [PATCH 210/408] GH-882: Add support for loading native library from a user specified location (#883) ## What's Changed Add an opt-in code path that attempts to load the native library relative to the path specified by the `arrow.cdata.library.path` system property. Closes #882. --- .../org/apache/arrow/c/jni/JniLoader.java | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/c/src/main/java/org/apache/arrow/c/jni/JniLoader.java b/c/src/main/java/org/apache/arrow/c/jni/JniLoader.java index f712b400bf..46c93f5541 100644 --- a/c/src/main/java/org/apache/arrow/c/jni/JniLoader.java +++ b/c/src/main/java/org/apache/arrow/c/jni/JniLoader.java @@ -75,8 +75,23 @@ private synchronized void loadRemaining() { } private void load(String name) { - final String libraryToLoad = - name + "/" + getNormalizedArch() + "/" + System.mapLibraryName(name); + String libraryName = System.mapLibraryName(name); + + // If 'arrow.cdata.library.path' is defined, try to load the native library from there + String libraryPath = System.getProperty("arrow.cdata.library.path"); + if (libraryPath != null) { + try { + File libraryFile = new File(libraryPath, libraryName); + if (libraryFile.isFile()) { + System.load(libraryFile.getAbsolutePath()); + return; + } + } catch (UnsatisfiedLinkError e) { + // Ignore this error and fall back to extracting from the JAR file + } + } + + final String libraryToLoad = name + "/" + getNormalizedArch() + "/" + libraryName; try { File temp = File.createTempFile("jnilib-", ".tmp", new File(System.getProperty("java.io.tmpdir"))); From a4f3f3ef8eaca1328a3a6032397caf7a2f9ac22d Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 29 Oct 2025 13:55:21 +0900 Subject: [PATCH 211/408] GH-899: [Dataset] Initialize compute module (#893) ## What's Changed Depends on https://github.com/apache/arrow-java/pull/865 Closes #899. --- dataset/src/main/cpp/jni_wrapper.cc | 8 ++++++++ .../main/java/org/apache/arrow/dataset/jni/JniLoader.java | 1 + .../java/org/apache/arrow/dataset/jni/JniWrapper.java | 3 +++ docs/source/substrait.rst | 2 +- 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/dataset/src/main/cpp/jni_wrapper.cc b/dataset/src/main/cpp/jni_wrapper.cc index 49cc85251c..e8087648eb 100644 --- a/dataset/src/main/cpp/jni_wrapper.cc +++ b/dataset/src/main/cpp/jni_wrapper.cc @@ -23,6 +23,7 @@ #include "arrow/array/concatenate.h" #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" +#include "arrow/compute/initialize.h" #include "arrow/dataset/api.h" #include "arrow/dataset/file_base.h" #ifdef ARROW_CSV @@ -807,6 +808,13 @@ JNIEXPORT void JNICALL Java_org_apache_arrow_dataset_jni_JniWrapper_ensureS3Fina JNI_METHOD_END() } +JNIEXPORT void JNICALL Java_org_apache_arrow_dataset_jni_JniWrapper_initialize( + JNIEnv* env, jobject) { + JNI_METHOD_START + JniAssertOkOrThrow(arrow::compute::Initialize()); + JNI_METHOD_END() +} + /* * Class: org_apache_arrow_dataset_file_JniWrapper * Method: makeFileSystemDatasetFactory diff --git a/dataset/src/main/java/org/apache/arrow/dataset/jni/JniLoader.java b/dataset/src/main/java/org/apache/arrow/dataset/jni/JniLoader.java index 631b8b1bbe..5fb4816488 100644 --- a/dataset/src/main/java/org/apache/arrow/dataset/jni/JniLoader.java +++ b/dataset/src/main/java/org/apache/arrow/dataset/jni/JniLoader.java @@ -56,6 +56,7 @@ public void ensureLoaded() { } loadRemaining(); ensureS3FinalizedOnShutdown(); + JniWrapper.get().initialize(); } private synchronized void loadRemaining() { diff --git a/dataset/src/main/java/org/apache/arrow/dataset/jni/JniWrapper.java b/dataset/src/main/java/org/apache/arrow/dataset/jni/JniWrapper.java index 6637c113d9..cfef098ec4 100644 --- a/dataset/src/main/java/org/apache/arrow/dataset/jni/JniWrapper.java +++ b/dataset/src/main/java/org/apache/arrow/dataset/jni/JniWrapper.java @@ -124,4 +124,7 @@ public native long createScanner( * uninitialized, then this is a noop. */ public native void ensureS3Finalized(); + + /** Initialize Arrow Compute. */ + public native void initialize(); } diff --git a/docs/source/substrait.rst b/docs/source/substrait.rst index b3678ac815..5ec07f1658 100644 --- a/docs/source/substrait.rst +++ b/docs/source/substrait.rst @@ -19,7 +19,7 @@ Substrait ========= -The ``arrow-dataset`` module can execute Substrait_ plans via the :external+arrow:doc:`Acero ` +The ``arrow-dataset`` module can execute Substrait_ plans via the :external+arrow:doc:`Acero ` query engine. Executing Queries Using Substrait Plans From c4d3c9e2777f5d7b55c3f8fdb5e5155b71bbd87a Mon Sep 17 00:00:00 2001 From: ViggoC Date: Wed, 29 Oct 2025 20:42:45 +0800 Subject: [PATCH 212/408] GH-109: Implement Vector Validators for StringView (#886) ## What's Changed Implement Vector Validators for StringView. Closes #109. --- .../validate/ValidateVectorBufferVisitor.java | 29 ++++++++++++++----- .../validate/ValidateVectorDataVisitor.java | 3 +- .../validate/ValidateVectorTypeVisitor.java | 9 +++++- .../validate/ValidateVectorVisitor.java | 9 ++++-- .../vector/TestVariableWidthViewVector.java | 17 ++++++++++- 5 files changed, 54 insertions(+), 13 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorBufferVisitor.java b/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorBufferVisitor.java index 5c7215437f..5cfe64b14e 100644 --- a/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorBufferVisitor.java +++ b/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorBufferVisitor.java @@ -52,14 +52,22 @@ private void validateVectorCommon(ValueVector vector) { if (vector instanceof FieldVector) { FieldVector fieldVector = (FieldVector) vector; - // TODO: https://github.com/apache/arrow/issues/41734 int typeBufferCount = TypeLayout.getTypeBufferCount(arrowType); - validateOrThrow( - fieldVector.getFieldBuffers().size() == typeBufferCount, - "Expected %s buffers in vector of type %s, got %s.", - typeBufferCount, - vector.getField().getType().toString(), - fieldVector.getFieldBuffers().size()); + if (TypeLayout.getTypeLayout(arrowType).isFixedBufferCount()) { + validateOrThrow( + fieldVector.getFieldBuffers().size() == typeBufferCount, + "Expected %s buffers in vector of type %s, got %s.", + typeBufferCount, + vector.getField().getType().toString(), + fieldVector.getFieldBuffers().size()); + } else { + validateOrThrow( + fieldVector.getFieldBuffers().size() >= typeBufferCount, + "Expected at least %s buffers in vector of type %s, got %s.", + typeBufferCount, + vector.getField().getType().toString(), + fieldVector.getFieldBuffers().size()); + } } } @@ -158,7 +166,12 @@ public Void visit(BaseLargeVariableWidthVector vector, Void value) { @Override public Void visit(BaseVariableWidthViewVector vector, Void value) { - throw new UnsupportedOperationException("View vectors are not supported."); + final int valueCount = vector.getValueCount(); + validateVectorCommon(vector); + validateOrThrow(vector.getFieldBuffers().size() >= 2, "Expected at least 2 buffers."); + validateValidityBuffer(vector, valueCount); + validateDataBuffer(vector, (long) valueCount * BaseVariableWidthViewVector.ELEMENT_SIZE); + return null; } @Override diff --git a/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorDataVisitor.java b/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorDataVisitor.java index c62bff79f7..9da8cc813e 100644 --- a/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorDataVisitor.java +++ b/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorDataVisitor.java @@ -121,7 +121,8 @@ public Void visit(BaseLargeVariableWidthVector vector, Void value) { @Override public Void visit(BaseVariableWidthViewVector vector, Void value) { - throw new UnsupportedOperationException("View vectors are not supported."); + vector.validateScalars(); + return null; } @Override diff --git a/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorTypeVisitor.java b/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorTypeVisitor.java index daad41dbdc..395852ef79 100644 --- a/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorTypeVisitor.java +++ b/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorTypeVisitor.java @@ -61,6 +61,8 @@ import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.ViewVarBinaryVector; +import org.apache.arrow.vector.ViewVarCharVector; import org.apache.arrow.vector.compare.VectorVisitor; import org.apache.arrow.vector.complex.DenseUnionVector; import org.apache.arrow.vector.complex.FixedSizeListVector; @@ -380,7 +382,12 @@ public Void visit(BaseLargeVariableWidthVector vector, Void value) { @Override public Void visit(BaseVariableWidthViewVector vector, Void value) { - throw new UnsupportedOperationException("View vectors are not supported."); + if (vector instanceof ViewVarCharVector) { + validateVectorCommon(vector, ArrowType.Utf8View.class); + } else if (vector instanceof ViewVarBinaryVector) { + validateVectorCommon(vector, ArrowType.BinaryView.class); + } + return null; } @Override diff --git a/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorVisitor.java b/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorVisitor.java index 5004ba488c..2111410016 100644 --- a/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorVisitor.java +++ b/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorVisitor.java @@ -107,8 +107,13 @@ public Void visit(BaseLargeVariableWidthVector left, Void value) { } @Override - public Void visit(BaseVariableWidthViewVector left, Void value) { - throw new UnsupportedOperationException("View vectors are not supported."); + public Void visit(BaseVariableWidthViewVector vector, Void value) { + if (vector.getValueCount() > 0) { + if (vector.getDataBuffer() == null || vector.getDataBuffer().capacity() == 0) { + throw new IllegalArgumentException("valueBuffer is null or capacity is 0"); + } + } + return null; } @Override diff --git a/vector/src/test/java/org/apache/arrow/vector/TestVariableWidthViewVector.java b/vector/src/test/java/org/apache/arrow/vector/TestVariableWidthViewVector.java index f7c66a00be..baf5e672c8 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestVariableWidthViewVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestVariableWidthViewVector.java @@ -61,6 +61,7 @@ import org.apache.arrow.vector.util.ReusableByteArray; import org.apache.arrow.vector.util.Text; import org.apache.arrow.vector.util.TransferPair; +import org.apache.arrow.vector.validate.ValidateUtil; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -2445,7 +2446,7 @@ public void testSplitAndTransferWithLongStringsOnValiditySplit() { final ViewVarBinaryVector sourceVector = newViewVarBinaryVector(EMPTY_SCHEMA_PATH, allocator)) { testSplitAndTransferOnValiditySplitHelper( - targetVector, sourceVector, startIndex, length, data); + targetVector, sourceVector, startIndex, length, binaryData); } } @@ -2852,4 +2853,18 @@ public void testVectorLoadUnloadOnMixedTypes() { } } } + + @Test + public void testValidate() { + try (final ViewVarCharVector vector = new ViewVarCharVector("v", allocator)) { + vector.validateFull(); + setVector(vector, STR1, STR2, STR3); + vector.validateFull(); + + vector.getDataBuffer().capacity(0); + ValidateUtil.ValidateException e = + assertThrows(ValidateUtil.ValidateException.class, () -> vector.validate()); + assertTrue(e.getMessage().contains("Not enough capacity for data buffer")); + } + } } From b8a23ddd8940d5a804b0091cfd7a908af4d0f532 Mon Sep 17 00:00:00 2001 From: Christopher Lambert Date: Thu, 30 Oct 2025 10:55:10 +0100 Subject: [PATCH 213/408] GH-900: Fix gandiva groupId in arrow-bom (#901) Use correct groupId for the `arrow-gandiva` artifact. Closes #900. --- bom/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bom/pom.xml b/bom/pom.xml index 61b452b9c1..655c33b813 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -165,7 +165,7 @@ under the License. ${project.version} - org.apache.arrow + org.apache.arrow.gandiva arrow-gandiva ${project.version} From 9f68d08f0d67699c2887bac011238ee7e0b01f06 Mon Sep 17 00:00:00 2001 From: ViggoC Date: Fri, 31 Oct 2025 15:55:12 +0800 Subject: [PATCH 214/408] GH-762: Implement VectorAppender for RunEndEncodedVector (#884) ## What's Changed Implement VectorAppender for RunEndEncodedVector Closes #762. --- .../arrow/vector/util/VectorAppender.java | 99 +++++++++++++++++++ .../arrow/vector/util/TestVectorAppender.java | 67 +++++++++++++ 2 files changed, 166 insertions(+) diff --git a/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java b/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java index 0dc96a4d4b..e7c0d11cb9 100644 --- a/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java +++ b/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java @@ -24,13 +24,17 @@ import org.apache.arrow.memory.util.MemoryUtil; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.BaseFixedWidthVector; +import org.apache.arrow.vector.BaseIntVector; import org.apache.arrow.vector.BaseLargeVariableWidthVector; import org.apache.arrow.vector.BaseVariableWidthVector; import org.apache.arrow.vector.BaseVariableWidthViewVector; +import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.BitVectorHelper; import org.apache.arrow.vector.ExtensionTypeVector; +import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.NullVector; +import org.apache.arrow.vector.SmallIntVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.compare.TypeEqualsVisitor; import org.apache.arrow.vector.compare.VectorVisitor; @@ -39,6 +43,7 @@ import org.apache.arrow.vector.complex.LargeListVector; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.NonNullableStructVector; +import org.apache.arrow.vector.complex.RunEndEncodedVector; import org.apache.arrow.vector.complex.UnionVector; /** Utility to append two vectors together. */ @@ -698,4 +703,98 @@ public ValueVector visit(ExtensionTypeVector deltaVector, Void value) { deltaVector.getUnderlyingVector().accept(underlyingAppender, null); return targetVector; } + + @Override + public ValueVector visit(RunEndEncodedVector deltaVector, Void value) { + Preconditions.checkArgument( + typeVisitor.equals(deltaVector), + "The deltaVector to append must have the same type as the targetVector"); + + if (deltaVector.getValueCount() == 0) { + return targetVector; // optimization, nothing to append, return + } + + RunEndEncodedVector targetEncodedVector = (RunEndEncodedVector) targetVector; + + final int targetLogicalValueCount = targetEncodedVector.getValueCount(); + + // Append the values vector first. + VectorAppender valueAppender = new VectorAppender(targetEncodedVector.getValuesVector()); + deltaVector.getValuesVector().accept(valueAppender, null); + + // Then append the run-ends vector. + BaseIntVector targetRunEndsVector = (BaseIntVector) targetEncodedVector.getRunEndsVector(); + BaseIntVector deltaRunEndsVector = (BaseIntVector) deltaVector.getRunEndsVector(); + appendRunEndsVector(targetRunEndsVector, deltaRunEndsVector, targetLogicalValueCount); + + targetEncodedVector.setValueCount(targetLogicalValueCount + deltaVector.getValueCount()); + return targetVector; + } + + private void appendRunEndsVector( + BaseIntVector targetRunEndsVector, + BaseIntVector deltaRunEndsVector, + int targetLogicalValueCount) { + int targetPhysicalValueCount = targetRunEndsVector.getValueCount(); + int newPhysicalValueCount = targetPhysicalValueCount + deltaRunEndsVector.getValueCount(); + + // make sure there is enough capacity + while (targetVector.getValueCapacity() < newPhysicalValueCount) { + targetVector.reAlloc(); + } + + // append validity buffer + BitVectorHelper.concatBits( + targetRunEndsVector.getValidityBuffer(), + targetRunEndsVector.getValueCount(), + deltaRunEndsVector.getValidityBuffer(), + deltaRunEndsVector.getValueCount(), + targetRunEndsVector.getValidityBuffer()); + + // shift and append data buffer + shiftAndAppendRunEndsDataBuffer( + targetRunEndsVector, + targetPhysicalValueCount, + deltaRunEndsVector.getDataBuffer(), + targetLogicalValueCount, + deltaRunEndsVector.getValueCount()); + + targetRunEndsVector.setValueCount(newPhysicalValueCount); + } + + private void shiftAndAppendRunEndsDataBuffer( + BaseIntVector toRunEndVector, + int toIndex, + ArrowBuf fromRunEndBuffer, + int offset, + int physicalLength) { + ArrowBuf toRunEndBuffer = toRunEndVector.getDataBuffer(); + if (toRunEndVector instanceof SmallIntVector) { + byte typeWidth = SmallIntVector.TYPE_WIDTH; + for (int i = 0; i < physicalLength; i++) { + toRunEndBuffer.setShort( + (long) (i + toIndex) * typeWidth, + fromRunEndBuffer.getShort((long) (i) * typeWidth) + offset); + } + + } else if (toRunEndVector instanceof IntVector) { + byte typeWidth = IntVector.TYPE_WIDTH; + for (int i = 0; i < physicalLength; i++) { + toRunEndBuffer.setInt( + (long) (i + toIndex) * typeWidth, + fromRunEndBuffer.getInt((long) (i) * typeWidth) + offset); + } + + } else if (toRunEndVector instanceof BigIntVector) { + byte typeWidth = BigIntVector.TYPE_WIDTH; + for (int i = 0; i < physicalLength; i++) { + toRunEndBuffer.setLong( + (long) (i + toIndex) * typeWidth, + fromRunEndBuffer.getLong((long) (i) * typeWidth) + offset); + } + } else { + throw new IllegalArgumentException( + "Run-end vector and must be of type int with size 16, 32, or 64 bits."); + } + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java b/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java index 4ee9630a4d..df5521a1ad 100644 --- a/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java +++ b/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java @@ -47,6 +47,7 @@ import org.apache.arrow.vector.complex.FixedSizeListVector; import org.apache.arrow.vector.complex.LargeListVector; import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.RunEndEncodedVector; import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.complex.UnionVector; import org.apache.arrow.vector.holders.NullableBigIntHolder; @@ -1025,6 +1026,72 @@ public void testAppendDenseUnionVectorMismatch() { } } + @Test + public void testAppendRunEndEncodedVector() { + final FieldType reeFieldType = FieldType.notNullable(ArrowType.RunEndEncoded.INSTANCE); + final Field runEndsField = + new Field("runEnds", FieldType.notNullable(Types.MinorType.INT.getType()), null); + final Field valuesField = Field.nullable("values", Types.MinorType.INT.getType()); + final List children = Arrays.asList(runEndsField, valuesField); + + final Field targetField = new Field("target", reeFieldType, children); + final Field deltaField = new Field("delta", reeFieldType, children); + try (RunEndEncodedVector target = new RunEndEncodedVector(targetField, allocator, null); + RunEndEncodedVector delta = new RunEndEncodedVector(deltaField, allocator, null)) { + + // populate target + target.allocateNew(); + // data: [1, 1, 2, null, 3, 3, 3] (7 values) + // values: [1, 2, null, 3] + // runEnds: [2, 3, 4, 7] + ValueVectorDataPopulator.setVector((IntVector) target.getValuesVector(), 1, 2, null, 3); + ValueVectorDataPopulator.setVector((IntVector) target.getRunEndsVector(), 2, 3, 4, 7); + target.setValueCount(7); + + // populate delta + delta.allocateNew(); + // data: [3, 4, 4, 5, null, null] (6 values) + // values: [3, 4, 5, null] + // runEnds: [1, 3, 4, 6] + ValueVectorDataPopulator.setVector((IntVector) delta.getValuesVector(), 3, 4, 5, null); + ValueVectorDataPopulator.setVector((IntVector) delta.getRunEndsVector(), 1, 3, 4, 6); + delta.setValueCount(6); + + VectorAppender appender = new VectorAppender(target); + delta.accept(appender, null); + + assertEquals(13, target.getValueCount()); + + final Field expectedField = new Field("expected", reeFieldType, children); + try (RunEndEncodedVector expected = new RunEndEncodedVector(expectedField, allocator, null)) { + expected.allocateNew(); + // expected data: [1, 1, 2, null, 3, 3, 3, 3, 4, 4, 5, null, null] (13 values) + // expected values: [1, 2, null, 3, 3, 4, 5, null] + // expected runEnds: [2, 3, 4, 7, 8, 10, 11, 13] + ValueVectorDataPopulator.setVector( + (IntVector) expected.getValuesVector(), 1, 2, null, 3, 3, 4, 5, null); + ValueVectorDataPopulator.setVector( + (IntVector) expected.getRunEndsVector(), 2, 3, 4, 7, 8, 10, 11, 13); + expected.setValueCount(13); + + assertVectorsEqual(expected, target); + } + + // Check that delta is unchanged. + final Field expectedDeltaField = new Field("expectedDelta", reeFieldType, children); + try (RunEndEncodedVector expectedDelta = + new RunEndEncodedVector(expectedDeltaField, allocator, null)) { + expectedDelta.allocateNew(); + ValueVectorDataPopulator.setVector( + (IntVector) expectedDelta.getValuesVector(), 3, 4, 5, null); + ValueVectorDataPopulator.setVector( + (IntVector) expectedDelta.getRunEndsVector(), 1, 3, 4, 6); + expectedDelta.setValueCount(6); + assertVectorsEqual(expectedDelta, delta); + } + } + } + @Test public void testAppendVectorNegative() { final int vectorLength = 10; From 03e0d4da63e1de747e645b0053d737b263d7f26d Mon Sep 17 00:00:00 2001 From: Fabio Buso Date: Tue, 4 Nov 2025 12:45:43 +0100 Subject: [PATCH 215/408] MINOR: Bump io.netty:netty-bom from 4.1.119.Final to 4.1.127.Final (#855) ## What's Changed This PR bumps the netty version to 4.1.127.Final to address CVE-2025-55163 Based on https://github.com/apache/arrow-java/pull/740 - It looks like the team would prefer to stay on Netty 4.1.x for the time being. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f6c9053db9..5b683acd1b 100644 --- a/pom.xml +++ b/pom.xml @@ -97,7 +97,7 @@ under the License. 5.12.2 2.0.17 33.4.8-jre - 4.1.119.Final + 4.1.127.Final 1.73.0 4.30.2 2.18.3 From ba2f7d62ebbb964956ddebb5a250e5637f816083 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Nov 2025 13:56:33 +0100 Subject: [PATCH 216/408] MINOR: Bump logback.version from 1.5.18 to 1.5.20 (#897) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps `logback.version` from 1.5.18 to 1.5.20. Updates `ch.qos.logback:logback-classic` from 1.5.18 to 1.5.20

Release notes

Sourced from ch.qos.logback:logback-classic's releases.

Logback 1.5.19

2025-09-30 Release of logback version 1.5.19

• Disallow "new" operator in the condition attribute of <if> elements. This fixes an ACE vulnerability recorded as CVE-2025-11226.

• At initialization time, slightly better reporting about watched configuration files.

• Softer message regarding usage of ConsoleAppender and its potential impact on performance.

• In ViewStatusMessagesServlet, restrict processing of "Clear" button to POST method. This change was proposed by Ralf Wiebicke who also provided the relevant PR.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit e572d4f87f06674788eb3ca7148e8d1dffc615fa associated with the tag v_1.5.19. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits
  • 930fb15 prepare release 1.5.20
  • 0b4432a provide an alternative to Janino based conditional configuration processing -...
  • 258558f provide an alternative to Janino based conditional configuration processing -...
  • ee77a70 provide an alternative to Janino based conditional configuration processing -...
  • 5ca7ce8 provide an alternative to Janino based conditional configuration processing -...
  • 728803f fix typo
  • aa5eeb1 start work on version 1.5.20-SNAPSHOT
  • e572d4f skip deployment of blackbox and example modules, published as version 1.5.9
  • 4adae8b add plugin for Maven Central deployment
  • ee70cf4 prepare release 1.5.19
  • Additional commits viewable in compare view

Updates `ch.qos.logback:logback-core` from 1.5.18 to 1.5.20
Release notes

Sourced from ch.qos.logback:logback-core's releases.

Logback 1.5.19

2025-09-30 Release of logback version 1.5.19

• Disallow "new" operator in the condition attribute of <if> elements. This fixes an ACE vulnerability recorded as CVE-2025-11226.

• At initialization time, slightly better reporting about watched configuration files.

• Softer message regarding usage of ConsoleAppender and its potential impact on performance.

• In ViewStatusMessagesServlet, restrict processing of "Clear" button to POST method. This change was proposed by Ralf Wiebicke who also provided the relevant PR.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit e572d4f87f06674788eb3ca7148e8d1dffc615fa associated with the tag v_1.5.19. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits
  • 930fb15 prepare release 1.5.20
  • 0b4432a provide an alternative to Janino based conditional configuration processing -...
  • 258558f provide an alternative to Janino based conditional configuration processing -...
  • ee77a70 provide an alternative to Janino based conditional configuration processing -...
  • 5ca7ce8 provide an alternative to Janino based conditional configuration processing -...
  • 728803f fix typo
  • aa5eeb1 start work on version 1.5.20-SNAPSHOT
  • e572d4f skip deployment of blackbox and example modules, published as version 1.5.9
  • 4adae8b add plugin for Maven Central deployment
  • ee70cf4 prepare release 1.5.19
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5b683acd1b..41b82fabab 100644 --- a/pom.xml +++ b/pom.xml @@ -111,7 +111,7 @@ under the License. true 2.37.0 3.49.3 - 1.5.18 + 1.5.21 none -Xdoclint:none From 87d727a85a33f10f67e7f311170a282ef5dbae29 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Nov 2025 15:22:49 +0100 Subject: [PATCH 217/408] MINOR: Bump com.github.luben:zstd-jni from 1.5.7-2 to 1.5.7-6 (#896) Bumps [com.github.luben:zstd-jni](https://github.com/luben/zstd-jni) from 1.5.7-2 to 1.5.7-6.
Commits
  • c3cf908 v1.5.7-6
  • 1941118 Restore getContentSize behaviour
  • 97b5262 Update the README with more info about the "cloud" flavour
  • 7093a69 Add new flavour of artifacr
  • 9c3386d fix: compress & decompress DirectByteBufferStream error is always ZSTD_error_...
  • 7de4360 Add sbt-sonatype plugin to the build
  • 8250e09 Changes to migrate to the new Sonatype publishing infrastructure
  • 81e0d71 fix: decompressFrame(byte[], int) fixed, throwing exception on get frame cont...
  • 8532971 fix: decompress(byte[]) method now decompresses all frames, new tests added
  • 051e8de refactor: throwing an exception on findFrameCompressedSize0 error
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.github.luben:zstd-jni&package-manager=maven&previous-version=1.5.7-2&new-version=1.5.7-6)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- compression/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compression/pom.xml b/compression/pom.xml index 6f60eb7d0a..ba13156243 100644 --- a/compression/pom.xml +++ b/compression/pom.xml @@ -55,7 +55,7 @@ under the License. com.github.luben zstd-jni - 1.5.7-2 + 1.5.7-6 From edc6cf350abc08e91fb1009c73be42288207d074 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Nov 2025 16:16:16 +0100 Subject: [PATCH 218/408] MINOR: [CI] Bump actions/download-artifact from 5.0.0 to 6.0.0 (#895) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 5.0.0 to 6.0.0.
Release notes

Sourced from actions/download-artifact's releases.

v6.0.0

What's Changed

BREAKING CHANGE: this update supports Node v24.x. This is not a breaking change per-se but we're treating it as such.

New Contributors

Full Changelog: https://github.com/actions/download-artifact/compare/v5...v6.0.0

Commits
  • 018cc2c Merge pull request #438 from actions/danwkennedy/prepare-6.0.0
  • 815651c Revert "Remove github.dep.yml"
  • bb3a066 Remove github.dep.yml
  • fa1ce46 Prepare v6.0.0
  • 4a24838 Merge pull request #431 from danwkennedy/patch-1
  • 5e3251c Readme: spell out the first use of GHES
  • abefc31 Merge pull request #424 from actions/yacaovsnc/update_readme
  • ac43a60 Update README with artifact extraction details
  • de96f46 Merge pull request #417 from actions/yacaovsnc/update_readme
  • 7993cb4 Remove migration guide for artifact download changes
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/download-artifact&package-manager=github_actions&previous-version=5.0.0&new-version=6.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index f71f8aeed1..b53812ef5e 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -101,7 +101,7 @@ jobs: packages: write steps: - name: Download source archive - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: name: release-source - name: Extract source archive @@ -174,7 +174,7 @@ jobs: MACOSX_DEPLOYMENT_TARGET: "14.0" steps: - name: Download source archive - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: name: release-source - name: Extract source archive @@ -302,7 +302,7 @@ jobs: arch: "x86_64" steps: - name: Download source archive - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: name: release-source - name: Extract source archive @@ -381,7 +381,7 @@ jobs: - jni-windows steps: - name: Download artifacts - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: path: artifacts - name: Decompress artifacts @@ -462,11 +462,11 @@ jobs: with: cache: 'pip' - name: Download source archive - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: name: release-source - name: Download Javadocs - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: name: reference - name: Extract source archive @@ -531,7 +531,7 @@ jobs: cp ../.asf.yaml ./ git add .nojekyll .asf.yaml - name: Download - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: name: release-html - name: Extract @@ -567,7 +567,7 @@ jobs: - ubuntu-latest steps: - name: Download release artifacts - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: pattern: release-* - name: Verify @@ -601,7 +601,7 @@ jobs: contents: write steps: - name: Download release artifacts - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: pattern: release-* path: artifacts From 973e974e15d5997bb55102d71e8b3f59f3ba7eea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Nov 2025 21:50:56 +0100 Subject: [PATCH 219/408] MINOR: Bump commons-codec:commons-codec from 1.18.0 to 1.19.0 (#871) Bumps [commons-codec:commons-codec](https://github.com/apache/commons-codec) from 1.18.0 to 1.19.0.
Changelog

Sourced from commons-codec:commons-codec's changelog.

Apache Commons Codec 1.19.0 Release Notes

The Apache Commons Codec team is pleased to announce the release of Apache Commons Codec 1.19.0.

The Apache Commons Codec component contains encoders and decoders for formats such as Base16, Base32, Base64, digest, and Hexadecimal. In addition to these widely used encoders and decoders, the codec package also maintains a collection of phonetic encoding utilities.

This is a feature and maintenance release. Java 8 or later is required.

New features

  •         Add HmacUtils.hmac(Path). Thanks to Gary Gregory.

  •         Add HmacUtils.hmacHex(Path). Thanks to Gary Gregory.

  •  Add PMD check to the default Maven goal. Thanks to Gary
Gregory.

  •  Add SpotBugs check to the default Maven goal. Thanks to Gary
Gregory.

Fixed Bugs

  •  Remove -nouses directive from maven-bundle-plugin. OSGi
package imports now state 'uses' definitions for package imports, this
doesn't affect JPMS (from org.apache.commons:commons-parent:80). Thanks
to Gary Gregory.

  •  Refactor DigestUtils.updateDigest(MessageDigest, File) to
use NIO. Thanks to Gary Gregory.
  • CODEC-328: Clarify Javadoc for org.apache.commons.codec.digest.UnixCrypt.crypt(byte[],String). Thanks to Gary Gregory.
  •  Precompile regular expressions in
DaitchMokotoffSoundex.Rule. Thanks to Gary Gregory.

  •  Precompile regular expressions in
DaitchMokotoffSoundex.parseRules(Scanner, String, Map, Map). Thanks to
Gary Gregory.

  •  Precompile regular expressions in
Lang.loadFromResource(String, Languages). Thanks to Gary Gregory.

  •  Precompile regular expressions in
PhoneticEngine.encode(String, LanguageSet). Thanks to Gary Gregory.

  •  Precompile regular expressions in
org.apache.commons.codec.language.bm.Rule.parse*(*). Thanks to Gary
Gregory.

  •  Remove redundant checks for whitespace in
DaitchMokotoffSoundex.soundex(String, boolean). Thanks to Gary Gregory.

  •  Javadoc typo in Base16.java
[#380](https://github.com/apache/commons-codec/issues/380). Thanks to
Sebastian Baunsgaard.

  •  Deprecate unused constant
org.apache.commons.codec.language.bm.Rule.ALL. Thanks to Gary Gregory.
  • CODEC-331: org.apache.commons.codec.language.bm.Rule.parsePhonemeExpr(String) adds duplicate empty phoneme when input ends with |. Thanks to IlikeCode, Gary Gregory.
  • CODEC-331: org.apache.commons.codec.language.DaitchMokotoffSoundex.cleanup(String) does not remove special characters like punctuation. Thanks to IlikeCode, Gary Gregory.
  •  Fix PMD multiple UnnecessaryFullyQualifiedName in
org.apache.commons.codec.binary.StringUtils. Thanks to Gary Gregory.

  •  Fix PMD UnusedFormalParameter in private constructor in
org.apache.commons.codec.binary.Base16. Thanks to Gary Gregory.

  •  Fix PMD multiple UnnecessaryFullyQualifiedName in
org.apache.commons.codec.digest.Blake3. Thanks to Gary Gregory.

  •  Fix PMD UnnecessaryFullyQualifiedName in
org.apache.commons.codec.digest.Md5Crypt. Thanks to Gary Gregory.

  •  Fix PMD EmptyControlStatement in
org.apache.commons.codec.language.Metaphone. Thanks to Gary Gregory.

  •  Fix SpotBugs [ERROR] Medium:
org.apache.commons.codec.binary.BaseNCodec$AbstractBuilder.setEncodeTable(byte[])
may expose internal representation by storing an externally mutable
object into BaseNCodec$AbstractBuilder.encodeTable
[org.apache.commons.codec.binary.BaseNCodec$AbstractBuilder] At
BaseNCodec.java:[line 131] EI_EXPOSE_REP2. Thanks to Gary Gregory.

  •  The method
org.apache.commons.codec.binary.BaseNCodec.AbstractBuilder.setLineSeparator(byte...)
now makes a defensive copy. Thanks to Gary Gregory.

  •  Avoid unnecessary String conversion in
org.apache.commons.codec.language.bm.PhoneticEngine.applyFinalRules(PhonemeBuilder,
Map). Thanks to Gary Gregory.

  •  Fix SpotBugs [ERROR] High: Potentially dangerous use of
non-short-circuit logic in
org.apache.commons.codec.language.DaitchMokotoffSoundex.cleanup(String)
[org.apache.commons.codec.language.DaitchMokotoffSoundex] At
DaitchMokotoffSoundex.java:[line 350] NS_DANGEROUS_NON_SHORT_CIRCUIT.
Thanks to Gary Gregory.

Changes

... (truncated)

Commits
  • 351cb22 Prepare for the release candidate 1.19.0 RC1
  • 0d501b6 Prepare for the next release candidate
  • d6d4b82 Refactor duplicate code
  • 6d6456c No need to exclude abstract test classes from Surefire runs
  • 22d62e4 No need to specify the default value for linkXref
  • c4daf34 No longer need to override the version of the Jacoco Maven plugin
  • 8f2b673 Remove workaround for [SUREFIRE-2253]
  • 466a61d Fix Javadoc
  • ca27bd3 Fix Checkstyle
  • 1dfb4e5 Better internal method name
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=commons-codec:commons-codec&package-manager=maven&previous-version=1.18.0&new-version=1.19.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- vector/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vector/pom.xml b/vector/pom.xml index 52ad5105ea..89e9779008 100644 --- a/vector/pom.xml +++ b/vector/pom.xml @@ -60,7 +60,7 @@ under the License. commons-codec commons-codec - 1.18.0 + 1.20.0 org.apache.arrow From 11414f652d90b56a697d26a18c81457132f502cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Nov 2025 06:50:17 +0100 Subject: [PATCH 220/408] MINOR: [CI] Bump actions/checkout from 4 to 5 (#820) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 5.
Release notes

Sourced from actions/checkout's releases.

v5.0.0

What's Changed

⚠️ Minimum Compatible Runner Version

v2.327.1
Release Notes

Make sure your runner is updated to this version or newer to use this release.

Full Changelog: https://github.com/actions/checkout/compare/v4...v5.0.0

v4.3.0

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v4...v4.3.0

v4.2.2

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v4.2.1...v4.2.2

v4.2.1

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v4.2.0...v4.2.1

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/checkout&package-manager=github_actions&previous-version=4&new-version=5)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: JB Onofré --- .github/workflows/dev.yml | 2 +- .github/workflows/dev_pr.yml | 2 +- .github/workflows/rc.yml | 20 ++++++++++---------- .github/workflows/release.yml | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 12968bbcca..27b874ddb9 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -33,7 +33,7 @@ jobs: name: "pre-commit" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml index 2f4a48d572..759383d37b 100644 --- a/.github/workflows/dev_pr.yml +++ b/.github/workflows/dev_pr.yml @@ -43,7 +43,7 @@ jobs: name: "Ensure PR format" runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@v5 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index b53812ef5e..121d188d3a 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -38,7 +38,7 @@ jobs: timeout-minutes: 5 steps: - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: submodules: recursive - name: Prepare for tag @@ -119,17 +119,17 @@ jobs: # ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ # if: github.event_name == 'schedule' - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: repository: apache/arrow path: arrow - name: Checkout apache/arrow-testing - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: repository: apache/arrow-testing path: arrow/testing - name: Checkout apache/parquet-testing - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing @@ -186,17 +186,17 @@ jobs: # ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ # if: github.event_name == 'schedule' - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: repository: apache/arrow path: arrow - name: Checkout apache/arrow-testing - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: repository: apache/arrow-testing path: arrow/testing - name: Checkout apache/parquet-testing - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing @@ -322,7 +322,7 @@ jobs: # ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ # if: github.event_name == 'schedule' - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: repository: apache/arrow path: arrow @@ -421,7 +421,7 @@ jobs: test -f jni/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll test -f jni/arrow_orc_jni/x86_64/arrow_orc_jni.dll - name: Checkout apache/arrow-testing - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: repository: apache/arrow-testing path: testing @@ -509,7 +509,7 @@ jobs: contents: write steps: - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: path: site - name: Prepare branch diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a955964cd8..d7a148bbbf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -65,7 +65,7 @@ jobs: $artifact done - name: Checkout for publishing docs - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: path: site - name: Publish docs From af37687fe501396f44561aca1ff0524c6d581a59 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Nov 2025 06:55:38 +0100 Subject: [PATCH 221/408] MINOR: [CI] Bump actions/upload-artifact from 4.6.2 to 5.0.0 (#894) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 4.6.2 to 5.0.0.
Release notes

Sourced from actions/upload-artifact's releases.

v5.0.0

What's Changed

BREAKING CHANGE: this update supports Node v24.x. This is not a breaking change per-se but we're treating it as such.

New Contributors

Full Changelog: https://github.com/actions/upload-artifact/compare/v4...v5.0.0

Commits
  • 330a01c Merge pull request #734 from actions/danwkennedy/prepare-5.0.0
  • 03f2824 Update github.dep.yml
  • 905a1ec Prepare v5.0.0
  • 2d9f9cd Merge pull request #725 from patrikpolyak/patch-1
  • 9687587 Merge branch 'main' into patch-1
  • 2848b2c Merge pull request #727 from danwkennedy/patch-1
  • 9b51177 Spell out the first use of GHES
  • cd231ca Update GHES guidance to include reference to Node 20 version
  • de65e23 Merge pull request #712 from actions/nebuk89-patch-1
  • 8747d8c Update README.md
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/upload-artifact&package-manager=github_actions&previous-version=4.6.2&new-version=5.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 121d188d3a..e21cae2d0b 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -71,7 +71,7 @@ jobs: run: | dev/release/run_rat.sh "${TAR_GZ}" - name: Upload source archive - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: release-source path: | @@ -154,7 +154,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf jni-linux-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: jni-linux-${{ matrix.platform.arch }} path: jni-linux-${{ matrix.platform.arch }}.tar.gz @@ -284,7 +284,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf jni-macos-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: jni-macos-${{ matrix.platform.arch }} path: jni-macos-${{ matrix.platform.arch }}.tar.gz @@ -368,7 +368,7 @@ jobs: shell: bash run: tar -cvzf jni-windows-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: jni-windows-${{ matrix.platform.arch }} path: jni-windows-${{ matrix.platform.arch }}.tar.gz @@ -440,12 +440,12 @@ jobs: cp -a target/site/apidocs reference tar -cvzf reference.tar.gz reference - name: Upload binaries - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: release-binaries path: binaries/* - name: Upload docs - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: reference path: reference.tar.gz @@ -483,7 +483,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf html.tar.gz -C docs/build html - name: Upload artifacts - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: release-html path: html.tar.gz From e366510ab96bdb4d18c48b26132d08382d67d33f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Nov 2025 18:13:27 +0100 Subject: [PATCH 222/408] MINOR: Bump checker.framework.version from 3.49.3 to 3.49.5 (#800) Bumps `checker.framework.version` from 3.49.3 to 3.49.5. Updates `org.checkerframework:checker-qual` from 3.49.3 to 3.49.5
Release notes

Sourced from org.checkerframework:checker-qual's releases.

Checker Framework 3.49.5

Version 3.49.5 (June 30, 2025)

User-visible changes:

The Checker Framework runs under JDK 25 -- that is, it runs on a version 25 JVM.

Closed issues:

#7093.

Checker Framework 3.49.4

Version 3.49.4 (June 2, 2025)

Closed issues:

#6740, #7013, #7038, #7070, #7082.

Changelog

Sourced from org.checkerframework:checker-qual's changelog.

Version 3.49.5 (June 30, 2025)

User-visible changes:

The Checker Framework runs under JDK 25 -- that is, it runs on a version 25 JVM.

Closed issues:

#7093.

Version 3.49.4 (June 2, 2025)

Closed issues:

#6740, #7013, #7038, #7070, #7082.

Commits
  • 7f6e237 new release 3.49.5
  • 57eee7e Prep for release.
  • 759bead Don't run inference job on JDK 11
  • 0d200f2 Produce Java 11 bytecodes for the wpi-many projects
  • 04fd6b4 Produce Java 11 bytecodes for the owning-field project
  • 9b51e58 Test under Java 25 (#7113)
  • 2403df2 Remove file SKIP-REQUIRE-JAVADOC
  • bbf907e Suppress "this-escape" warnings; in the future, carefully examine (#7130)
  • 76d3e74 Inferring type arguments is slow (#7125)
  • 3d9448e Warn about slow type-checking of method invocations (#7124)
  • Additional commits viewable in compare view

Updates `org.checkerframework:checker` from 3.49.3 to 3.49.5
Release notes

Sourced from org.checkerframework:checker's releases.

Checker Framework 3.49.5

Version 3.49.5 (June 30, 2025)

User-visible changes:

The Checker Framework runs under JDK 25 -- that is, it runs on a version 25 JVM.

Closed issues:

#7093.

Checker Framework 3.49.4

Version 3.49.4 (June 2, 2025)

Closed issues:

#6740, #7013, #7038, #7070, #7082.

Changelog

Sourced from org.checkerframework:checker's changelog.

Version 3.49.5 (June 30, 2025)

User-visible changes:

The Checker Framework runs under JDK 25 -- that is, it runs on a version 25 JVM.

Closed issues:

#7093.

Version 3.49.4 (June 2, 2025)

Closed issues:

#6740, #7013, #7038, #7070, #7082.

Commits
  • 7f6e237 new release 3.49.5
  • 57eee7e Prep for release.
  • 759bead Don't run inference job on JDK 11
  • 0d200f2 Produce Java 11 bytecodes for the wpi-many projects
  • 04fd6b4 Produce Java 11 bytecodes for the owning-field project
  • 9b51e58 Test under Java 25 (#7113)
  • 2403df2 Remove file SKIP-REQUIRE-JAVADOC
  • bbf907e Suppress "this-escape" warnings; in the future, carefully examine (#7130)
  • 76d3e74 Inferring type arguments is slow (#7125)
  • 3d9448e Warn about slow type-checking of method invocations (#7124)
  • Additional commits viewable in compare view

You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 41b82fabab..3e09f982fe 100644 --- a/pom.xml +++ b/pom.xml @@ -110,7 +110,7 @@ under the License. 10.23.0 true 2.37.0 - 3.49.3 + 3.52.0 1.5.21 none -Xdoclint:none From ca5c06bbb94343c2bf69d2397c5beb08b66e3a14 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Nov 2025 13:03:56 +0100 Subject: [PATCH 223/408] MINOR: Bump error_prone_core.version from 2.37.0 to 2.42.0 (#749) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps `error_prone_core.version` from 2.37.0 to 2.38.0. Updates `com.google.errorprone:error_prone_annotations` from 2.37.0 to 2.38.0
Release notes

Sourced from com.google.errorprone:error_prone_annotations's releases.

Error Prone 2.38.0

New checks:

Closed issues: #4924, #4897, #4995

Full changelog: https://github.com/google/error-prone/compare/v2.37.0...v2.38.0

Commits
  • a07bd3e Release Error Prone 2.38.0
  • 09fd394 Fix typo in NullTernary.md
  • 4171fd7 FindIdentifiers: find binding variables declared by enclosing or earlier if...
  • d78f515 Audit each use of ElementKind.LOCAL_VARIABLE, and add BINDING_VARIABLE if app...
  • 6f94a97 Tolerate default cases in switches as being present to handle version skew
  • 0223abb Support @LenientFormatString in LenientFormatStringValidation.
  • cb7dfaf Remove the Side enum.
  • d64c9ce Promote error prone check TestExceptionChecker to ERROR within Google (blaze ...
  • c0ce475 Move TargetType to a top-level class alongside ASTHelpers.
  • 90b8efb Allow binding to BINDING_VARIABLEs in GuardedByBinder.
  • Additional commits viewable in compare view

Updates `com.google.errorprone:error_prone_core` from 2.37.0 to 2.38.0
Release notes

Sourced from com.google.errorprone:error_prone_core's releases.

Error Prone 2.38.0

New checks:

Closed issues: #4924, #4897, #4995

Full changelog: https://github.com/google/error-prone/compare/v2.37.0...v2.38.0

Commits
  • a07bd3e Release Error Prone 2.38.0
  • 09fd394 Fix typo in NullTernary.md
  • 4171fd7 FindIdentifiers: find binding variables declared by enclosing or earlier if...
  • d78f515 Audit each use of ElementKind.LOCAL_VARIABLE, and add BINDING_VARIABLE if app...
  • 6f94a97 Tolerate default cases in switches as being present to handle version skew
  • 0223abb Support @LenientFormatString in LenientFormatStringValidation.
  • cb7dfaf Remove the Side enum.
  • d64c9ce Promote error prone check TestExceptionChecker to ERROR within Google (blaze ...
  • c0ce475 Move TargetType to a top-level class alongside ASTHelpers.
  • 90b8efb Allow binding to BINDING_VARIABLEs in GuardedByBinder.
  • Additional commits viewable in compare view

You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: JB Onofré Co-authored-by: JB Onofré --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3e09f982fe..7f070a2f01 100644 --- a/pom.xml +++ b/pom.xml @@ -109,7 +109,7 @@ under the License. 2 10.23.0 true - 2.37.0 + 2.42.0 3.52.0 1.5.21 none From ba5057a6b857ff42958ff69c479b098e61672719 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Nov 2025 14:17:01 +0100 Subject: [PATCH 224/408] MINOR: Bump org.apache.orc:orc-core from 2.1.1 to 2.2.1 (#874) Bumps org.apache.orc:orc-core from 2.1.1 to 2.2.1. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.orc:orc-core&package-manager=maven&previous-version=2.1.1&new-version=2.2.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- adapter/orc/pom.xml | 2 +- dataset/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/adapter/orc/pom.xml b/adapter/orc/pom.xml index e3ae7d5163..ef72b2de65 100644 --- a/adapter/orc/pom.xml +++ b/adapter/orc/pom.xml @@ -61,7 +61,7 @@ under the License. org.apache.orc orc-core - 2.1.1 + 2.2.1 test diff --git a/dataset/pom.xml b/dataset/pom.xml index 6e56d555b7..66233c3970 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -130,7 +130,7 @@ under the License. org.apache.orc orc-core - 2.1.1 + 2.2.1 test From 8ecbea604628679140d08a9783491ac5a4c20087 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Nov 2025 15:03:25 +0100 Subject: [PATCH 225/408] MINOR: Bump com.google.protobuf:protobuf-bom from 4.30.2 to 4.33.0 (#888) Bumps [com.google.protobuf:protobuf-bom](https://github.com/protocolbuffers/protobuf) from 4.30.2 to 4.33.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.protobuf:protobuf-bom&package-manager=maven&previous-version=4.30.2&new-version=4.33.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7f070a2f01..e402f4efe7 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,7 @@ under the License. 33.4.8-jre 4.1.127.Final 1.73.0 - 4.30.2 + 4.33.1 2.18.3 3.4.1 25.2.10 From 266dfc12151a49dd49153890eb16e97937c5667c Mon Sep 17 00:00:00 2001 From: Aleksei Starikov Date: Mon, 17 Nov 2025 01:12:50 +0100 Subject: [PATCH 226/408] GH-586: Override fixedSizeBinary method for UnionMapWriter (#885) ## What's Changed `UnionMapWriter` will null out the entire map struct entry instead of setting the value to null in: ``` childWriter.value().fixedSizeBinary().writeNull(); ``` This PR overrides the `fixedSizeBinary()` method for the `UnionMapWriter`, resolving it. In addition, it introduces the `fixedSizeBinary(int byteWidth)` which needs to be used to initialize `key` and `value` writes in `UnionMapWriter` if they have not been initialized. Closes #586. --- .../codegen/templates/UnionMapWriter.java | 23 ++ .../apache/arrow/vector/TestMapVector.java | 221 ++++++++++++++++++ 2 files changed, 244 insertions(+) diff --git a/vector/src/main/codegen/templates/UnionMapWriter.java b/vector/src/main/codegen/templates/UnionMapWriter.java index 8b2f091215..8bbf6ae0a4 100644 --- a/vector/src/main/codegen/templates/UnionMapWriter.java +++ b/vector/src/main/codegen/templates/UnionMapWriter.java @@ -243,4 +243,27 @@ public ExtensionWriter extension(ArrowType type) { return super.extension(type); } } + + public FixedSizeBinaryWriter fixedSizeBinary(int byteWidth) { + switch (mode) { + case KEY: + return entryWriter.fixedSizeBinary(MapVector.KEY_NAME, byteWidth); + case VALUE: + return entryWriter.fixedSizeBinary(MapVector.VALUE_NAME, byteWidth); + default: + return this; + } + } + + @Override + public FixedSizeBinaryWriter fixedSizeBinary() { + switch (mode) { + case KEY: + return entryWriter.fixedSizeBinary(MapVector.KEY_NAME); + case VALUE: + return entryWriter.fixedSizeBinary(MapVector.VALUE_NAME); + default: + return this; + } + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java index 1a1810d0f7..8605d250fd 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java @@ -16,10 +16,12 @@ */ package org.apache.arrow.vector; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.nio.ByteBuffer; @@ -41,6 +43,7 @@ import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; import org.apache.arrow.vector.complex.writer.FieldWriter; import org.apache.arrow.vector.holder.UuidHolder; +import org.apache.arrow.vector.holders.FixedSizeBinaryHolder; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; @@ -1359,4 +1362,222 @@ public void testCopyFromForExtensionType() throws Exception { assertEquals(u2, actualUuid); } } + + private FixedSizeBinaryHolder getFixedSizeBinaryHolder(byte[] array) { + FixedSizeBinaryHolder holder = new FixedSizeBinaryHolder(); + holder.byteWidth = array.length; + holder.buffer = allocator.buffer(array.length); + for (int i = 0; i < array.length; i++) { + holder.buffer.setByte(i, array[i]); + } + + return holder; + } + + /** + * Regression test for GH-586: UnionMapWriter.fixedSizeBinary() should properly delegate to the + * entry writer for both key and value paths. + */ + @Test + public void testFixedSizeBinaryWriter() { + try (MapVector mapVector = MapVector.empty("map_vector", allocator, false)) { + UnionMapWriter writer = mapVector.getWriter(); + writer.allocate(); + + // populate input vector with the following records + // {[11, 22] -> [32, 21]} + // {1 -> [11, 22], 2 -> [32, 21]} + // null + // {[11, 22] -> 1, [32, 21] -> 2} + // {[11, 22] -> null} + // {null -> [32, 21]} - wrong "for a given entry, the "key" is non-nullable" - todo: it + // shouldn't work. Should it? + FixedSizeBinaryHolder holder1 = getFixedSizeBinaryHolder(new byte[] {11, 22}); + FixedSizeBinaryHolder holder2 = getFixedSizeBinaryHolder(new byte[] {32, 21}); + + writer.setPosition(0); // optional + writer.startMap(); + writer.startEntry(); + writer + .key() + .fixedSizeBinary(holder1.byteWidth) + .write(holder1); // need to initialize with byteWidth - NPE otherwise + writer.value().fixedSizeBinary(holder2.byteWidth).write(holder2); + writer.endEntry(); + holder1.buffer.close(); + holder2.buffer.close(); + writer.endMap(); + + // {1 -> [11, 22], 2 -> [32, 21]} + holder1 = getFixedSizeBinaryHolder(new byte[] {11, 22}); + holder2 = getFixedSizeBinaryHolder(new byte[] {32, 21}); + writer.setPosition(1); + writer.startMap(); + writer.startEntry(); + writer.key().bigInt().writeBigInt(1); + writer.value().fixedSizeBinary().write(holder1); + writer.endEntry(); + holder1.buffer.close(); + writer.startEntry(); + writer.key().bigInt().writeBigInt(2); + writer.value().fixedSizeBinary().write(holder2); + writer.endEntry(); + writer.endMap(); + holder2.buffer.close(); + + // {[11, 22] -> 1, [32, 21] -> 2} + holder1 = getFixedSizeBinaryHolder(new byte[] {11, 22}); + holder2 = getFixedSizeBinaryHolder(new byte[] {32, 21}); + writer.setPosition(3); + writer.startMap(); + writer.startEntry(); + writer.key().fixedSizeBinary().write(holder1); + writer.value().bigInt().writeBigInt(1); + writer.endEntry(); + holder1.buffer.close(); + writer.startEntry(); + writer.key().fixedSizeBinary().write(holder2); + writer.value().bigInt().writeBigInt(2); + writer.endEntry(); + writer.endMap(); + holder2.buffer.close(); + + // {[11, 22] -> null} + holder1 = getFixedSizeBinaryHolder(new byte[] {11, 22}); + writer.setPosition(4); + writer.startMap(); + writer.startEntry(); + writer.key().fixedSizeBinary().write(holder1); + writer.endEntry(); + writer.endMap(); + holder1.buffer.close(); + + // {null -> [32, 21]} + holder2 = getFixedSizeBinaryHolder(new byte[] {32, 21}); + writer.setPosition(5); + writer.startMap(); + writer.startEntry(); + writer.value().fixedSizeBinary().write(holder2); + writer.endEntry(); + writer.endMap(); + holder2.buffer.close(); + + writer.setValueCount(6); + + // assert the output vector is correct + FieldReader reader = mapVector.getReader(); + assertTrue(reader.isSet(), "shouldn't be null"); + reader.setPosition(1); + assertTrue(reader.isSet(), "shouldn't be null"); + reader.setPosition(2); + assertFalse(reader.isSet(), "should be null"); + reader.setPosition(3); + assertTrue(reader.isSet(), "shouldn't be null"); + reader.setPosition(4); + assertTrue(reader.isSet(), "shouldn't be null"); + reader.setPosition(5); + assertTrue(reader.isSet(), "shouldn't be null"); + + /* index 0 */ + Object result = mapVector.getObject(0); + ArrayList resultSet = (ArrayList) result; + assertEquals(1, resultSet.size()); + Map resultStruct = (Map) resultSet.get(0); + assertTrue(resultStruct.containsKey(MapVector.KEY_NAME)); + assertTrue(resultStruct.containsKey(MapVector.VALUE_NAME)); + assertArrayEquals(new byte[] {11, 22}, (byte[]) resultStruct.get(MapVector.KEY_NAME)); + assertArrayEquals(new byte[] {32, 21}, (byte[]) resultStruct.get(MapVector.VALUE_NAME)); + + /* index 1 */ + result = mapVector.getObject(1); + resultSet = (ArrayList) result; + assertEquals(2, resultSet.size()); + resultStruct = (Map) resultSet.get(0); + assertEquals(1L, getResultKey(resultStruct)); + assertTrue(resultStruct.containsKey(MapVector.VALUE_NAME)); + assertArrayEquals(new byte[] {11, 22}, (byte[]) resultStruct.get(MapVector.VALUE_NAME)); + resultStruct = (Map) resultSet.get(1); + assertEquals(2L, getResultKey(resultStruct)); + assertTrue(resultStruct.containsKey(MapVector.VALUE_NAME)); + assertArrayEquals(new byte[] {32, 21}, (byte[]) resultStruct.get(MapVector.VALUE_NAME)); + + /* index 2 */ + result = mapVector.getObject(2); + assertNull(result); + + /* index 3 */ + result = mapVector.getObject(3); + resultSet = (ArrayList) result; + assertEquals(2, resultSet.size()); + resultStruct = (Map) resultSet.get(0); + assertTrue(resultStruct.containsKey(MapVector.KEY_NAME)); + assertArrayEquals(new byte[] {11, 22}, (byte[]) resultStruct.get(MapVector.KEY_NAME)); + assertEquals(1L, getResultValue(resultStruct)); + resultStruct = (Map) resultSet.get(1); + assertTrue(resultStruct.containsKey(MapVector.KEY_NAME)); + assertArrayEquals(new byte[] {32, 21}, (byte[]) resultStruct.get(MapVector.KEY_NAME)); + assertEquals(2L, getResultValue(resultStruct)); + + /* index 4 */ + result = mapVector.getObject(4); + resultSet = (ArrayList) result; + assertEquals(1, resultSet.size()); + resultStruct = (Map) resultSet.get(0); + assertTrue(resultStruct.containsKey(MapVector.KEY_NAME)); + assertArrayEquals(new byte[] {11, 22}, (byte[]) resultStruct.get(MapVector.KEY_NAME)); + assertFalse(resultStruct.containsKey(MapVector.VALUE_NAME)); + + /* index 5 */ + result = mapVector.getObject(5); + resultSet = (ArrayList) result; + assertEquals(1, resultSet.size()); + resultStruct = (Map) resultSet.get(0); + assertFalse(resultStruct.containsKey(MapVector.KEY_NAME)); + assertTrue(resultStruct.containsKey(MapVector.VALUE_NAME)); + assertArrayEquals(new byte[] {32, 21}, (byte[]) resultStruct.get(MapVector.VALUE_NAME)); + } + } + + @Test + public void testFixedSizeBinaryFirstInitialization() { + try (MapVector mapVector = MapVector.empty("map_vector", allocator, false)) { + UnionMapWriter writer = mapVector.getWriter(); + writer.allocate(); + + // populate input vector with the following records + // {[11, 22] -> [32, 21]} + FixedSizeBinaryHolder holder1 = getFixedSizeBinaryHolder(new byte[] {11, 22}); + FixedSizeBinaryHolder holder2 = getFixedSizeBinaryHolder(new byte[] {32, 21}); + + writer.setPosition(0); // optional + writer.startMap(); + writer.startEntry(); + // require byteWidth parameter for first-time initialization of `key` or `value` writers + assertThrows(NullPointerException.class, () -> writer.key().fixedSizeBinary().write(holder1)); + assertThrows( + NullPointerException.class, () -> writer.value().fixedSizeBinary().write(holder2)); + writer.key().fixedSizeBinary(holder1.byteWidth).write(holder1); + writer.value().fixedSizeBinary(holder2.byteWidth).write(holder2); + writer.endEntry(); + holder1.buffer.close(); + holder2.buffer.close(); + writer.endMap(); + + writer.setValueCount(1); + + // assert the output vector is correct + FieldReader reader = mapVector.getReader(); + assertTrue(reader.isSet(), "shouldn't be null"); + + /* index 0 */ + Object result = mapVector.getObject(0); + ArrayList resultSet = (ArrayList) result; + assertEquals(1, resultSet.size()); + Map resultStruct = (Map) resultSet.get(0); + assertTrue(resultStruct.containsKey(MapVector.KEY_NAME)); + assertTrue(resultStruct.containsKey(MapVector.VALUE_NAME)); + assertArrayEquals(new byte[] {11, 22}, (byte[]) resultStruct.get(MapVector.KEY_NAME)); + assertArrayEquals(new byte[] {32, 21}, (byte[]) resultStruct.get(MapVector.VALUE_NAME)); + } + } } From 382790df634a9237403e6c0a2195dd0405766ef2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 15:32:54 +0900 Subject: [PATCH 227/408] MINOR: [CI] Bump actions/checkout from 5 to 6 (#911) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/checkout](https://github.com/actions/checkout) from 5 to 6.
Release notes

Sourced from actions/checkout's releases.

v6.0.0

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v5.0.0...v6.0.0

v6-beta

What's Changed

Updated persist-credentials to store the credentials under $RUNNER_TEMP instead of directly in the local git config.

This requires a minimum Actions Runner version of v2.329.0 to access the persisted credentials for Docker container action scenarios.

v5.0.1

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v5...v5.0.1

Changelog

Sourced from actions/checkout's changelog.

Changelog

V6.0.0

V5.0.1

V5.0.0

V4.3.1

V4.3.0

v4.2.2

v4.2.1

v4.2.0

v4.1.7

v4.1.6

v4.1.5

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/checkout&package-manager=github_actions&previous-version=5&new-version=6)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/dev.yml | 2 +- .github/workflows/dev_pr.yml | 2 +- .github/workflows/rc.yml | 20 ++++++++++---------- .github/workflows/release.yml | 2 +- .github/workflows/test.yml | 20 ++++++++++---------- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 27b874ddb9..57432d75f1 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -33,7 +33,7 @@ jobs: name: "pre-commit" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml index 759383d37b..84740628ee 100644 --- a/.github/workflows/dev_pr.yml +++ b/.github/workflows/dev_pr.yml @@ -43,7 +43,7 @@ jobs: name: "Ensure PR format" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index e21cae2d0b..0ae4399ef1 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -38,7 +38,7 @@ jobs: timeout-minutes: 5 steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: submodules: recursive - name: Prepare for tag @@ -119,17 +119,17 @@ jobs: # ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ # if: github.event_name == 'schedule' - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: repository: apache/arrow path: arrow - name: Checkout apache/arrow-testing - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: repository: apache/arrow-testing path: arrow/testing - name: Checkout apache/parquet-testing - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing @@ -186,17 +186,17 @@ jobs: # ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ # if: github.event_name == 'schedule' - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: repository: apache/arrow path: arrow - name: Checkout apache/arrow-testing - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: repository: apache/arrow-testing path: arrow/testing - name: Checkout apache/parquet-testing - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing @@ -322,7 +322,7 @@ jobs: # ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ # if: github.event_name == 'schedule' - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: repository: apache/arrow path: arrow @@ -421,7 +421,7 @@ jobs: test -f jni/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll test -f jni/arrow_orc_jni/x86_64/arrow_orc_jni.dll - name: Checkout apache/arrow-testing - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: repository: apache/arrow-testing path: testing @@ -509,7 +509,7 @@ jobs: contents: write steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: path: site - name: Prepare branch diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d7a148bbbf..5ca9cf9b73 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -65,7 +65,7 @@ jobs: $artifact done - name: Checkout for publishing docs - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: path: site - name: Publish docs diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 65fbc262fc..8d8fcb5052 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,7 +58,7 @@ jobs: MAVEN: ${{ matrix.maven }} steps: - name: Checkout Arrow - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: 0 submodules: recursive @@ -100,7 +100,7 @@ jobs: distribution: 'temurin' java-version: ${{ matrix.jdk }} - name: Checkout Arrow - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: 0 submodules: recursive @@ -131,7 +131,7 @@ jobs: java-version: ${{ matrix.jdk }} distribution: 'temurin' - name: Checkout Arrow - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: 0 submodules: recursive @@ -152,37 +152,37 @@ jobs: timeout-minutes: 60 steps: - name: Checkout Arrow - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: 0 repository: apache/arrow submodules: recursive - name: Checkout Arrow Rust - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: repository: apache/arrow-rs path: rust - name: Checkout Arrow nanoarrow - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: repository: apache/arrow-nanoarrow path: nanoarrow - name: Checkout Arrow .NET - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: repository: apache/arrow-dotnet path: dotnet - name: Checkout Arrow Go - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: repository: apache/arrow-go path: go - name: Checkout Arrow Java - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: path: java - name: Checkout Arrow JavaScript - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: repository: apache/arrow-js path: js From 4d7cf041d0abeae5a75d1c59cfdf59e12f9db686 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Nov 2025 18:20:37 +0100 Subject: [PATCH 228/408] MINOR: Bump org.codehaus.mojo:versions-maven-plugin from 2.18.0 to 2.20.0 (#912) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.codehaus.mojo:versions-maven-plugin](https://github.com/mojohaus/versions) from 2.18.0 to 2.20.0.
Release notes

Sourced from org.codehaus.mojo:versions-maven-plugin's releases.

2.20.0

🚀 New features and improvements

🐛 Bug Fixes

📝 Documentation updates

👻 Maintenance

📦 Dependency updates

2.19.1

🐛 Bug Fixes

... (truncated)

Commits
  • 2467d99 [maven-release-plugin] prepare release 2.20.0
  • 4c240e7 Bump org.apache.commons:commons-lang3 from 3.19.0 to 3.20.0
  • 6d64537 Bump byteBuddyVersion from 1.18.0 to 1.18.1
  • 7736ca6 Bump org.codehaus.plexus:plexus-archiver from 4.10.3 to 4.10.4
  • 37a5330 Bump byteBuddyVersion from 1.17.7 to 1.18.0
  • edeb5e7 Bump commons-codec:commons-codec from 1.19.0 to 1.20.0
  • 88874e0 Bump commons-io:commons-io from 2.20.0 to 2.21.0
  • 6769f03 Bump org.codehaus.plexus:plexus-i18n from 1.0.0 to 1.1.0
  • 9c122de Bump org.codehaus.plexus:plexus-interactivity-api from 1.4 to 1.5.1
  • 3f51967 Bump org.apache.maven.plugin-testing:maven-plugin-testing-harness
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.codehaus.mojo:versions-maven-plugin&package-manager=maven&previous-version=2.18.0&new-version=2.20.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- bom/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index 655c33b813..9efde53243 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -208,7 +208,7 @@ under the License. org.codehaus.mojo versions-maven-plugin - 2.18.0 + 2.20.0 diff --git a/pom.xml b/pom.xml index e402f4efe7..79b12dcb46 100644 --- a/pom.xml +++ b/pom.xml @@ -510,7 +510,7 @@ under the License. org.codehaus.mojo versions-maven-plugin - 2.18.0 + 2.20.0 pl.project13.maven From 8577cff9257afd9dc1561cbab6d250ee374dbcaf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Nov 2025 18:33:24 +0100 Subject: [PATCH 229/408] MINOR: Bump org.bouncycastle:bcpkix-jdk18on from 1.80 to 1.82 (#919) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.bouncycastle:bcpkix-jdk18on](https://github.com/bcgit/bc-java) from 1.80 to 1.82.
Changelog

Sourced from org.bouncycastle:bcpkix-jdk18on's changelog.

2.1.1 Version Release: 1.83 Date:      TBD

2.2.1 Version Release: 1.82 Date:      2025, 17th September.

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.bouncycastle:bcpkix-jdk18on&package-manager=maven&previous-version=1.80&new-version=1.82)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index d8e012101c..965e071e72 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -140,7 +140,7 @@ under the License. org.bouncycastle bcpkix-jdk18on - 1.80 + 1.82 From 13f1bd28b5a5fa9f6c2f8b52c108d6bf13bc0e63 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Nov 2025 19:05:52 +0100 Subject: [PATCH 230/408] MINOR: Bump org.apache.drill.tools:drill-fmpp-maven-plugin from 1.21.2 to 1.22.0 (#918) Bumps org.apache.drill.tools:drill-fmpp-maven-plugin from 1.21.2 to 1.22.0. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.drill.tools:drill-fmpp-maven-plugin&package-manager=maven&previous-version=1.21.2&new-version=1.22.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 79b12dcb46..5d2567beef 100644 --- a/pom.xml +++ b/pom.xml @@ -525,7 +525,7 @@ under the License. org.apache.drill.tools drill-fmpp-maven-plugin - 1.21.2 + 1.22.0 From 82baf1828df6e4ed40e8f2b62a3648235ffca76b Mon Sep 17 00:00:00 2001 From: Joana Hrotko Date: Fri, 28 Nov 2025 12:05:53 +0000 Subject: [PATCH 231/408] GH-825: Add UUID canonical extension type (#903) --- .../arrow/vector/BaseFixedWidthVector.java | 1 + .../apache/arrow/vector/FixedWidthVector.java | 3 + .../org/apache/arrow/vector/UuidVector.java | 481 ++++++++++++++++++ .../impl/ExtensionTypeWriterFactory.java | 4 +- .../vector/complex/impl/UuidReaderImpl.java | 35 +- .../complex/impl/UuidWriterFactory.java | 14 + .../vector/complex/impl/UuidWriterImpl.java | 41 +- .../arrow/vector/extension/UuidType.java | 109 ++++ .../vector/holders/NullableUuidHolder.java | 35 ++ .../arrow/vector/holders}/UuidHolder.java | 21 +- .../apache/arrow/vector/util/UuidUtility.java | 77 +++ .../apache/arrow/vector/TestListVector.java | 18 +- .../apache/arrow/vector/TestMapVector.java | 18 +- .../apache/arrow/vector/TestStructVector.java | 2 +- .../org/apache/arrow/vector/TestUtils.java | 11 + .../org/apache/arrow/vector/TestUuidType.java | 275 ++++++++++ .../apache/arrow/vector/TestUuidVector.java | 451 ++++++++++++++++ .../org/apache/arrow/vector/UuidVector.java | 127 ----- .../complex/impl/TestComplexCopier.java | 2 +- .../complex/impl/TestPromotableWriter.java | 2 +- .../complex/writer/TestComplexWriter.java | 8 +- .../complex/writer/TestSimpleWriter.java | 40 -- .../vector/types/pojo/TestExtensionType.java | 18 +- .../arrow/vector/types/pojo/UuidType.java | 60 --- 24 files changed, 1571 insertions(+), 282 deletions(-) create mode 100644 vector/src/main/java/org/apache/arrow/vector/UuidVector.java rename vector/src/{test => main}/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java (61%) rename vector/src/{test => main}/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java (72%) rename vector/src/{test => main}/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java (51%) create mode 100644 vector/src/main/java/org/apache/arrow/vector/extension/UuidType.java create mode 100644 vector/src/main/java/org/apache/arrow/vector/holders/NullableUuidHolder.java rename vector/src/{test/java/org/apache/arrow/vector/holder => main/java/org/apache/arrow/vector/holders}/UuidHolder.java (62%) create mode 100644 vector/src/main/java/org/apache/arrow/vector/util/UuidUtility.java create mode 100644 vector/src/test/java/org/apache/arrow/vector/TestUuidType.java create mode 100644 vector/src/test/java/org/apache/arrow/vector/TestUuidVector.java delete mode 100644 vector/src/test/java/org/apache/arrow/vector/UuidVector.java delete mode 100644 vector/src/test/java/org/apache/arrow/vector/types/pojo/UuidType.java diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java index f6e2a3b225..df1ac74f9b 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java @@ -70,6 +70,7 @@ public BaseFixedWidthVector(Field field, final BufferAllocator allocator, final refreshValueCapacity(); } + @Override public int getTypeWidth() { return typeWidth; } diff --git a/vector/src/main/java/org/apache/arrow/vector/FixedWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/FixedWidthVector.java index e22a973f3b..61a5574898 100644 --- a/vector/src/main/java/org/apache/arrow/vector/FixedWidthVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/FixedWidthVector.java @@ -31,4 +31,7 @@ public interface FixedWidthVector extends ElementAddressableVector { /** Zero out the underlying buffer backing this vector. */ void zeroVector(); + + /** Get the width of the type in bytes. */ + int getTypeWidth(); } diff --git a/vector/src/main/java/org/apache/arrow/vector/UuidVector.java b/vector/src/main/java/org/apache/arrow/vector/UuidVector.java new file mode 100644 index 0000000000..c662a6e064 --- /dev/null +++ b/vector/src/main/java/org/apache/arrow/vector/UuidVector.java @@ -0,0 +1,481 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector; + +import static org.apache.arrow.vector.extension.UuidType.UUID_BYTE_WIDTH; + +import java.nio.ByteBuffer; +import java.util.UUID; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.util.ArrowBufPointer; +import org.apache.arrow.memory.util.hash.ArrowBufHasher; +import org.apache.arrow.vector.complex.impl.UuidReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.extension.UuidType; +import org.apache.arrow.vector.holders.NullableUuidHolder; +import org.apache.arrow.vector.holders.UuidHolder; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.TransferPair; +import org.apache.arrow.vector.util.UuidUtility; + +/** + * Vector implementation for UUID values using {@link UuidType}. + * + *

Supports setting and retrieving UUIDs with efficient storage and nullable value handling. + * + *

Usage: + * + * 

{@code
+ * UuidVector vector = new UuidVector("uuid_col", allocator);
+ * vector.set(0, UUID.randomUUID());
+ * UUID value = vector.getObject(0);
+ * }
+ * + * @see UuidType + * @see UuidHolder + * @see NullableUuidHolder + */ +public class UuidVector extends ExtensionTypeVector + implements ValueIterableVector, FixedWidthVector { + private final Field field; + + /** The fixed byte width of UUID values (16 bytes). */ + public static final int TYPE_WIDTH = UUID_BYTE_WIDTH; + + /** + * Constructs a UUID vector with the given name, allocator, and underlying vector. + * + * @param name the name of the vector + * @param allocator the buffer allocator + * @param underlyingVector the underlying FixedSizeBinaryVector for storage + */ + public UuidVector( + String name, BufferAllocator allocator, FixedSizeBinaryVector underlyingVector) { + super(name, allocator, underlyingVector); + this.field = new Field(name, FieldType.nullable(new UuidType()), null); + } + + /** + * Constructs a UUID vector with the given name, field type, allocator, and underlying vector. + * + * @param name the name of the vector + * @param fieldType the field type (should contain UuidType) + * @param allocator the buffer allocator + * @param underlyingVector the underlying FixedSizeBinaryVector for storage + */ + public UuidVector( + String name, + FieldType fieldType, + BufferAllocator allocator, + FixedSizeBinaryVector underlyingVector) { + super(name, allocator, underlyingVector); + this.field = new Field(name, fieldType, null); + } + + /** + * Constructs a UUID vector with the given name and allocator. + * + *

Creates a new underlying FixedSizeBinaryVector with 16-byte width. + * + * @param name the name of the vector + * @param allocator the buffer allocator + */ + public UuidVector(String name, BufferAllocator allocator) { + super(name, allocator, new FixedSizeBinaryVector(name, allocator, UUID_BYTE_WIDTH)); + this.field = new Field(name, FieldType.nullable(new UuidType()), null); + } + + /** + * Constructs a UUID vector from a field and allocator. + * + * @param field the field definition (should contain UuidType) + * @param allocator the buffer allocator + */ + public UuidVector(Field field, BufferAllocator allocator) { + super( + field.getName(), + allocator, + new FixedSizeBinaryVector(field.getName(), allocator, UUID_BYTE_WIDTH)); + this.field = field; + } + + @Override + public UUID getObject(int index) { + if (isSet(index) == 0) { + return null; + } + final ByteBuffer bb = ByteBuffer.wrap(getUnderlyingVector().getObject(index)); + return new UUID(bb.getLong(), bb.getLong()); + } + + @Override + public int hashCode(int index) { + return hashCode(index, null); + } + + @Override + public int hashCode(int index, ArrowBufHasher hasher) { + return getUnderlyingVector().hashCode(index, hasher); + } + + /** + * Checks if the value at the given index is set (non-null). + * + * @param index the index to check + * @return 1 if the value is set, 0 if null + */ + public int isSet(int index) { + return getUnderlyingVector().isSet(index); + } + + /** + * Gets the UUID value at the given index as an ArrowBuf. + * + * @param index the index to retrieve + * @return a buffer slice containing the 16-byte UUID + * @throws IllegalStateException if the value at the index is null and null checking is enabled + */ + public ArrowBuf get(int index) throws IllegalStateException { + if (NullCheckingForGet.NULL_CHECKING_ENABLED && this.isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } else { + return getBufferSlicePostNullCheck(index); + } + } + + /** + * Reads the UUID value at the given index into a NullableUuidHolder. + * + * @param index the index to read from + * @param holder the holder to populate with the UUID data + */ + public void get(int index, NullableUuidHolder holder) { + if (NullCheckingForGet.NULL_CHECKING_ENABLED && this.isSet(index) == 0) { + holder.isSet = 0; + } else { + holder.isSet = 1; + holder.buffer = getBufferSlicePostNullCheck(index); + } + } + + /** + * Reads the UUID value at the given index into a UuidHolder. + * + * @param index the index to read from + * @param holder the holder to populate with the UUID data + */ + public void get(int index, UuidHolder holder) { + holder.isSet = 1; + holder.buffer = getBufferSlicePostNullCheck(index); + } + + /** + * Sets the UUID value at the given index. + * + * @param index the index to set + * @param value the UUID value to set, or null to set a null value + */ + public void set(int index, UUID value) { + if (value != null) { + set(index, UuidUtility.getBytesFromUUID(value)); + } else { + getUnderlyingVector().setNull(index); + } + } + + /** + * Sets the UUID value at the given index from a UuidHolder. + * + * @param index the index to set + * @param holder the holder containing the UUID data + */ + public void set(int index, UuidHolder holder) { + this.set(index, holder.isSet, holder.buffer); + } + + /** + * Sets the UUID value at the given index from a NullableUuidHolder. + * + * @param index the index to set + * @param holder the holder containing the UUID data + */ + public void set(int index, NullableUuidHolder holder) { + this.set(index, holder.isSet, holder.buffer); + } + + /** + * Sets the UUID value at the given index with explicit null flag. + * + * @param index the index to set + * @param isSet 1 if the value is set, 0 if null + * @param buffer the buffer containing the 16-byte UUID data + */ + public void set(int index, int isSet, ArrowBuf buffer) { + getUnderlyingVector().set(index, isSet, buffer); + } + + /** + * Sets the UUID value at the given index from an ArrowBuf. + * + * @param index the index to set + * @param value the buffer containing the 16-byte UUID data + */ + public void set(int index, ArrowBuf value) { + getUnderlyingVector().set(index, value); + } + + /** + * Sets the UUID value at the given index by copying from a source buffer. + * + * @param index the index to set + * @param source the source buffer to copy from + * @param sourceOffset the offset in the source buffer where the UUID data starts + */ + public void set(int index, ArrowBuf source, int sourceOffset) { + // Copy bytes from source buffer to target vector data buffer + ArrowBuf dataBuffer = getUnderlyingVector().getDataBuffer(); + dataBuffer.setBytes((long) index * UUID_BYTE_WIDTH, source, sourceOffset, UUID_BYTE_WIDTH); + getUnderlyingVector().setIndexDefined(index); + } + + /** + * Sets the UUID value at the given index from a byte array. + * + * @param index the index to set + * @param value the 16-byte array containing the UUID data + */ + public void set(int index, byte[] value) { + getUnderlyingVector().set(index, value); + } + + /** + * Sets the UUID value at the given index, expanding capacity if needed. + * + * @param index the index to set + * @param value the UUID value to set, or null to set a null value + */ + public void setSafe(int index, UUID value) { + if (value != null) { + setSafe(index, UuidUtility.getBytesFromUUID(value)); + } else { + getUnderlyingVector().setNull(index); + } + } + + /** + * Sets the UUID value at the given index from a NullableUuidHolder, expanding capacity if needed. + * + * @param index the index to set + * @param holder the holder containing the UUID data, or null to set a null value + */ + public void setSafe(int index, NullableUuidHolder holder) { + if (holder != null) { + getUnderlyingVector().setSafe(index, holder.isSet, holder.buffer); + } else { + getUnderlyingVector().setNull(index); + } + } + + /** + * Sets the UUID value at the given index from a UuidHolder, expanding capacity if needed. + * + * @param index the index to set + * @param holder the holder containing the UUID data, or null to set a null value + */ + public void setSafe(int index, UuidHolder holder) { + if (holder != null) { + getUnderlyingVector().setSafe(index, holder.isSet, holder.buffer); + } else { + getUnderlyingVector().setNull(index); + } + } + + /** + * Sets the UUID value at the given index from a byte array, expanding capacity if needed. + * + * @param index the index to set + * @param value the 16-byte array containing the UUID data + */ + public void setSafe(int index, byte[] value) { + getUnderlyingVector().setIndexDefined(index); + getUnderlyingVector().setSafe(index, value); + } + + /** + * Sets the UUID value at the given index from an ArrowBuf, expanding capacity if needed. + * + * @param index the index to set + * @param value the buffer containing the 16-byte UUID data + */ + public void setSafe(int index, ArrowBuf value) { + getUnderlyingVector().setSafe(index, value); + } + + @Override + public void copyFrom(int fromIndex, int thisIndex, ValueVector from) { + getUnderlyingVector() + .copyFromSafe(fromIndex, thisIndex, ((UuidVector) from).getUnderlyingVector()); + } + + @Override + public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { + getUnderlyingVector() + .copyFromSafe(fromIndex, thisIndex, ((UuidVector) from).getUnderlyingVector()); + } + + @Override + public Field getField() { + return field; + } + + @Override + public ArrowBufPointer getDataPointer(int i) { + return getUnderlyingVector().getDataPointer(i); + } + + @Override + public ArrowBufPointer getDataPointer(int i, ArrowBufPointer arrowBufPointer) { + return getUnderlyingVector().getDataPointer(i, arrowBufPointer); + } + + @Override + public void allocateNew(int valueCount) { + getUnderlyingVector().allocateNew(valueCount); + } + + @Override + public void zeroVector() { + getUnderlyingVector().zeroVector(); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((UuidVector) to); + } + + @Override + protected FieldReader getReaderImpl() { + return new UuidReaderImpl(this); + } + + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator) { + return new TransferImpl(field, allocator); + } + + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator, CallBack callBack) { + return getTransferPair(field, allocator); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return getTransferPair(ref, allocator); + } + + @Override + public TransferPair getTransferPair(BufferAllocator allocator) { + return getTransferPair(this.getField().getName(), allocator); + } + + private ArrowBuf getBufferSlicePostNullCheck(int index) { + return getUnderlyingVector() + .getDataBuffer() + .slice((long) index * UUID_BYTE_WIDTH, UUID_BYTE_WIDTH); + } + + @Override + public int getTypeWidth() { + return getUnderlyingVector().getTypeWidth(); + } + + /** {@link TransferPair} for {@link UuidVector}. */ + public class TransferImpl implements TransferPair { + UuidVector to; + + /** + * Constructs a transfer pair with the given target vector. + * + * @param to the target UUID vector + */ + public TransferImpl(UuidVector to) { + this.to = to; + } + + /** + * Constructs a transfer pair, creating a new target vector from the field and allocator. + * + * @param field the field definition for the target vector + * @param allocator the buffer allocator for the target vector + */ + public TransferImpl(Field field, BufferAllocator allocator) { + this.to = new UuidVector(field, allocator); + } + + /** + * Constructs a transfer pair, creating a new target vector with the given name and allocator. + * + * @param ref the name for the target vector + * @param allocator the buffer allocator for the target vector + */ + public TransferImpl(String ref, BufferAllocator allocator) { + this.to = new UuidVector(ref, allocator); + } + + /** + * Gets the target vector of this transfer pair. + * + * @return the target UUID vector + */ + public UuidVector getTo() { + return this.to; + } + + /** Transfers ownership of data from the source vector to the target vector. */ + public void transfer() { + getUnderlyingVector().transferTo(to.getUnderlyingVector()); + } + + /** + * Splits and transfers a range of values from the source vector to the target vector. + * + * @param startIndex the starting index in the source vector + * @param length the number of values to transfer + */ + public void splitAndTransfer(int startIndex, int length) { + getUnderlyingVector().splitAndTransferTo(startIndex, length, to.getUnderlyingVector()); + } + + /** + * Copies a value from the source vector to the target vector, expanding capacity if needed. + * + * @param fromIndex the index in the source vector + * @param toIndex the index in the target vector + */ + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, (ValueVector) UuidVector.this); + } + } +} diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/ExtensionTypeWriterFactory.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/ExtensionTypeWriterFactory.java index 09f0314c5f..a01d591555 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/impl/ExtensionTypeWriterFactory.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/ExtensionTypeWriterFactory.java @@ -20,8 +20,8 @@ import org.apache.arrow.vector.complex.writer.FieldWriter; /** - * A factory interface for creating instances of {@link ExtensionTypeWriter}. This factory allows - * configuring writer implementations for specific {@link ExtensionTypeVector}. + * A factory interface for creating instances of {@link AbstractExtensionTypeWriter}. This factory + * allows configuring writer implementations for specific {@link ExtensionTypeVector}. * * @param the type of writer implementation for a specific {@link ExtensionTypeVector}. */ diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java similarity index 61% rename from vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java rename to vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java index 6b98d3b340..bb35b960d3 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java @@ -17,15 +17,30 @@ package org.apache.arrow.vector.complex.impl; import org.apache.arrow.vector.UuidVector; -import org.apache.arrow.vector.holder.UuidHolder; import org.apache.arrow.vector.holders.ExtensionHolder; +import org.apache.arrow.vector.holders.NullableUuidHolder; +import org.apache.arrow.vector.holders.UuidHolder; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.Field; +/** + * Reader implementation for {@link UuidVector}. + * + *

Provides methods to read UUID values from a vector, including support for reading into {@link + * UuidHolder} and retrieving values as {@link java.util.UUID} objects. + * + * @see UuidVector + * @see org.apache.arrow.vector.extension.UuidType + */ public class UuidReaderImpl extends AbstractFieldReader { private final UuidVector vector; + /** + * Constructs a reader for the given UUID vector. + * + * @param vector the UUID vector to read from + */ public UuidReaderImpl(UuidVector vector) { super(); this.vector = vector; @@ -48,12 +63,26 @@ public boolean isSet() { @Override public void read(ExtensionHolder holder) { - vector.get(idx(), (UuidHolder) holder); + if (holder instanceof UuidHolder) { + vector.get(idx(), (UuidHolder) holder); + } else if (holder instanceof NullableUuidHolder) { + vector.get(idx(), (NullableUuidHolder) holder); + } else { + throw new IllegalArgumentException( + "Unsupported holder type for UuidReader: " + holder.getClass()); + } } @Override public void read(int arrayIndex, ExtensionHolder holder) { - vector.get(arrayIndex, (UuidHolder) holder); + if (holder instanceof UuidHolder) { + vector.get(arrayIndex, (UuidHolder) holder); + } else if (holder instanceof NullableUuidHolder) { + vector.get(arrayIndex, (NullableUuidHolder) holder); + } else { + throw new IllegalArgumentException( + "Unsupported holder type for UuidReader: " + holder.getClass()); + } } @Override diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java similarity index 72% rename from vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java rename to vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java index 1b1bf4e6e4..35988129cb 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java @@ -19,8 +19,22 @@ import org.apache.arrow.vector.ExtensionTypeVector; import org.apache.arrow.vector.UuidVector; +/** + * Factory for creating {@link UuidWriterImpl} instances. + * + *

This factory is used to create writers for UUID extension type vectors. + * + * @see UuidWriterImpl + * @see org.apache.arrow.vector.extension.UuidType + */ public class UuidWriterFactory implements ExtensionTypeWriterFactory { + /** + * Creates a writer implementation for the given extension type vector. + * + * @param extensionTypeVector the vector to create a writer for + * @return a {@link UuidWriterImpl} if the vector is a {@link UuidVector}, null otherwise + */ @Override public AbstractFieldWriter getWriterImpl(ExtensionTypeVector extensionTypeVector) { if (extensionTypeVector instanceof UuidVector) { diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java similarity index 51% rename from vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java rename to vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java index 68029b1df5..8a78add11c 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java @@ -16,32 +16,53 @@ */ package org.apache.arrow.vector.complex.impl; -import java.nio.ByteBuffer; -import java.util.UUID; +import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.vector.UuidVector; -import org.apache.arrow.vector.holder.UuidHolder; import org.apache.arrow.vector.holders.ExtensionHolder; +import org.apache.arrow.vector.holders.NullableUuidHolder; +import org.apache.arrow.vector.holders.UuidHolder; +/** + * Writer implementation for {@link UuidVector}. + * + *

Supports writing UUID values in multiple formats: {@link java.util.UUID}, byte arrays, and + * {@link ArrowBuf}. Also handles {@link UuidHolder} and {@link NullableUuidHolder}. + * + * @see UuidVector + * @see org.apache.arrow.vector.extension.UuidType + */ public class UuidWriterImpl extends AbstractExtensionTypeWriter { + /** + * Constructs a writer for the given UUID vector. + * + * @param vector the UUID vector to write to + */ public UuidWriterImpl(UuidVector vector) { super(vector); } @Override public void writeExtension(Object value) { - UUID uuid = (UUID) value; - ByteBuffer bb = ByteBuffer.allocate(16); - bb.putLong(uuid.getMostSignificantBits()); - bb.putLong(uuid.getLeastSignificantBits()); - vector.setSafe(getPosition(), bb.array()); + if (value instanceof byte[]) { + vector.setSafe(getPosition(), (byte[]) value); + } else if (value instanceof ArrowBuf) { + vector.setSafe(getPosition(), (ArrowBuf) value); + } else if (value instanceof java.util.UUID) { + vector.setSafe(getPosition(), (java.util.UUID) value); + } else { + throw new IllegalArgumentException("Unsupported value type for UUID: " + value.getClass()); + } vector.setValueCount(getPosition() + 1); } @Override public void write(ExtensionHolder holder) { - UuidHolder uuidHolder = (UuidHolder) holder; - vector.setSafe(getPosition(), uuidHolder.value); + if (holder instanceof UuidHolder) { + vector.setSafe(getPosition(), (UuidHolder) holder); + } else if (holder instanceof NullableUuidHolder) { + vector.setSafe(getPosition(), (NullableUuidHolder) holder); + } vector.setValueCount(getPosition() + 1); } } diff --git a/vector/src/main/java/org/apache/arrow/vector/extension/UuidType.java b/vector/src/main/java/org/apache/arrow/vector/extension/UuidType.java new file mode 100644 index 0000000000..f0f2636c82 --- /dev/null +++ b/vector/src/main/java/org/apache/arrow/vector/extension/UuidType.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.extension; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.FixedSizeBinaryVector; +import org.apache.arrow.vector.UuidVector; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ArrowType.ExtensionType; +import org.apache.arrow.vector.types.pojo.ExtensionTypeRegistry; +import org.apache.arrow.vector.types.pojo.FieldType; + +/** + * Extension type for UUID (Universally Unique Identifier) values. + * + *

UUIDs are stored as 16-byte fixed-size binary values. This extension type provides a + * standardized way to represent UUIDs in Arrow, making them interoperable across different systems + * and languages.π + * + *

The extension name is "arrow.uuid" and it uses {@link ArrowType.FixedSizeBinary} with 16 bytes + * as the storage type. + * + *

Usage: + * + * 

{@code
+ * UuidVector vector = new UuidVector("uuid_col", allocator);
+ * vector.set(0, UUID.randomUUID());
+ * UUID value = vector.getObject(0);
+ * }
+ * + * @see UuidVector + * @see org.apache.arrow.vector.holders.UuidHolder + * @see org.apache.arrow.vector.holders.NullableUuidHolder + */ +public class UuidType extends ExtensionType { + /** Singleton instance of UuidType. */ + public static final UuidType INSTANCE = new UuidType(); + + /** Extension name registered in the Arrow extension type registry. */ + public static final String EXTENSION_NAME = "arrow.uuid"; + + /** Number of bytes used to store a UUID (128 bits = 16 bytes). */ + public static final int UUID_BYTE_WIDTH = 16; + + /** Number of characters in the standard UUID string representation (with hyphens). */ + public static final int UUID_STRING_WIDTH = 36; + + /** Storage type for UUID: FixedSizeBinary(16). */ + public static final ArrowType STORAGE_TYPE = new ArrowType.FixedSizeBinary(UUID_BYTE_WIDTH); + + static { + ExtensionTypeRegistry.register(INSTANCE); + } + + @Override + public ArrowType storageType() { + return STORAGE_TYPE; + } + + @Override + public String extensionName() { + return EXTENSION_NAME; + } + + @Override + public boolean extensionEquals(ExtensionType other) { + return other instanceof UuidType; + } + + @Override + public ArrowType deserialize(ArrowType storageType, String serializedData) { + if (!storageType.equals(storageType())) { + throw new UnsupportedOperationException( + "Cannot construct UuidType from underlying type " + storageType); + } + return INSTANCE; + } + + @Override + public String serialize() { + return ""; + } + + @Override + public boolean isComplex() { + return false; + } + + @Override + public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator) { + return new UuidVector( + name, fieldType, allocator, new FixedSizeBinaryVector(name, allocator, UUID_BYTE_WIDTH)); + } +} diff --git a/vector/src/main/java/org/apache/arrow/vector/holders/NullableUuidHolder.java b/vector/src/main/java/org/apache/arrow/vector/holders/NullableUuidHolder.java new file mode 100644 index 0000000000..e5398d82cf --- /dev/null +++ b/vector/src/main/java/org/apache/arrow/vector/holders/NullableUuidHolder.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.holders; + +import org.apache.arrow.memory.ArrowBuf; + +/** + * Value holder for nullable UUID values. + * + *

The {@code isSet} field controls nullability: when {@code isSet = 1}, the holder contains a + * valid UUID in {@code buffer}; when {@code isSet = 0}, the holder represents a null value and + * {@code buffer} should not be accessed. + * + * @see UuidHolder + * @see org.apache.arrow.vector.UuidVector + * @see org.apache.arrow.vector.extension.UuidType + */ +public class NullableUuidHolder extends ExtensionHolder { + /** Buffer containing 16-byte UUID data. */ + public ArrowBuf buffer; +} diff --git a/vector/src/test/java/org/apache/arrow/vector/holder/UuidHolder.java b/vector/src/main/java/org/apache/arrow/vector/holders/UuidHolder.java similarity index 62% rename from vector/src/test/java/org/apache/arrow/vector/holder/UuidHolder.java rename to vector/src/main/java/org/apache/arrow/vector/holders/UuidHolder.java index 207b0951a7..484e05c24b 100644 --- a/vector/src/test/java/org/apache/arrow/vector/holder/UuidHolder.java +++ b/vector/src/main/java/org/apache/arrow/vector/holders/UuidHolder.java @@ -14,10 +14,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.arrow.vector.holder; +package org.apache.arrow.vector.holders; -import org.apache.arrow.vector.holders.ExtensionHolder; +import org.apache.arrow.memory.ArrowBuf; +/** + * Value holder for non-nullable UUID values. + * + *

Contains a 16-byte UUID in {@code buffer} with {@code isSet} always 1. + * + * @see NullableUuidHolder + * @see org.apache.arrow.vector.UuidVector + * @see org.apache.arrow.vector.extension.UuidType + */ public class UuidHolder extends ExtensionHolder { - public byte[] value; + /** Buffer containing 16-byte UUID data. */ + public ArrowBuf buffer; + + /** Constructs a UuidHolder with isSet = 1. */ + public UuidHolder() { + this.isSet = 1; + } } diff --git a/vector/src/main/java/org/apache/arrow/vector/util/UuidUtility.java b/vector/src/main/java/org/apache/arrow/vector/util/UuidUtility.java new file mode 100644 index 0000000000..a1b0b54579 --- /dev/null +++ b/vector/src/main/java/org/apache/arrow/vector/util/UuidUtility.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.util; + +import static org.apache.arrow.vector.extension.UuidType.UUID_BYTE_WIDTH; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.UUID; +import org.apache.arrow.memory.ArrowBuf; + +/** + * Utility class for UUID conversions and operations. + * + *

Provides methods to convert between {@link UUID} objects and byte representations used in + * Arrow vectors. + * + * @see org.apache.arrow.vector.UuidVector + * @see org.apache.arrow.vector.extension.UuidType + */ +public class UuidUtility { + /** + * Converts a UUID to a 16-byte array. + * + *

The UUID is stored in big-endian byte order, with the most significant bits first. + * + * @param uuid the UUID to convert + * @return a 16-byte array representing the UUID + */ + public static byte[] getBytesFromUUID(UUID uuid) { + byte[] result = new byte[16]; + long msb = uuid.getMostSignificantBits(); + long lsb = uuid.getLeastSignificantBits(); + for (int i = 15; i >= 8; i--) { + result[i] = (byte) (lsb & 0xFF); + lsb >>= 8; + } + for (int i = 7; i >= 0; i--) { + result[i] = (byte) (msb & 0xFF); + msb >>= 8; + } + return result; + } + + /** + * Constructs a UUID from bytes stored in an ArrowBuf at the specified index. + * + *

Reads 16 bytes from the buffer starting at the given index and interprets them as a UUID in + * big-endian byte order. + * + * @param buffer the buffer containing UUID data + * @param index the byte offset in the buffer where the UUID starts + * @return the UUID constructed from the buffer data + */ + public static UUID uuidFromArrowBuf(ArrowBuf buffer, long index) { + ByteBuffer buf = buffer.nioBuffer(index, UUID_BYTE_WIDTH); + + buf.order(ByteOrder.BIG_ENDIAN); + long mostSigBits = buf.getLong(0); + long leastSigBits = buf.getLong(Long.BYTES); + return new UUID(mostSigBits, leastSigBits); + } +} diff --git a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index c6c7c5c862..41a95a8d11 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -24,7 +24,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -39,17 +38,18 @@ import org.apache.arrow.vector.complex.impl.UuidWriterFactory; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.complex.writer.BaseWriter.ExtensionWriter; -import org.apache.arrow.vector.holder.UuidHolder; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.holders.DurationHolder; import org.apache.arrow.vector.holders.FixedSizeBinaryHolder; import org.apache.arrow.vector.holders.TimeStampMilliTZHolder; +import org.apache.arrow.vector.holders.UuidHolder; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.types.pojo.UuidType; import org.apache.arrow.vector.util.TransferPair; +import org.apache.arrow.vector.util.UuidUtility; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -1259,14 +1259,12 @@ public void testListVectorReaderForExtensionType() throws Exception { FieldReader uuidReader = reader.reader(); UuidHolder holder = new UuidHolder(); uuidReader.read(holder); - ByteBuffer bb = ByteBuffer.wrap(holder.value); - UUID actualUuid = new UUID(bb.getLong(), bb.getLong()); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); assertEquals(u1, actualUuid); reader.next(); uuidReader = reader.reader(); uuidReader.read(holder); - bb = ByteBuffer.wrap(holder.value); - actualUuid = new UUID(bb.getLong(), bb.getLong()); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); assertEquals(u2, actualUuid); } } @@ -1302,14 +1300,12 @@ public void testCopyFromForExtensionType() throws Exception { FieldReader uuidReader = reader.reader(); UuidHolder holder = new UuidHolder(); uuidReader.read(holder); - ByteBuffer bb = ByteBuffer.wrap(holder.value); - UUID actualUuid = new UUID(bb.getLong(), bb.getLong()); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); assertEquals(u1, actualUuid); reader.next(); uuidReader = reader.reader(); uuidReader.read(holder); - bb = ByteBuffer.wrap(holder.value); - actualUuid = new UUID(bb.getLong(), bb.getLong()); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); assertEquals(u2, actualUuid); } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java index 8605d250fd..df8f338f45 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java @@ -24,7 +24,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -42,15 +41,16 @@ import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; import org.apache.arrow.vector.complex.writer.FieldWriter; -import org.apache.arrow.vector.holder.UuidHolder; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.holders.FixedSizeBinaryHolder; +import org.apache.arrow.vector.holders.UuidHolder; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.types.pojo.UuidType; import org.apache.arrow.vector.util.JsonStringArrayList; import org.apache.arrow.vector.util.TransferPair; +import org.apache.arrow.vector.util.UuidUtility; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -1304,14 +1304,12 @@ public void testMapVectorWithExtensionType() throws Exception { FieldReader uuidReader = mapReader.value(); UuidHolder holder = new UuidHolder(); uuidReader.read(holder); - ByteBuffer bb = ByteBuffer.wrap(holder.value); - UUID actualUuid = new UUID(bb.getLong(), bb.getLong()); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); assertEquals(u1, actualUuid); mapReader.next(); uuidReader = mapReader.value(); uuidReader.read(holder); - bb = ByteBuffer.wrap(holder.value); - actualUuid = new UUID(bb.getLong(), bb.getLong()); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); assertEquals(u2, actualUuid); } } @@ -1351,14 +1349,12 @@ public void testCopyFromForExtensionType() throws Exception { FieldReader uuidReader = mapReader.value(); UuidHolder holder = new UuidHolder(); uuidReader.read(holder); - ByteBuffer bb = ByteBuffer.wrap(holder.value); - UUID actualUuid = new UUID(bb.getLong(), bb.getLong()); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); assertEquals(u1, actualUuid); mapReader.next(); uuidReader = mapReader.value(); uuidReader.read(holder); - bb = ByteBuffer.wrap(holder.value); - actualUuid = new UUID(bb.getLong(), bb.getLong()); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); assertEquals(u2, actualUuid); } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java b/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java index d40af9ae89..b8abfe1ef6 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java @@ -35,6 +35,7 @@ import org.apache.arrow.vector.complex.impl.NullableStructWriter; import org.apache.arrow.vector.complex.writer.Float8Writer; import org.apache.arrow.vector.complex.writer.IntWriter; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.holders.ComplexHolder; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.Types.MinorType; @@ -42,7 +43,6 @@ import org.apache.arrow.vector.types.pojo.ArrowType.Struct; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.types.pojo.UuidType; import org.apache.arrow.vector.util.TransferPair; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; diff --git a/vector/src/test/java/org/apache/arrow/vector/TestUtils.java b/vector/src/test/java/org/apache/arrow/vector/TestUtils.java index 82295f8037..c28751aa58 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestUtils.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestUtils.java @@ -20,6 +20,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ExtensionTypeRegistry; import org.apache.arrow.vector.types.pojo.FieldType; public class TestUtils { @@ -62,4 +63,14 @@ public static String generateRandomString(int length) { } return sb.toString(); } + + /* + * Ensure the extension type is registered, as there might other tests trying to unregister the + * type. ex.: TestExtensionType#readUnderlyingType + */ + public static void ensureRegistered(ArrowType.ExtensionType type) { + if (ExtensionTypeRegistry.lookup(type.extensionName()) == null) { + ExtensionTypeRegistry.register(type); + } + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestUuidType.java b/vector/src/test/java/org/apache/arrow/vector/TestUuidType.java new file mode 100644 index 0000000000..9f7c65b82b --- /dev/null +++ b/vector/src/test/java/org/apache/arrow/vector/TestUuidType.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector; + +import static org.apache.arrow.vector.TestUtils.ensureRegistered; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.UUID; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.dictionary.DictionaryProvider; +import org.apache.arrow.vector.extension.UuidType; +import org.apache.arrow.vector.ipc.ArrowStreamReader; +import org.apache.arrow.vector.ipc.ArrowStreamWriter; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.arrow.vector.util.UuidUtility; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class TestUuidType { + BufferAllocator allocator; + + @BeforeEach + void beforeEach() { + allocator = new RootAllocator(); + } + + @AfterEach + void afterEach() { + allocator.close(); + } + + @Test + void testConstants() { + assertEquals("arrow.uuid", UuidType.EXTENSION_NAME); + assertNotNull(UuidType.INSTANCE); + assertNotNull(UuidType.STORAGE_TYPE); + assertInstanceOf(ArrowType.FixedSizeBinary.class, UuidType.STORAGE_TYPE); + assertEquals( + UuidType.UUID_BYTE_WIDTH, + ((ArrowType.FixedSizeBinary) UuidType.STORAGE_TYPE).getByteWidth()); + } + + @Test + void testStorageType() { + UuidType type = new UuidType(); + assertEquals(UuidType.STORAGE_TYPE, type.storageType()); + assertInstanceOf(ArrowType.FixedSizeBinary.class, type.storageType()); + } + + @Test + void testExtensionName() { + UuidType type = new UuidType(); + assertEquals("arrow.uuid", type.extensionName()); + } + + @Test + void testExtensionEquals() { + UuidType type1 = new UuidType(); + UuidType type2 = new UuidType(); + UuidType type3 = UuidType.INSTANCE; + + assertTrue(type1.extensionEquals(type2)); + assertTrue(type1.extensionEquals(type3)); + assertTrue(type2.extensionEquals(type3)); + } + + @Test + void testIsComplex() { + UuidType type = new UuidType(); + assertFalse(type.isComplex()); + } + + @Test + void testSerialize() { + UuidType type = new UuidType(); + String serialized = type.serialize(); + assertEquals("", serialized); + } + + @Test + void testDeserializeValid() { + UuidType type = new UuidType(); + ArrowType storageType = new ArrowType.FixedSizeBinary(UuidType.UUID_BYTE_WIDTH); + + ArrowType deserialized = assertDoesNotThrow(() -> type.deserialize(storageType, "")); + assertInstanceOf(UuidType.class, deserialized); + assertEquals(UuidType.INSTANCE, deserialized); + } + + @Test + void testDeserializeInvalidStorageType() { + UuidType type = new UuidType(); + ArrowType wrongStorageType = new ArrowType.FixedSizeBinary(32); + + assertThrows(UnsupportedOperationException.class, () -> type.deserialize(wrongStorageType, "")); + } + + @Test + void testGetNewVector() { + UuidType type = new UuidType(); + try (FieldVector vector = + type.getNewVector("uuid_field", FieldType.nullable(type), allocator)) { + assertInstanceOf(UuidVector.class, vector); + assertEquals("uuid_field", vector.getField().getName()); + assertEquals(type, vector.getField().getType()); + } + } + + @Test + void testVectorOperations() { + UuidType type = new UuidType(); + try (FieldVector vector = + type.getNewVector("uuid_field", FieldType.nullable(type), allocator)) { + UuidVector uuidVector = (UuidVector) vector; + + UUID uuid1 = UUID.randomUUID(); + UUID uuid2 = UUID.randomUUID(); + + uuidVector.setSafe(0, uuid1); + uuidVector.setSafe(1, uuid2); + uuidVector.setNull(2); + uuidVector.setValueCount(3); + + assertEquals(uuid1, uuidVector.getObject(0)); + assertEquals(uuid2, uuidVector.getObject(1)); + assertNull(uuidVector.getObject(2)); + assertFalse(uuidVector.isNull(0)); + assertFalse(uuidVector.isNull(1)); + assertTrue(uuidVector.isNull(2)); + } + } + + @Test + void testIpcRoundTrip() { + UuidType type = UuidType.INSTANCE; + ensureRegistered(type); + + Schema schema = new Schema(Collections.singletonList(Field.nullable("uuid", type))); + byte[] serialized = schema.serializeAsMessage(); + Schema deserialized = Schema.deserializeMessage(ByteBuffer.wrap(serialized)); + assertEquals(schema, deserialized); + } + + @Test + void testVectorIpcRoundTrip() throws IOException { + UuidType type = UuidType.INSTANCE; + ensureRegistered(type); + + UUID uuid1 = UUID.randomUUID(); + UUID uuid2 = UUID.randomUUID(); + + try (FieldVector vector = type.getNewVector("field", FieldType.nullable(type), allocator)) { + UuidVector uuidVector = (UuidVector) vector; + uuidVector.setSafe(0, uuid1); + uuidVector.setNull(1); + uuidVector.setSafe(2, uuid2); + uuidVector.setValueCount(3); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (VectorSchemaRoot root = new VectorSchemaRoot(Collections.singletonList(uuidVector)); + ArrowStreamWriter writer = + new ArrowStreamWriter(root, new DictionaryProvider.MapDictionaryProvider(), baos)) { + writer.start(); + writer.writeBatch(); + } + + try (ArrowStreamReader reader = + new ArrowStreamReader(new ByteArrayInputStream(baos.toByteArray()), allocator)) { + assertTrue(reader.loadNextBatch()); + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + assertEquals(3, root.getRowCount()); + assertEquals( + new Schema(Collections.singletonList(uuidVector.getField())), root.getSchema()); + + UuidVector actual = assertInstanceOf(UuidVector.class, root.getVector("field")); + assertFalse(actual.isNull(0)); + assertTrue(actual.isNull(1)); + assertFalse(actual.isNull(2)); + assertEquals(uuid1, actual.getObject(0)); + assertNull(actual.getObject(1)); + assertEquals(uuid2, actual.getObject(2)); + } + } + } + + @Test + void testVectorByteArrayOperations() { + UuidType type = new UuidType(); + try (FieldVector vector = + type.getNewVector("uuid_field", FieldType.nullable(type), allocator)) { + UuidVector uuidVector = (UuidVector) vector; + + UUID uuid = UUID.randomUUID(); + byte[] uuidBytes = UuidUtility.getBytesFromUUID(uuid); + + uuidVector.setSafe(0, uuidBytes); + uuidVector.setValueCount(1); + + assertEquals(uuid, uuidVector.getObject(0)); + + // Verify the bytes match + byte[] actualBytes = new byte[UuidType.UUID_BYTE_WIDTH]; + uuidVector.get(0).getBytes(0, actualBytes); + assertArrayEquals(uuidBytes, actualBytes); + } + } + + @Test + void testGetNewVectorWithCustomFieldType() { + UuidType type = new UuidType(); + FieldType fieldType = new FieldType(false, type, null); + + try (FieldVector vector = type.getNewVector("non_nullable_uuid", fieldType, allocator)) { + assertInstanceOf(UuidVector.class, vector); + assertEquals("non_nullable_uuid", vector.getField().getName()); + assertFalse(vector.getField().isNullable()); + } + } + + @Test + void testSingleton() { + UuidType type1 = UuidType.INSTANCE; + UuidType type2 = UuidType.INSTANCE; + + // Same instance + assertSame(type1, type2); + assertTrue(type1.extensionEquals(type2)); + } + + @Test + void testUnderlyingVector() { + UuidType type = new UuidType(); + try (FieldVector vector = + type.getNewVector("uuid_field", FieldType.nullable(type), allocator)) { + UuidVector uuidVector = (UuidVector) vector; + FixedSizeBinaryVector underlying = uuidVector.getUnderlyingVector(); + + assertInstanceOf(FixedSizeBinaryVector.class, underlying); + assertEquals(UuidType.UUID_BYTE_WIDTH, underlying.getByteWidth()); + } + } +} diff --git a/vector/src/test/java/org/apache/arrow/vector/TestUuidVector.java b/vector/src/test/java/org/apache/arrow/vector/TestUuidVector.java new file mode 100644 index 0000000000..3d70238ece --- /dev/null +++ b/vector/src/test/java/org/apache/arrow/vector/TestUuidVector.java @@ -0,0 +1,451 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.ByteBuffer; +import java.util.UUID; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.complex.impl.UuidReaderImpl; +import org.apache.arrow.vector.complex.impl.UuidWriterImpl; +import org.apache.arrow.vector.extension.UuidType; +import org.apache.arrow.vector.holders.ExtensionHolder; +import org.apache.arrow.vector.holders.NullableUuidHolder; +import org.apache.arrow.vector.holders.UuidHolder; +import org.apache.arrow.vector.util.UuidUtility; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** Tests for UuidVector, UuidWriterImpl, and UuidReaderImpl. */ +class TestUuidVector { + + private BufferAllocator allocator; + + @BeforeEach + void beforeEach() { + allocator = new RootAllocator(); + } + + @AfterEach + void afterEach() { + allocator.close(); + } + + // ========== Writer Tests ========== + + @Test + void testWriteToExtensionVector() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator); + UuidWriterImpl writer = new UuidWriterImpl(vector)) { + UUID uuid = UUID.randomUUID(); + ByteBuffer bb = ByteBuffer.allocate(UuidType.UUID_BYTE_WIDTH); + bb.putLong(uuid.getMostSignificantBits()); + bb.putLong(uuid.getLeastSignificantBits()); + + // Allocate ArrowBuf for the holder + try (ArrowBuf buf = allocator.buffer(UuidType.UUID_BYTE_WIDTH)) { + buf.setBytes(0, bb.array()); + + UuidHolder holder = new UuidHolder(); + holder.buffer = buf; + + writer.write(holder); + UUID result = vector.getObject(0); + assertEquals(uuid, result); + } + } + } + + @Test + void testWriteExtensionWithUUID() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator); + UuidWriterImpl writer = new UuidWriterImpl(vector)) { + UUID uuid = UUID.randomUUID(); + writer.setPosition(0); + writer.writeExtension(uuid); + + UUID result = vector.getObject(0); + assertEquals(uuid, result); + assertEquals(1, vector.getValueCount()); + } + } + + @Test + void testWriteExtensionWithByteArray() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator); + UuidWriterImpl writer = new UuidWriterImpl(vector)) { + UUID uuid = UUID.randomUUID(); + byte[] uuidBytes = UuidUtility.getBytesFromUUID(uuid); + + writer.setPosition(0); + writer.writeExtension(uuidBytes); + + UUID result = vector.getObject(0); + assertEquals(uuid, result); + assertEquals(1, vector.getValueCount()); + } + } + + @Test + void testWriteExtensionWithArrowBuf() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator); + UuidWriterImpl writer = new UuidWriterImpl(vector); + ArrowBuf buf = allocator.buffer(UuidType.UUID_BYTE_WIDTH)) { + UUID uuid = UUID.randomUUID(); + byte[] uuidBytes = UuidUtility.getBytesFromUUID(uuid); + buf.setBytes(0, uuidBytes); + + writer.setPosition(0); + writer.writeExtension(buf); + + UUID result = vector.getObject(0); + assertEquals(uuid, result); + assertEquals(1, vector.getValueCount()); + } + } + + @Test + void testWriteExtensionWithUnsupportedType() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator); + UuidWriterImpl writer = new UuidWriterImpl(vector)) { + writer.setPosition(0); + + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> writer.writeExtension("invalid-type")); + + assertEquals( + "Unsupported value type for UUID: class java.lang.String", exception.getMessage()); + } + } + + @Test + void testWriteExtensionMultipleValues() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator); + UuidWriterImpl writer = new UuidWriterImpl(vector)) { + UUID uuid1 = UUID.randomUUID(); + UUID uuid2 = UUID.randomUUID(); + UUID uuid3 = UUID.randomUUID(); + + writer.setPosition(0); + writer.writeExtension(uuid1); + writer.setPosition(1); + writer.writeExtension(uuid2); + writer.setPosition(2); + writer.writeExtension(uuid3); + + assertEquals(uuid1, vector.getObject(0)); + assertEquals(uuid2, vector.getObject(1)); + assertEquals(uuid3, vector.getObject(2)); + assertEquals(3, vector.getValueCount()); + } + } + + @Test + void testWriteWithUuidHolder() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator); + UuidWriterImpl writer = new UuidWriterImpl(vector); + ArrowBuf buf = allocator.buffer(UuidType.UUID_BYTE_WIDTH)) { + UUID uuid = UUID.randomUUID(); + byte[] uuidBytes = UuidUtility.getBytesFromUUID(uuid); + buf.setBytes(0, uuidBytes); + + UuidHolder holder = new UuidHolder(); + holder.buffer = buf; + holder.isSet = 1; + + writer.setPosition(0); + writer.write(holder); + + UUID result = vector.getObject(0); + assertEquals(uuid, result); + assertEquals(1, vector.getValueCount()); + } + } + + @Test + void testWriteWithNullableUuidHolder() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator); + UuidWriterImpl writer = new UuidWriterImpl(vector); + ArrowBuf buf = allocator.buffer(UuidType.UUID_BYTE_WIDTH)) { + UUID uuid = UUID.randomUUID(); + byte[] uuidBytes = UuidUtility.getBytesFromUUID(uuid); + buf.setBytes(0, uuidBytes); + + NullableUuidHolder holder = new NullableUuidHolder(); + holder.buffer = buf; + holder.isSet = 1; + + writer.setPosition(0); + writer.write(holder); + + UUID result = vector.getObject(0); + assertEquals(uuid, result); + assertEquals(1, vector.getValueCount()); + } + } + + @Test + void testWriteWithNullableUuidHolderNull() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator); + UuidWriterImpl writer = new UuidWriterImpl(vector)) { + NullableUuidHolder holder = new NullableUuidHolder(); + holder.isSet = 0; + + writer.setPosition(0); + writer.write(holder); + + assertTrue(vector.isNull(0)); + assertEquals(1, vector.getValueCount()); + } + } + + // ========== Reader Tests ========== + + @Test + void testReaderCopyAsValueExtensionVector() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator); + UuidVector vectorForRead = new UuidVector("test2", allocator); + UuidWriterImpl writer = new UuidWriterImpl(vector)) { + UUID uuid = UUID.randomUUID(); + vectorForRead.setValueCount(1); + vectorForRead.set(0, uuid); + UuidReaderImpl reader = (UuidReaderImpl) vectorForRead.getReader(); + reader.copyAsValue(writer); + UuidReaderImpl reader2 = (UuidReaderImpl) vector.getReader(); + UuidHolder holder = new UuidHolder(); + reader2.read(0, holder); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + assertEquals(uuid, actualUuid); + } + } + + @Test + void testReaderReadWithUuidHolder() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid = UUID.randomUUID(); + vector.setSafe(0, uuid); + vector.setValueCount(1); + + UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); + reader.setPosition(0); + + UuidHolder holder = new UuidHolder(); + reader.read(holder); + + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + assertEquals(uuid, actualUuid); + assertEquals(1, holder.isSet); + } + } + + @Test + void testReaderReadWithNullableUuidHolder() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid = UUID.randomUUID(); + vector.setSafe(0, uuid); + vector.setValueCount(1); + + UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); + reader.setPosition(0); + + NullableUuidHolder holder = new NullableUuidHolder(); + reader.read(holder); + + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + assertEquals(uuid, actualUuid); + assertEquals(1, holder.isSet); + } + } + + @Test + void testReaderReadWithNullableUuidHolderNull() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + vector.setNull(0); + vector.setValueCount(1); + + UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); + reader.setPosition(0); + + NullableUuidHolder holder = new NullableUuidHolder(); + reader.read(holder); + + assertEquals(0, holder.isSet); + } + } + + @Test + void testReaderReadWithArrayIndexUuidHolder() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid1 = UUID.randomUUID(); + UUID uuid2 = UUID.randomUUID(); + UUID uuid3 = UUID.randomUUID(); + + vector.setSafe(0, uuid1); + vector.setSafe(1, uuid2); + vector.setSafe(2, uuid3); + vector.setValueCount(3); + + UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); + + UuidHolder holder = new UuidHolder(); + reader.read(1, holder); + + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + assertEquals(uuid2, actualUuid); + assertEquals(1, holder.isSet); + } + } + + @Test + void testReaderReadWithArrayIndexNullableUuidHolder() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid1 = UUID.randomUUID(); + UUID uuid2 = UUID.randomUUID(); + + vector.setSafe(0, uuid1); + vector.setNull(1); + vector.setSafe(2, uuid2); + vector.setValueCount(3); + + UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); + + NullableUuidHolder holder1 = new NullableUuidHolder(); + reader.read(0, holder1); + assertEquals(uuid1, UuidUtility.uuidFromArrowBuf(holder1.buffer, 0)); + assertEquals(1, holder1.isSet); + + NullableUuidHolder holder2 = new NullableUuidHolder(); + reader.read(1, holder2); + assertEquals(0, holder2.isSet); + + NullableUuidHolder holder3 = new NullableUuidHolder(); + reader.read(2, holder3); + assertEquals(uuid2, UuidUtility.uuidFromArrowBuf(holder3.buffer, 0)); + assertEquals(1, holder3.isSet); + } + } + + @Test + void testReaderReadWithUnsupportedHolder() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid = UUID.randomUUID(); + vector.setSafe(0, uuid); + vector.setValueCount(1); + + UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); + reader.setPosition(0); + + // Create a mock unsupported holder + ExtensionHolder unsupportedHolder = new ExtensionHolder() {}; + + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> reader.read(unsupportedHolder)); + + assertTrue(exception.getMessage().contains("Unsupported holder type for UuidReader")); + } + } + + @Test + void testReaderReadWithArrayIndexUnsupportedHolder() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid = UUID.randomUUID(); + vector.setSafe(0, uuid); + vector.setValueCount(1); + + UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); + + // Create a mock unsupported holder + ExtensionHolder unsupportedHolder = new ExtensionHolder() {}; + + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> reader.read(0, unsupportedHolder)); + + assertTrue(exception.getMessage().contains("Unsupported holder type for UuidReader")); + } + } + + @Test + void testReaderIsSet() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid = UUID.randomUUID(); + vector.setSafe(0, uuid); + vector.setNull(1); + vector.setSafe(2, uuid); + vector.setValueCount(3); + + UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); + + reader.setPosition(0); + assertTrue(reader.isSet()); + + reader.setPosition(1); + assertFalse(reader.isSet()); + + reader.setPosition(2); + assertTrue(reader.isSet()); + } + } + + @Test + void testReaderReadObject() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid1 = UUID.randomUUID(); + UUID uuid2 = UUID.randomUUID(); + + vector.setSafe(0, uuid1); + vector.setNull(1); + vector.setSafe(2, uuid2); + vector.setValueCount(3); + + UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); + + reader.setPosition(0); + assertEquals(uuid1, reader.readObject()); + + reader.setPosition(1); + assertNull(reader.readObject()); + + reader.setPosition(2); + assertEquals(uuid2, reader.readObject()); + } + } + + @Test + void testReaderGetMinorType() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); + assertEquals(vector.getMinorType(), reader.getMinorType()); + } + } + + @Test + void testReaderGetField() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); + assertEquals(vector.getField(), reader.getField()); + assertEquals("test", reader.getField().getName()); + } + } +} diff --git a/vector/src/test/java/org/apache/arrow/vector/UuidVector.java b/vector/src/test/java/org/apache/arrow/vector/UuidVector.java deleted file mode 100644 index 72ba4aa555..0000000000 --- a/vector/src/test/java/org/apache/arrow/vector/UuidVector.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.arrow.vector; - -import java.nio.ByteBuffer; -import java.util.UUID; -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.memory.util.hash.ArrowBufHasher; -import org.apache.arrow.vector.complex.impl.UuidReaderImpl; -import org.apache.arrow.vector.complex.reader.FieldReader; -import org.apache.arrow.vector.holder.UuidHolder; -import org.apache.arrow.vector.types.pojo.Field; -import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.types.pojo.UuidType; -import org.apache.arrow.vector.util.TransferPair; - -public class UuidVector extends ExtensionTypeVector - implements ValueIterableVector { - private final Field field; - - public UuidVector( - String name, BufferAllocator allocator, FixedSizeBinaryVector underlyingVector) { - super(name, allocator, underlyingVector); - this.field = new Field(name, FieldType.nullable(new UuidType()), null); - } - - public UuidVector(String name, BufferAllocator allocator) { - super(name, allocator, new FixedSizeBinaryVector(name, allocator, 16)); - this.field = new Field(name, FieldType.nullable(new UuidType()), null); - } - - @Override - public UUID getObject(int index) { - final ByteBuffer bb = ByteBuffer.wrap(getUnderlyingVector().getObject(index)); - return new UUID(bb.getLong(), bb.getLong()); - } - - @Override - public int hashCode(int index) { - return hashCode(index, null); - } - - @Override - public int hashCode(int index, ArrowBufHasher hasher) { - return getUnderlyingVector().hashCode(index, hasher); - } - - public void set(int index, UUID uuid) { - ByteBuffer bb = ByteBuffer.allocate(16); - bb.putLong(uuid.getMostSignificantBits()); - bb.putLong(uuid.getLeastSignificantBits()); - getUnderlyingVector().set(index, bb.array()); - } - - @Override - public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { - getUnderlyingVector() - .copyFromSafe(fromIndex, thisIndex, ((UuidVector) from).getUnderlyingVector()); - } - - @Override - public Field getField() { - return field; - } - - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((UuidVector) to); - } - - @Override - protected FieldReader getReaderImpl() { - return new UuidReaderImpl(this); - } - - public void setSafe(int index, byte[] value) { - getUnderlyingVector().setIndexDefined(index); - getUnderlyingVector().setSafe(index, value); - } - - public void get(int index, UuidHolder holder) { - holder.value = getUnderlyingVector().get(index); - holder.isSet = 1; - } - - public class TransferImpl implements TransferPair { - UuidVector to; - ValueVector targetUnderlyingVector; - TransferPair tp; - - public TransferImpl(UuidVector to) { - this.to = to; - targetUnderlyingVector = this.to.getUnderlyingVector(); - tp = getUnderlyingVector().makeTransferPair(targetUnderlyingVector); - } - - public UuidVector getTo() { - return this.to; - } - - public void transfer() { - tp.transfer(); - } - - public void splitAndTransfer(int startIndex, int length) { - tp.splitAndTransfer(startIndex, length); - } - - public void copyValueSafe(int fromIndex, int toIndex) { - tp.copyValueSafe(fromIndex, toIndex); - } - } -} diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java index 738e8905e3..493a4b26ab 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java @@ -34,11 +34,11 @@ import org.apache.arrow.vector.complex.writer.BaseWriter.ExtensionWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.StructWriter; import org.apache.arrow.vector.complex.writer.FieldWriter; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.holders.DecimalHolder; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.types.pojo.UuidType; import org.apache.arrow.vector.util.DecimalUtility; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java index 7b8b1f9ef9..a4594024fa 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java @@ -41,6 +41,7 @@ import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.complex.UnionVector; import org.apache.arrow.vector.complex.writer.BaseWriter.StructWriter; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.holders.DurationHolder; import org.apache.arrow.vector.holders.FixedSizeBinaryHolder; import org.apache.arrow.vector.holders.NullableDecimalHolder; @@ -54,7 +55,6 @@ import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.types.pojo.UuidType; import org.apache.arrow.vector.util.DecimalUtility; import org.apache.arrow.vector.util.Text; import org.junit.jupiter.api.AfterEach; diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index f374eb41e4..46c259bda0 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -78,7 +78,7 @@ import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.StructWriter; -import org.apache.arrow.vector.holder.UuidHolder; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.holders.DecimalHolder; import org.apache.arrow.vector.holders.DurationHolder; import org.apache.arrow.vector.holders.FixedSizeBinaryHolder; @@ -88,6 +88,7 @@ import org.apache.arrow.vector.holders.NullableTimeStampMilliTZHolder; import org.apache.arrow.vector.holders.NullableTimeStampNanoTZHolder; import org.apache.arrow.vector.holders.TimeStampMilliTZHolder; +import org.apache.arrow.vector.holders.UuidHolder; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -99,13 +100,13 @@ import org.apache.arrow.vector.types.pojo.ArrowType.Utf8; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.types.pojo.UuidType; import org.apache.arrow.vector.util.CallBack; import org.apache.arrow.vector.util.DecimalUtility; import org.apache.arrow.vector.util.JsonStringArrayList; import org.apache.arrow.vector.util.JsonStringHashMap; import org.apache.arrow.vector.util.Text; import org.apache.arrow.vector.util.TransferPair; +import org.apache.arrow.vector.util.UuidUtility; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -2521,8 +2522,7 @@ public void extensionWriterReader() throws Exception { uuidReader.setPosition(0); UuidHolder uuidHolder = new UuidHolder(); uuidReader.read(uuidHolder); - final ByteBuffer bb = ByteBuffer.wrap(uuidHolder.value); - UUID actualUuid = new UUID(bb.getLong(), bb.getLong()); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(uuidHolder.buffer, 0); assertEquals(u1, actualUuid); assertTrue(uuidReader.isSet()); assertEquals(uuidReader.getMinorType(), MinorType.EXTENSIONTYPE); diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java index 269cff0670..5bb5962704 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java @@ -20,21 +20,16 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import java.nio.ByteBuffer; -import java.util.UUID; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.LargeVarBinaryVector; import org.apache.arrow.vector.LargeVarCharVector; -import org.apache.arrow.vector.UuidVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.complex.impl.LargeVarBinaryWriterImpl; import org.apache.arrow.vector.complex.impl.LargeVarCharWriterImpl; -import org.apache.arrow.vector.complex.impl.UuidReaderImpl; -import org.apache.arrow.vector.complex.impl.UuidWriterImpl; import org.apache.arrow.vector.complex.impl.VarBinaryWriterImpl; import org.apache.arrow.vector.complex.impl.VarCharWriterImpl; -import org.apache.arrow.vector.holder.UuidHolder; import org.apache.arrow.vector.util.Text; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -189,39 +184,4 @@ public void testWriteTextToLargeVarChar() throws Exception { assertEquals(input, result); } } - - @Test - public void testWriteToExtensionVector() throws Exception { - try (UuidVector vector = new UuidVector("test", allocator); - UuidWriterImpl writer = new UuidWriterImpl(vector)) { - UUID uuid = UUID.randomUUID(); - ByteBuffer bb = ByteBuffer.allocate(16); - bb.putLong(uuid.getMostSignificantBits()); - bb.putLong(uuid.getLeastSignificantBits()); - UuidHolder holder = new UuidHolder(); - holder.value = bb.array(); - writer.write(holder); - UUID result = vector.getObject(0); - assertEquals(uuid, result); - } - } - - @Test - public void testReaderCopyAsValueExtensionVector() throws Exception { - try (UuidVector vector = new UuidVector("test", allocator); - UuidVector vectorForRead = new UuidVector("test2", allocator); - UuidWriterImpl writer = new UuidWriterImpl(vector)) { - UUID uuid = UUID.randomUUID(); - vectorForRead.setValueCount(1); - vectorForRead.set(0, uuid); - UuidReaderImpl reader = (UuidReaderImpl) vectorForRead.getReader(); - reader.copyAsValue(writer); - UuidReaderImpl reader2 = (UuidReaderImpl) vector.getReader(); - UuidHolder holder = new UuidHolder(); - reader2.read(0, holder); - final ByteBuffer bb = ByteBuffer.wrap(holder.value); - UUID actualUuid = new UUID(bb.getLong(), bb.getLong()); - assertEquals(uuid, actualUuid); - } - } } diff --git a/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java b/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java index d24708d66c..2ac4045aa2 100644 --- a/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java +++ b/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.vector.types.pojo; +import static org.apache.arrow.vector.TestUtils.ensureRegistered; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -47,6 +48,7 @@ import org.apache.arrow.vector.compare.Range; import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.complex.StructVector; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.ipc.ArrowFileReader; import org.apache.arrow.vector.ipc.ArrowFileWriter; import org.apache.arrow.vector.types.FloatingPointPrecision; @@ -59,9 +61,9 @@ public class TestExtensionType { /** Test that a custom UUID type can be round-tripped through a temporary file. */ @Test public void roundtripUuid() throws IOException { - ExtensionTypeRegistry.register(new UuidType()); + ensureRegistered(UuidType.INSTANCE); final Schema schema = - new Schema(Collections.singletonList(Field.nullable("a", new UuidType()))); + new Schema(Collections.singletonList(Field.nullable("a", UuidType.INSTANCE))); try (final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { UUID u1 = UUID.randomUUID(); @@ -89,7 +91,7 @@ public void roundtripUuid() throws IOException { assertEquals(root.getSchema(), readerRoot.getSchema()); final Field field = readerRoot.getSchema().getFields().get(0); - final UuidType expectedType = new UuidType(); + final UuidType expectedType = UuidType.INSTANCE; assertEquals( field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_NAME), expectedType.extensionName()); @@ -113,9 +115,9 @@ public void roundtripUuid() throws IOException { /** Test that a custom UUID type can be read as its underlying type. */ @Test public void readUnderlyingType() throws IOException { - ExtensionTypeRegistry.register(new UuidType()); + ensureRegistered(UuidType.INSTANCE); final Schema schema = - new Schema(Collections.singletonList(Field.nullable("a", new UuidType()))); + new Schema(Collections.singletonList(Field.nullable("a", UuidType.INSTANCE))); try (final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { UUID u1 = UUID.randomUUID(); @@ -135,7 +137,7 @@ public void readUnderlyingType() throws IOException { writer.end(); } - ExtensionTypeRegistry.unregister(new UuidType()); + ExtensionTypeRegistry.unregister(UuidType.INSTANCE); try (final SeekableByteChannel channel = Files.newByteChannel(Paths.get(file.getAbsolutePath())); @@ -153,7 +155,7 @@ public void readUnderlyingType() throws IOException { .getByteWidth()); final Field field = readerRoot.getSchema().getFields().get(0); - final UuidType expectedType = new UuidType(); + final UuidType expectedType = UuidType.INSTANCE; assertEquals( field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_NAME), expectedType.extensionName()); @@ -254,7 +256,7 @@ public void roundtripLocation() throws IOException { @Test public void testVectorCompare() { - UuidType uuidType = new UuidType(); + UuidType uuidType = UuidType.INSTANCE; ExtensionTypeRegistry.register(uuidType); try (final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); UuidVector a1 = diff --git a/vector/src/test/java/org/apache/arrow/vector/types/pojo/UuidType.java b/vector/src/test/java/org/apache/arrow/vector/types/pojo/UuidType.java deleted file mode 100644 index 5e2bd8881b..0000000000 --- a/vector/src/test/java/org/apache/arrow/vector/types/pojo/UuidType.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.arrow.vector.types.pojo; - -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.FixedSizeBinaryVector; -import org.apache.arrow.vector.UuidVector; -import org.apache.arrow.vector.types.pojo.ArrowType.ExtensionType; - -public class UuidType extends ExtensionType { - - @Override - public ArrowType storageType() { - return new ArrowType.FixedSizeBinary(16); - } - - @Override - public String extensionName() { - return "uuid"; - } - - @Override - public boolean extensionEquals(ExtensionType other) { - return other instanceof UuidType; - } - - @Override - public ArrowType deserialize(ArrowType storageType, String serializedData) { - if (!storageType.equals(storageType())) { - throw new UnsupportedOperationException( - "Cannot construct UuidType from underlying type " + storageType); - } - return new UuidType(); - } - - @Override - public String serialize() { - return ""; - } - - @Override - public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator) { - return new UuidVector(name, allocator, new FixedSizeBinaryVector(name, allocator, 16)); - } -} From 9dc1410a3f4cc1fcb28a54bbe0e463585af16d08 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Dec 2025 17:41:10 +0100 Subject: [PATCH 232/408] MINOR: Bump dep.hadoop.version from 3.4.1 to 3.4.2 (#915) Bumps `dep.hadoop.version` from 3.4.1 to 3.4.2. Updates `org.apache.hadoop:hadoop-client-runtime` from 3.4.1 to 3.4.2 Updates `org.apache.hadoop:hadoop-client-api` from 3.4.1 to 3.4.2 Updates `org.apache.hadoop:hadoop-common` from 3.4.1 to 3.4.2 Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---

Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5d2567beef..777f7d5cef 100644 --- a/pom.xml +++ b/pom.xml @@ -101,7 +101,7 @@ under the License. 1.73.0 4.33.1 2.18.3 - 3.4.1 + 3.4.2 25.2.10 1.12.0 5.17.0 From 033ecc3f101de35063fb5db83ae5cbcccaee9409 Mon Sep 17 00:00:00 2001 From: ViggoC Date: Thu, 4 Dec 2025 17:30:46 +0800 Subject: [PATCH 233/408] GH-110: bind StringView/BinaryView for Flight SQL JDBC (#905) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What's Changed Closes #110. --------- Co-authored-by: 张林伟 --- .../ArrowFlightJdbcAccessorFactory.java | 8 +++++ .../ArrowFlightJdbcBinaryVectorAccessor.java | 8 +++++ .../ArrowFlightJdbcVarCharVectorAccessor.java | 8 +++++ .../BinaryViewAvaticaParameterConverter.java | 8 ++++- .../Utf8ViewAvaticaParameterConverter.java | 9 ++++- .../jdbc/utils/AvaticaParameterBinder.java | 6 ++-- .../arrow/driver/jdbc/utils/SqlTypes.java | 2 ++ .../ArrowFlightJdbcAccessorFactoryTest.java | 26 ++++++++++++++ ...owFlightJdbcVarCharVectorAccessorTest.java | 26 ++++++++++++++ .../driver/jdbc/utils/ConvertUtilsTest.java | 34 +++++++++++++++++++ .../arrow/driver/jdbc/utils/SqlTypesTest.java | 4 +++ 11 files changed, 135 insertions(+), 4 deletions(-) diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactory.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactory.java index dad1fa5f73..bbfe88a78a 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactory.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactory.java @@ -68,6 +68,8 @@ import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.ViewVarBinaryVector; +import org.apache.arrow.vector.ViewVarCharVector; import org.apache.arrow.vector.complex.DenseUnionVector; import org.apache.arrow.vector.complex.FixedSizeListVector; import org.apache.arrow.vector.complex.LargeListVector; @@ -130,6 +132,9 @@ public static ArrowFlightJdbcAccessor createAccessor( } else if (vector instanceof VarBinaryVector) { return new ArrowFlightJdbcBinaryVectorAccessor( (VarBinaryVector) vector, getCurrentRow, setCursorWasNull); + } else if (vector instanceof ViewVarBinaryVector) { + return new ArrowFlightJdbcBinaryVectorAccessor( + (ViewVarBinaryVector) vector, getCurrentRow, setCursorWasNull); } else if (vector instanceof LargeVarBinaryVector) { return new ArrowFlightJdbcBinaryVectorAccessor( (LargeVarBinaryVector) vector, getCurrentRow, setCursorWasNull); @@ -163,6 +168,9 @@ public static ArrowFlightJdbcAccessor createAccessor( } else if (vector instanceof LargeVarCharVector) { return new ArrowFlightJdbcVarCharVectorAccessor( (LargeVarCharVector) vector, getCurrentRow, setCursorWasNull); + } else if (vector instanceof ViewVarCharVector) { + return new ArrowFlightJdbcVarCharVectorAccessor( + (ViewVarCharVector) vector, getCurrentRow, setCursorWasNull); } else if (vector instanceof DurationVector) { return new ArrowFlightJdbcDurationVectorAccessor( (DurationVector) vector, getCurrentRow, setCursorWasNull); diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/binary/ArrowFlightJdbcBinaryVectorAccessor.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/binary/ArrowFlightJdbcBinaryVectorAccessor.java index 30dfffce64..e71b6380a9 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/binary/ArrowFlightJdbcBinaryVectorAccessor.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/binary/ArrowFlightJdbcBinaryVectorAccessor.java @@ -27,6 +27,7 @@ import org.apache.arrow.vector.FixedSizeBinaryVector; import org.apache.arrow.vector.LargeVarBinaryVector; import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.ViewVarBinaryVector; /** * Accessor for the Arrow types: {@link FixedSizeBinaryVector}, {@link VarBinaryVector} and {@link @@ -61,6 +62,13 @@ public ArrowFlightJdbcBinaryVectorAccessor( this(vector::get, currentRowSupplier, setCursorWasNull); } + public ArrowFlightJdbcBinaryVectorAccessor( + ViewVarBinaryVector vector, + IntSupplier currentRowSupplier, + ArrowFlightJdbcAccessorFactory.WasNullConsumer setCursorWasNull) { + this(vector::get, currentRowSupplier, setCursorWasNull); + } + private ArrowFlightJdbcBinaryVectorAccessor( ByteArrayGetter getter, IntSupplier currentRowSupplier, diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/text/ArrowFlightJdbcVarCharVectorAccessor.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/text/ArrowFlightJdbcVarCharVectorAccessor.java index ebebf6ca74..7b04e89346 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/text/ArrowFlightJdbcVarCharVectorAccessor.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/text/ArrowFlightJdbcVarCharVectorAccessor.java @@ -35,6 +35,7 @@ import org.apache.arrow.driver.jdbc.utils.DateTimeUtils; import org.apache.arrow.vector.LargeVarCharVector; import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.ViewVarCharVector; import org.apache.arrow.vector.util.Text; /** Accessor for the Arrow types: {@link VarCharVector} and {@link LargeVarCharVector}. */ @@ -62,6 +63,13 @@ public ArrowFlightJdbcVarCharVectorAccessor( this(vector::get, currentRowSupplier, setCursorWasNull); } + public ArrowFlightJdbcVarCharVectorAccessor( + ViewVarCharVector vector, + IntSupplier currentRowSupplier, + ArrowFlightJdbcAccessorFactory.WasNullConsumer setCursorWasNull) { + this(vector::get, currentRowSupplier, setCursorWasNull); + } + ArrowFlightJdbcVarCharVectorAccessor( Getter getter, IntSupplier currentRowSupplier, diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/BinaryViewAvaticaParameterConverter.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/BinaryViewAvaticaParameterConverter.java index a035bbba49..d692f39372 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/BinaryViewAvaticaParameterConverter.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/BinaryViewAvaticaParameterConverter.java @@ -17,6 +17,7 @@ package org.apache.arrow.driver.jdbc.converter.impl; import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.ViewVarBinaryVector; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.calcite.avatica.AvaticaParameter; @@ -29,7 +30,12 @@ public BinaryViewAvaticaParameterConverter(ArrowType.BinaryView type) {} @Override public boolean bindParameter(FieldVector vector, TypedValue typedValue, int index) { - throw new UnsupportedOperationException("Not implemented"); + byte[] value = (byte[]) typedValue.toJdbc(null); + if (vector instanceof ViewVarBinaryVector) { + ((ViewVarBinaryVector) vector).setSafe(index, value); + return true; + } + return false; } @Override diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/Utf8ViewAvaticaParameterConverter.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/Utf8ViewAvaticaParameterConverter.java index 076fefc42a..c9d9f2926b 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/Utf8ViewAvaticaParameterConverter.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/Utf8ViewAvaticaParameterConverter.java @@ -17,8 +17,10 @@ package org.apache.arrow.driver.jdbc.converter.impl; import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.ViewVarCharVector; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.util.Text; import org.apache.calcite.avatica.AvaticaParameter; import org.apache.calcite.avatica.remote.TypedValue; @@ -29,7 +31,12 @@ public Utf8ViewAvaticaParameterConverter(ArrowType.Utf8View type) {} @Override public boolean bindParameter(FieldVector vector, TypedValue typedValue, int index) { - throw new UnsupportedOperationException("Utf8View not supported"); + String value = (String) typedValue.toLocal(); + if (vector instanceof ViewVarCharVector) { + ((ViewVarCharVector) vector).setSafe(index, new Text(value)); + return true; + } + return false; } @Override diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java index 0fd99de539..8c98ee4077 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java @@ -19,6 +19,7 @@ import java.util.List; import org.apache.arrow.driver.jdbc.client.ArrowFlightSqlClientHandler.PreparedStatement; import org.apache.arrow.driver.jdbc.converter.impl.BinaryAvaticaParameterConverter; +import org.apache.arrow.driver.jdbc.converter.impl.BinaryViewAvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.BoolAvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.DateAvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.DecimalAvaticaParameterConverter; @@ -39,6 +40,7 @@ import org.apache.arrow.driver.jdbc.converter.impl.TimestampAvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.UnionAvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.Utf8AvaticaParameterConverter; +import org.apache.arrow.driver.jdbc.converter.impl.Utf8ViewAvaticaParameterConverter; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.VectorSchemaRoot; @@ -208,7 +210,7 @@ public Boolean visit(ArrowType.Utf8 type) { @Override public Boolean visit(ArrowType.Utf8View type) { - throw new UnsupportedOperationException("Utf8View is unsupported"); + return new Utf8ViewAvaticaParameterConverter(type).bindParameter(vector, typedValue, index); } @Override @@ -223,7 +225,7 @@ public Boolean visit(ArrowType.Binary type) { @Override public Boolean visit(ArrowType.BinaryView type) { - throw new UnsupportedOperationException("BinaryView is unsupported"); + return new BinaryViewAvaticaParameterConverter(type).bindParameter(vector, typedValue, index); } @Override diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/SqlTypes.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/SqlTypes.java index 1b76ca0c95..5ba3957f8b 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/SqlTypes.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/SqlTypes.java @@ -107,12 +107,14 @@ public static int getSqlTypeIdFromArrowType(ArrowType arrowType) { } break; case Binary: + case BinaryView: return Types.VARBINARY; case FixedSizeBinary: return Types.BINARY; case LargeBinary: return Types.LONGVARBINARY; case Utf8: + case Utf8View: return Types.VARCHAR; case LargeUtf8: return Types.LONGVARCHAR; diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactoryTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactoryTest.java index b56bf3c63d..8b39041f0c 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactoryTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactoryTest.java @@ -46,6 +46,8 @@ import org.apache.arrow.vector.LargeVarCharVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.ViewVarBinaryVector; +import org.apache.arrow.vector.ViewVarCharVector; import org.apache.arrow.vector.complex.DenseUnionVector; import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.StructVector; @@ -239,6 +241,18 @@ public void createAccessorForFixedSizeBinaryVector() { } } + @Test + public void createAccessorForViewVarBinaryVector() { + try (ValueVector valueVector = + new ViewVarBinaryVector("", rootAllocatorTestExtension.getRootAllocator())) { + ArrowFlightJdbcAccessor accessor = + ArrowFlightJdbcAccessorFactory.createAccessor( + valueVector, GET_CURRENT_ROW, (boolean wasNull) -> {}); + + assertTrue(accessor instanceof ArrowFlightJdbcBinaryVectorAccessor); + } + } + @Test public void createAccessorForTimeStampVector() { try (ValueVector valueVector = rootAllocatorTestExtension.createTimeStampMilliVector()) { @@ -340,6 +354,18 @@ public void createAccessorForLargeVarCharVector() { } } + @Test + public void createAccessorForViewVarCharVector() { + try (ValueVector valueVector = + new ViewVarCharVector("", rootAllocatorTestExtension.getRootAllocator())) { + ArrowFlightJdbcAccessor accessor = + ArrowFlightJdbcAccessorFactory.createAccessor( + valueVector, GET_CURRENT_ROW, (boolean wasNull) -> {}); + + assertTrue(accessor instanceof ArrowFlightJdbcVarCharVectorAccessor); + } + } + @Test public void createAccessorForDurationVector() { try (ValueVector valueVector = diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/text/ArrowFlightJdbcVarCharVectorAccessorTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/text/ArrowFlightJdbcVarCharVectorAccessorTest.java index a2f6fd586f..82876f4aa1 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/text/ArrowFlightJdbcVarCharVectorAccessorTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/text/ArrowFlightJdbcVarCharVectorAccessorTest.java @@ -24,6 +24,8 @@ import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.when; @@ -46,6 +48,8 @@ import org.apache.arrow.vector.DateMilliVector; import org.apache.arrow.vector.TimeMilliVector; import org.apache.arrow.vector.TimeStampVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.ViewVarCharVector; import org.apache.arrow.vector.util.Text; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -695,4 +699,26 @@ public void testShouldGetObjectClassReturnString() { final Class clazz = accessor.getObjectClass(); assertThat(clazz, equalTo(String.class)); } + + @Test + public void testViewVarcharVector() throws Exception { + try (VarCharVector varCharVector = + new VarCharVector("", rootAllocatorTestExtension.getRootAllocator()); + ViewVarCharVector viewVarCharVector = + new ViewVarCharVector("", rootAllocatorTestExtension.getRootAllocator())) { + varCharVector.allocateNew(1); + viewVarCharVector.allocateNew(1); + + ArrowFlightJdbcVarCharVectorAccessor varCharVectorAccessor = + new ArrowFlightJdbcVarCharVectorAccessor(varCharVector, () -> 0, (boolean wasNull) -> {}); + ArrowFlightJdbcVarCharVectorAccessor viewVarcharVectorAccessor = + new ArrowFlightJdbcVarCharVectorAccessor( + viewVarCharVector, () -> 0, (boolean wasNull) -> {}); + assertNull(viewVarcharVectorAccessor.getString()); + + varCharVector.set(0, new Text("looooong_string")); + viewVarCharVector.set(0, new Text("looooong_string")); + assertEquals(varCharVectorAccessor.getString(), viewVarcharVectorAccessor.getString()); + } + } } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ConvertUtilsTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ConvertUtilsTest.java index b6fdc99694..f128ca7c73 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ConvertUtilsTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ConvertUtilsTest.java @@ -69,6 +69,19 @@ public void testShouldConvertArrowFieldsToColumnMetaDataList() { .tableName("table1") .build() .getMetadataMap()), + null), + new Field( + "col2", + new FieldType( + true, + ArrowType.Utf8View.INSTANCE, + null, + new FlightSqlColumnMetadata.Builder() + .catalogName("catalog1") + .schemaName("schema1") + .tableName("table1") + .build() + .getMetadataMap()), null)); final List expectedColumnMetaData = @@ -78,6 +91,25 @@ public void testShouldConvertArrowFieldsToColumnMetaDataList() { .setCatalogName("catalog1") .setSchemaName("schema1") .setTableName("table1") + .setColumnName("col1") + .setType( + Common.AvaticaType.newBuilder() + .setId(SqlTypes.getSqlTypeIdFromArrowType(ArrowType.Utf8.INSTANCE)) + .setName(SqlTypes.getSqlTypeNameFromArrowType(ArrowType.Utf8.INSTANCE)) + .build()) + .build()), + ColumnMetaData.fromProto( + Common.ColumnMetaData.newBuilder() + .setCatalogName("catalog1") + .setSchemaName("schema1") + .setTableName("table1") + .setColumnName("col2") + .setType( + Common.AvaticaType.newBuilder() + .setId(SqlTypes.getSqlTypeIdFromArrowType(ArrowType.Utf8View.INSTANCE)) + .setName( + SqlTypes.getSqlTypeNameFromArrowType(ArrowType.Utf8View.INSTANCE)) + .build()) .build())); final List actualColumnMetaData = @@ -95,6 +127,8 @@ private void assertColumnMetaData( assertThat(expectedColumnMetaData.catalogName, equalTo(actualColumnMetaData.catalogName)); assertThat(expectedColumnMetaData.schemaName, equalTo(actualColumnMetaData.schemaName)); assertThat(expectedColumnMetaData.tableName, equalTo(actualColumnMetaData.tableName)); + assertThat(expectedColumnMetaData.columnName, equalTo(actualColumnMetaData.columnName)); + assertThat(expectedColumnMetaData.type, equalTo(actualColumnMetaData.type)); assertThat(expectedColumnMetaData.readOnly, equalTo(actualColumnMetaData.readOnly)); assertThat(expectedColumnMetaData.autoIncrement, equalTo(actualColumnMetaData.autoIncrement)); assertThat(expectedColumnMetaData.precision, equalTo(actualColumnMetaData.precision)); diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/SqlTypesTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/SqlTypesTest.java index a6dd6b3275..d69c549296 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/SqlTypesTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/SqlTypesTest.java @@ -40,9 +40,11 @@ public void testGetSqlTypeIdFromArrowType() { assertEquals(Types.BINARY, getSqlTypeIdFromArrowType(new ArrowType.FixedSizeBinary(1024))); assertEquals(Types.VARBINARY, getSqlTypeIdFromArrowType(new ArrowType.Binary())); + assertEquals(Types.VARBINARY, getSqlTypeIdFromArrowType(new ArrowType.BinaryView())); assertEquals(Types.LONGVARBINARY, getSqlTypeIdFromArrowType(new ArrowType.LargeBinary())); assertEquals(Types.VARCHAR, getSqlTypeIdFromArrowType(new ArrowType.Utf8())); + assertEquals(Types.VARCHAR, getSqlTypeIdFromArrowType(new ArrowType.Utf8View())); assertEquals(Types.LONGVARCHAR, getSqlTypeIdFromArrowType(new ArrowType.LargeUtf8())); assertEquals(Types.DATE, getSqlTypeIdFromArrowType(new ArrowType.Date(DateUnit.MILLISECOND))); @@ -94,9 +96,11 @@ public void testGetSqlTypeNameFromArrowType() { assertEquals("BINARY", getSqlTypeNameFromArrowType(new ArrowType.FixedSizeBinary(1024))); assertEquals("VARBINARY", getSqlTypeNameFromArrowType(new ArrowType.Binary())); + assertEquals("VARBINARY", getSqlTypeNameFromArrowType(new ArrowType.BinaryView())); assertEquals("LONGVARBINARY", getSqlTypeNameFromArrowType(new ArrowType.LargeBinary())); assertEquals("VARCHAR", getSqlTypeNameFromArrowType(new ArrowType.Utf8())); + assertEquals("VARCHAR", getSqlTypeNameFromArrowType(new ArrowType.Utf8View())); assertEquals("LONGVARCHAR", getSqlTypeNameFromArrowType(new ArrowType.LargeUtf8())); assertEquals("DATE", getSqlTypeNameFromArrowType(new ArrowType.Date(DateUnit.MILLISECOND))); From 2329174e0bf35733b5ac99b7e36465a601ae20e8 Mon Sep 17 00:00:00 2001 From: Pedro Matias Date: Thu, 4 Dec 2025 09:33:49 +0000 Subject: [PATCH 234/408] GH-863: [JDBC] Suppress benign exceptions from gRPC layer on ArrowFlightSqlClientHandler#close (#910) ## What's Changed When using the Flight SQL JDBC driver with connection pooling and a catalog parameter, ArrowFlightSqlClientHandler.close() performs a CloseSession RPC that can fail during gRPC channel shutdown. These transient failures (UNAVAILABLE or INTERNAL with "Connection closed after GOAWAY") cause noisy errors in pooling frameworks like Apache Commons DBCP. With this PR these exceptions will instead be suppressed and logged, following the procedure that was used for [ARROW-17785](https://issues.apache.org/jira/browse/ARROW-17785) ### Are these changes tested? Yes Closes #863 --------- Co-authored-by: Vando Pereira --- .../client/ArrowFlightSqlClientHandler.java | 81 +++++++++++++++-- .../ArrowFlightSqlClientHandlerTest.java | 88 +++++++++++++++++++ 2 files changed, 160 insertions(+), 9 deletions(-) create mode 100644 flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerTest.java diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java index a3f6900373..5dc7e0e2e9 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java @@ -262,15 +262,85 @@ public FlightInfo getInfo(final String query) { @Override public void close() throws SQLException { if (catalog.isPresent()) { - sqlClient.closeSession(new CloseSessionRequest(), getOptions()); + try { + sqlClient.closeSession(new CloseSessionRequest(), getOptions()); + } catch (FlightRuntimeException fre) { + handleBenignCloseException( + fre, "Failed to close Flight SQL session.", "closing Flight SQL session"); + } } try { AutoCloseables.close(sqlClient); + } catch (FlightRuntimeException fre) { + handleBenignCloseException( + fre, "Failed to clean up client resources.", "closing Flight SQL client"); } catch (final Exception e) { throw new SQLException("Failed to clean up client resources.", e); } } + /** + * Handles FlightRuntimeException during close operations, suppressing benign gRPC shutdown errors + * while re-throwing genuine failures. + * + * @param fre the FlightRuntimeException to handle + * @param sqlErrorMessage the SQLException message to use for genuine failures + * @param operationDescription description of the operation for logging + * @throws SQLException if the exception represents a genuine failure + */ + private void handleBenignCloseException( + FlightRuntimeException fre, String sqlErrorMessage, String operationDescription) + throws SQLException { + if (isBenignCloseException(fre)) { + logSuppressedCloseException(fre, operationDescription); + } else { + throw new SQLException(sqlErrorMessage, fre); + } + } + + /** + * Handles FlightRuntimeException during close operations, suppressing benign gRPC shutdown errors + * while re-throwing genuine failures as FlightRuntimeException. + * + * @param fre the FlightRuntimeException to handle + * @param operationDescription description of the operation for logging + * @throws FlightRuntimeException if the exception represents a genuine failure + */ + private void handleBenignCloseException(FlightRuntimeException fre, String operationDescription) + throws FlightRuntimeException { + if (isBenignCloseException(fre)) { + logSuppressedCloseException(fre, operationDescription); + } else { + throw fre; + } + } + + /** + * Determines if a FlightRuntimeException represents a benign close operation error that should be + * suppressed. + * + * @param fre the FlightRuntimeException to check + * @return true if the exception should be suppressed, false otherwise + */ + private boolean isBenignCloseException(FlightRuntimeException fre) { + return fre.status().code().equals(FlightStatusCode.UNAVAILABLE) + || (fre.status().code().equals(FlightStatusCode.INTERNAL) + && fre.getMessage() != null + && fre.getMessage().contains("Connection closed after GOAWAY")); + } + + /** + * Logs a suppressed close exception with appropriate level based on debug settings. + * + * @param fre the FlightRuntimeException being suppressed + * @param operationDescription description of the operation for logging + */ + private void logSuppressedCloseException( + FlightRuntimeException fre, String operationDescription) { + // ARROW-17785 and GH-863: suppress exceptions caused by flaky gRPC layer during shutdown + LOGGER.debug("Suppressed error {}", operationDescription, fre); + } + /** A prepared statement handler. */ public interface PreparedStatement extends AutoCloseable { /** @@ -386,14 +456,7 @@ public void close() { try { preparedStatement.close(getOptions()); } catch (FlightRuntimeException fre) { - // ARROW-17785: suppress exceptions caused by flaky gRPC layer - if (fre.status().code().equals(FlightStatusCode.UNAVAILABLE) - || (fre.status().code().equals(FlightStatusCode.INTERNAL) - && fre.getMessage().contains("Connection closed after GOAWAY"))) { - LOGGER.warn("Supressed error closing PreparedStatement", fre); - return; - } - throw fre; + handleBenignCloseException(fre, "closing PreparedStatement"); } } }; diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerTest.java new file mode 100644 index 0000000000..d5973ab5d8 --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerTest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.client; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Optional; +import org.apache.arrow.flight.CallOption; +import org.apache.arrow.flight.CallStatus; +import org.apache.arrow.flight.CloseSessionRequest; +import org.apache.arrow.flight.FlightStatusCode; +import org.apache.arrow.flight.sql.FlightSqlClient; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +public class ArrowFlightSqlClientHandlerTest { + + @ParameterizedTest + @MethodSource + public void testCloseHandlesFlightRuntimeException( + boolean throwFromCloseSession, CallStatus callStatus, boolean shouldSuppress) + throws Exception { + FlightSqlClient sqlClient = mock(FlightSqlClient.class); + String cacheKey = "cacheKey"; + Optional catalog = + throwFromCloseSession ? Optional.of("test_catalog") : Optional.empty(); + final Collection credentialOptions = new ArrayList<>(); + ArrowFlightSqlClientHandler.Builder builder = new ArrowFlightSqlClientHandler.Builder(); + + if (throwFromCloseSession) { + doThrow(callStatus.toRuntimeException()) + .when(sqlClient) + .closeSession(any(CloseSessionRequest.class), any(CallOption[].class)); + } else { + doThrow(callStatus.toRuntimeException()).when(sqlClient).close(); + } + + ArrowFlightSqlClientHandler sqlClientHandler = + new ArrowFlightSqlClientHandler( + cacheKey, sqlClient, builder, credentialOptions, catalog, null); + + if (shouldSuppress) { + assertDoesNotThrow(sqlClientHandler::close); + } else { + assertThrows(SQLException.class, sqlClientHandler::close); + } + } + + private static Object[] testCloseHandlesFlightRuntimeException() { + CallStatus benignInternalError = + new CallStatus(FlightStatusCode.INTERNAL, null, "Connection closed after GOAWAY", null); + CallStatus notBenignInternalError = + new CallStatus(FlightStatusCode.INTERNAL, null, "Not a benign internal error", null); + CallStatus unavailableError = new CallStatus(FlightStatusCode.UNAVAILABLE, null, null, null); + CallStatus unknownError = new CallStatus(FlightStatusCode.UNKNOWN, null, null, null); + return new Object[] { + new Object[] {true, benignInternalError, true}, + new Object[] {false, benignInternalError, true}, + new Object[] {true, notBenignInternalError, false}, + new Object[] {false, notBenignInternalError, false}, + new Object[] {true, unavailableError, true}, + new Object[] {false, unavailableError, true}, + new Object[] {true, unknownError, false}, + new Object[] {false, unknownError, false}, + }; + } +} From 92816711d9a7da18ec2037395c71f9293a0f0301 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 08:48:02 +0100 Subject: [PATCH 235/408] MINOR: [CI] Bump actions/upload-artifact from 5.0.0 to 6.0.0 (#934) --- .github/workflows/rc.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 0ae4399ef1..b71d49b314 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -71,7 +71,7 @@ jobs: run: | dev/release/run_rat.sh "${TAR_GZ}" - name: Upload source archive - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: release-source path: | @@ -154,7 +154,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf jni-linux-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: jni-linux-${{ matrix.platform.arch }} path: jni-linux-${{ matrix.platform.arch }}.tar.gz @@ -284,7 +284,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf jni-macos-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: jni-macos-${{ matrix.platform.arch }} path: jni-macos-${{ matrix.platform.arch }}.tar.gz @@ -368,7 +368,7 @@ jobs: shell: bash run: tar -cvzf jni-windows-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: jni-windows-${{ matrix.platform.arch }} path: jni-windows-${{ matrix.platform.arch }}.tar.gz @@ -440,12 +440,12 @@ jobs: cp -a target/site/apidocs reference tar -cvzf reference.tar.gz reference - name: Upload binaries - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: release-binaries path: binaries/* - name: Upload docs - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: reference path: reference.tar.gz @@ -483,7 +483,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf html.tar.gz -C docs/build html - name: Upload artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: release-html path: html.tar.gz From 8f616f47483cd2d4a0ca169933da4d7565756928 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 08:48:26 +0100 Subject: [PATCH 236/408] MINOR: [CI] Bump actions/download-artifact from 6.0.0 to 7.0.0 (#935) --- .github/workflows/rc.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index b71d49b314..5dcbc664a7 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -101,7 +101,7 @@ jobs: packages: write steps: - name: Download source archive - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: release-source - name: Extract source archive @@ -174,7 +174,7 @@ jobs: MACOSX_DEPLOYMENT_TARGET: "14.0" steps: - name: Download source archive - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: release-source - name: Extract source archive @@ -302,7 +302,7 @@ jobs: arch: "x86_64" steps: - name: Download source archive - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: release-source - name: Extract source archive @@ -381,7 +381,7 @@ jobs: - jni-windows steps: - name: Download artifacts - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: path: artifacts - name: Decompress artifacts @@ -462,11 +462,11 @@ jobs: with: cache: 'pip' - name: Download source archive - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: release-source - name: Download Javadocs - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: reference - name: Extract source archive @@ -531,7 +531,7 @@ jobs: cp ../.asf.yaml ./ git add .nojekyll .asf.yaml - name: Download - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: release-html - name: Extract @@ -567,7 +567,7 @@ jobs: - ubuntu-latest steps: - name: Download release artifacts - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: pattern: release-* - name: Verify @@ -601,7 +601,7 @@ jobs: contents: write steps: - name: Download release artifacts - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: pattern: release-* path: artifacts From 96156ccc2bf933c75c852ca7c04418a61f87defd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 08:48:46 +0100 Subject: [PATCH 237/408] MINOR: [CI] Bump actions/cache from 4 to 5 (#936) --- .github/workflows/dev.yml | 2 +- .github/workflows/rc.yml | 8 ++++---- .github/workflows/test.yml | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 57432d75f1..2111f47254 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -42,7 +42,7 @@ jobs: with: python-version: '3.x' - name: pre-commit (cache) - uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 with: path: ~/.cache/pre-commit key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 5dcbc664a7..efa69533d3 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -139,7 +139,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Cache - uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 with: path: .docker key: jni-linux-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} @@ -270,7 +270,7 @@ jobs: run: | echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} - name: Cache ccache - uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 with: path: ccache key: jni-macos-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} @@ -352,7 +352,7 @@ jobs: run: | echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} - name: Cache ccache - uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 with: path: ccache key: jni-windows-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} @@ -426,7 +426,7 @@ jobs: repository: apache/arrow-testing path: testing - name: Cache ~/.m2 - uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 with: path: ~/.m2 key: binaries-build-${{ hashFiles('**/*.java', '**/pom.xml') }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8d8fcb5052..e1beea793e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,7 +63,7 @@ jobs: fetch-depth: 0 submodules: recursive - name: Cache Docker Volumes - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: .docker key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('compose.yaml', '**/pom.xml', '**/*.java') }} @@ -190,7 +190,7 @@ jobs: run: | ci/scripts/util_free_space.sh - name: Cache Docker Volumes - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: .docker key: integration-conda-${{ hashFiles('cpp/**') }} From 8d1802cef6bacd27d1cc8a0c2cf3efcddb6bf255 Mon Sep 17 00:00:00 2001 From: Kaustav Sarkar <70840177+Kaustav-Sarkar@users.noreply.github.com> Date: Mon, 29 Dec 2025 14:01:10 +0530 Subject: [PATCH 238/408] GH-399: Check for null writers in DenseUnionWriter#setPosition (#938) ## GH-399 Fix setPosition fails with NullPointerException Fixed a `NullPointerException` in `DenseUnionWriter#setPosition`. The issue was that `setPosition` tried to update all writers in its internal array, even if they hadn't been initialized yet. I added a null check so it only updates writers that actually exist. Also added a regression test (`TestDenseUnionWriterNPE`) to verify the fix and updated `.gitignore`. Closes #399. --- .gitignore | 2 + .../codegen/templates/DenseUnionWriter.java | 4 +- .../complex/writer/TestComplexWriter.java | 43 +++++++++++++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b57597af47..17d1d43ae1 100644 --- a/.gitignore +++ b/.gitignore @@ -7,10 +7,12 @@ .buildpath .checkstyle .classpath +.cursor/ .factorypath .idea/ .project .settings/ +.vscode/ /*-build/ /.mvn/.develocity/ /apache-arrow-java-* diff --git a/vector/src/main/codegen/templates/DenseUnionWriter.java b/vector/src/main/codegen/templates/DenseUnionWriter.java index 8515b759e6..9aeea5b054 100644 --- a/vector/src/main/codegen/templates/DenseUnionWriter.java +++ b/vector/src/main/codegen/templates/DenseUnionWriter.java @@ -55,7 +55,9 @@ public DenseUnionWriter(DenseUnionVector vector, NullableStructWriterFactory nul public void setPosition(int index) { super.setPosition(index); for (BaseWriter writer : writers) { - writer.setPosition(index); + if (writer != null) { + writer.setPosition(index); + } } } diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index 46c259bda0..871a3cc461 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -2530,4 +2530,47 @@ public void extensionWriterReader() throws Exception { } } } + + @Test + void testListOfDenseUnionWriterNPE() { + // Regression test for https://github.com/apache/arrow-java/issues/399 + try (ListVector listVector = ListVector.empty("list", allocator)) { + listVector.addOrGetVector(FieldType.nullable(MinorType.DENSEUNION.getType())); + UnionListWriter listWriter = listVector.getWriter(); + + listWriter.startList(); + listWriter.endList(); + } + } + + @Test + void testListOfDenseUnionWriterWithData() { + try (ListVector listVector = ListVector.empty("list", allocator)) { + listVector.addOrGetVector(FieldType.nullable(MinorType.DENSEUNION.getType())); + + UnionListWriter listWriter = listVector.getWriter(); + listWriter.startList(); + listWriter.writeInt(100); + listWriter.writeBigInt(200L); + listWriter.endList(); + + listWriter.startList(); + listWriter.writeFloat4(3.14f); + listWriter.endList(); + + listVector.setValueCount(2); + + assertEquals(2, listVector.getValueCount()); + + List value0 = (List) listVector.getObject(0); + List value1 = (List) listVector.getObject(1); + + assertEquals(2, value0.size()); + assertEquals(100, value0.get(0)); + assertEquals(200L, value0.get(1)); + + assertEquals(1, value1.size()); + assertEquals(3.14f, value1.get(0)); + } + } } From 385b51eb5bf001e34d71d63b5980a5eabaa38294 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 29 Dec 2025 19:44:18 +0900 Subject: [PATCH 239/408] MINOR: Update macos amd64 runner (#940) ## What's Changed Replace the macOS 13 runner. --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e1beea793e..2602592799 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -89,7 +89,7 @@ jobs: include: - arch: AMD64 jdk: 11 - macos: 13 + macos: 15-intel - arch: AArch64 jdk: 11 macos: latest From b9e40fa0a90143eef36ca53d2f937f2fef494402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Eickler?= <797483+eickler@users.noreply.github.com> Date: Mon, 5 Jan 2026 07:39:15 +0100 Subject: [PATCH 240/408] GH-942: Fix JDBC Connection.setCatalog() (#943) ## What's Changed Connection.setCatalog() is not silently ignored anymore (through the default implementation in Calcite) but instead it updates the catalog session option in the same way as during the initial connection. Closes #942. --- .../driver/jdbc/ArrowFlightMetaImpl.java | 23 +++++- .../client/ArrowFlightSqlClientHandler.java | 80 ++++++++++++------- .../arrow/driver/jdbc/ConnectionTest.java | 40 +++++++++- .../jdbc/utils/MockFlightSqlProducer.java | 19 +++++ 4 files changed, 131 insertions(+), 31 deletions(-) diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java index 21cc3e431f..64529b50c8 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java @@ -79,7 +79,8 @@ static Signature newSignature(final String sql, Schema resultSetSchema, Schema p public void closeStatement(final StatementHandle statementHandle) { PreparedStatement preparedStatement = statementHandlePreparedStatementMap.remove(new StatementHandleKey(statementHandle)); - // Testing if the prepared statement was created because the statement can be not created until + // Testing if the prepared statement was created because the statement can be + // not created until // this moment if (preparedStatement != null) { preparedStatement.close(); @@ -224,7 +225,8 @@ public ExecuteResult prepareAndExecute( MetaResultSet.create(handle.connectionId, handle.id, false, handle.signature, null); return new ExecuteResult(Collections.singletonList(metaResultSet)); } catch (SQLTimeoutException e) { - // So far AvaticaStatement(executeInternal) only handles NoSuchStatement and Runtime + // So far AvaticaStatement(executeInternal) only handles NoSuchStatement and + // Runtime // Exceptions. throw new RuntimeException(e); } catch (SQLException e) { @@ -253,6 +255,20 @@ public boolean syncResults( return false; } + @Override + public ConnectionProperties connectionSync(ConnectionHandle ch, ConnectionProperties connProps) { + final ConnectionProperties result = super.connectionSync(ch, connProps); + final String newCatalog = this.connProps.getCatalog(); + if (newCatalog != null) { + try { + ((ArrowFlightConnection) connection).getClientHandler().setCatalog(newCatalog); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + return result; + } + void setDefaultConnectionProperties() { // TODO Double-check this. connProps @@ -268,7 +284,8 @@ PreparedStatement getPreparedStatement(StatementHandle statementHandle) { return statementHandlePreparedStatementMap.get(new StatementHandleKey(statementHandle)); } - // Helper used to look up prepared statement instances later. Avatica doesn't give us the + // Helper used to look up prepared statement instances later. Avatica doesn't + // give us the // signature in // an UPDATE code path so we can't directly use StatementHandle as a map key. private static final class StatementHandleKey { diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java index 5dc7e0e2e9..666996cd95 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java @@ -47,7 +47,6 @@ import org.apache.arrow.flight.FlightStatusCode; import org.apache.arrow.flight.Location; import org.apache.arrow.flight.LocationSchemes; -import org.apache.arrow.flight.SessionOptionValue; import org.apache.arrow.flight.SessionOptionValueFactory; import org.apache.arrow.flight.SetSessionOptionsRequest; import org.apache.arrow.flight.SetSessionOptionsResult; @@ -147,20 +146,26 @@ public List getStreams(final FlightInfo flightInfo) try { for (FlightEndpoint endpoint : flightInfo.getEndpoints()) { if (endpoint.getLocations().isEmpty()) { - // Create a stream using the current client only and do not close the client at the end. + // Create a stream using the current client only and do not close the client at + // the end. endpoints.add( new CloseableEndpointStreamPair( sqlClient.getStream(endpoint.getTicket(), getOptions()), null)); } else { // Clone the builder and then set the new endpoint on it. - // GH-38574: Currently a new FlightClient will be made for each partition that returns a - // non-empty Location then disposed of. It may be better to cache clients because a server - // may report the same Locations. It would also be good to identify when the reported + // GH-38574: Currently a new FlightClient will be made for each partition that + // returns a + // non-empty Location then disposed of. It may be better to cache clients + // because a server + // may report the same Locations. It would also be good to identify when the + // reported // location - // is the same as the original connection's Location and skip creating a FlightClient in + // is the same as the original connection's Location and skip creating a + // FlightClient in // that scenario. - // Also copy the cache to the client so we can share a cache. Cache needs to cache + // Also copy the cache to the client so we can share a cache. Cache needs to + // cache // negative attempts too. List exceptions = new ArrayList<>(); CloseableEndpointStreamPair stream = null; @@ -337,7 +342,8 @@ private boolean isBenignCloseException(FlightRuntimeException fre) { */ private void logSuppressedCloseException( FlightRuntimeException fre, String operationDescription) { - // ARROW-17785 and GH-863: suppress exceptions caused by flaky gRPC layer during shutdown + // ARROW-17785 and GH-863: suppress exceptions caused by flaky gRPC layer during + // shutdown LOGGER.debug("Suppressed error {}", operationDescription, fre); } @@ -388,25 +394,40 @@ public interface PreparedStatement extends AutoCloseable { /** A connection is created with catalog set as a session option. */ private void setSetCatalogInSessionIfPresent() { if (catalog.isPresent()) { - final SetSessionOptionsRequest setSessionOptionRequest = - new SetSessionOptionsRequest( - ImmutableMap.builder() - .put(CATALOG, SessionOptionValueFactory.makeSessionOptionValue(catalog.get())) - .build()); - final SetSessionOptionsResult result = - sqlClient.setSessionOptions(setSessionOptionRequest, getOptions()); + try { + setCatalog(catalog.get()); + } catch (SQLException e) { + throw CallStatus.INVALID_ARGUMENT + .withDescription(e.getMessage()) + .withCause(e) + .toRuntimeException(); + } + } + } + /** + * Sets the catalog for the current session. + * + * @param catalog the catalog to set. + * @throws SQLException if an error occurs while setting the catalog. + */ + public void setCatalog(final String catalog) throws SQLException { + final SetSessionOptionsRequest request = + new SetSessionOptionsRequest( + ImmutableMap.of(CATALOG, SessionOptionValueFactory.makeSessionOptionValue(catalog))); + try { + final SetSessionOptionsResult result = sqlClient.setSessionOptions(request, getOptions()); if (result.hasErrors()) { - Map errors = result.getErrors(); - for (Map.Entry error : errors.entrySet()) { + final Map errors = result.getErrors(); + for (final Map.Entry error : errors.entrySet()) { LOGGER.warn(error.toString()); } - throw CallStatus.INVALID_ARGUMENT - .withDescription( - String.format( - "Cannot set session option for catalog = %s. Check log for details.", catalog)) - .toRuntimeException(); + throw new SQLException( + String.format( + "Cannot set session option for catalog = %s. Check log for details.", catalog)); } + } catch (final FlightRuntimeException e) { + throw new SQLException(e); } } @@ -654,7 +675,8 @@ public static final class Builder { @VisibleForTesting @Nullable Duration connectTimeout; - // These two middleware are for internal use within build() and should not be exposed by builder + // These two middleware are for internal use within build() and should not be + // exposed by builder // APIs. // Note that these middleware may not necessarily be registered. @VisibleForTesting @@ -980,7 +1002,8 @@ public Location getLocation() { * @throws SQLException on error. */ public ArrowFlightSqlClientHandler build() throws SQLException { - // Copy middleware so that the build method doesn't change the state of the builder fields + // Copy middleware so that the build method doesn't change the state of the + // builder fields // itself. Set buildTimeMiddlewareFactories = new HashSet<>(this.middlewareFactories); @@ -988,7 +1011,8 @@ public ArrowFlightSqlClientHandler build() throws SQLException { boolean isUsingUserPasswordAuth = username != null && token == null; try { - // Token should take priority since some apps pass in a username/password even when a token + // Token should take priority since some apps pass in a username/password even + // when a token // is provided if (isUsingUserPasswordAuth) { buildTimeMiddlewareFactories.add(authFactory); @@ -1047,8 +1071,10 @@ public ArrowFlightSqlClientHandler build() throws SQLException { allocator, channelBuilder.build(), clientBuilder.middleware()); final ArrayList credentialOptions = new ArrayList<>(); if (isUsingUserPasswordAuth) { - // If the authFactory has already been used for a handshake, use the existing token. - // This can occur if the authFactory is being re-used for a new connection spawned for + // If the authFactory has already been used for a handshake, use the existing + // token. + // This can occur if the authFactory is being re-used for a new connection + // spawned for // getStream(). if (authFactory.getCredentialCallOption() != null) { credentialOptions.add(authFactory.getCredentialCallOption()); diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java index 72e4b222a3..46762f3319 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.driver.jdbc; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -26,12 +27,15 @@ import java.sql.Driver; import java.sql.DriverManager; import java.sql.SQLException; +import java.util.Map; import java.util.Properties; import org.apache.arrow.driver.jdbc.authentication.UserPasswordAuthentication; import org.apache.arrow.driver.jdbc.client.ArrowFlightSqlClientHandler; import org.apache.arrow.driver.jdbc.utils.ArrowFlightConnectionConfigImpl.ArrowFlightConnectionProperty; import org.apache.arrow.driver.jdbc.utils.MockFlightSqlProducer; import org.apache.arrow.flight.FlightMethod; +import org.apache.arrow.flight.NoOpSessionOptionValueVisitor; +import org.apache.arrow.flight.SessionOptionValue; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.util.AutoCloseables; @@ -614,7 +618,8 @@ public void testJdbcDriverVersionIntegration() throws Exception { var expectedUserAgent = "JDBC Flight SQL Driver " + driverVersion.getDriverVersion().versionString; - // Driver appends version to grpc user-agent header. Assert the header starts with the + // Driver appends version to grpc user-agent header. Assert the header starts + // with the // expected // value and ignored grpc version. assertTrue( @@ -622,4 +627,37 @@ public void testJdbcDriverVersionIntegration() throws Exception { "Expected: " + expectedUserAgent + " but found: " + actualUserAgent); } } + + @Test + public void testSetCatalogShouldUpdateSessionOptions() throws Exception { + final Properties properties = new Properties(); + properties.put(ArrowFlightConnectionProperty.USER.camelName(), userTest); + properties.put(ArrowFlightConnectionProperty.PASSWORD.camelName(), passTest); + properties.put("useEncryption", false); + + try (Connection connection = + DriverManager.getConnection( + "jdbc:arrow-flight-sql://" + + FLIGHT_SERVER_TEST_EXTENSION.getHost() + + ":" + + FLIGHT_SERVER_TEST_EXTENSION.getPort(), + properties)) { + final String catalog = "new_catalog"; + connection.setCatalog(catalog); + + final Map options = PRODUCER.getSessionOptions(); + assertTrue(options.containsKey("catalog")); + String actualCatalog = + options + .get("catalog") + .acceptVisitor( + new NoOpSessionOptionValueVisitor() { + @Override + public String visit(String value) { + return value; + } + }); + assertEquals(catalog, actualCatalog); + } + } } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/MockFlightSqlProducer.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/MockFlightSqlProducer.java index a8874c4869..45c2a96404 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/MockFlightSqlProducer.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/MockFlightSqlProducer.java @@ -52,6 +52,9 @@ import org.apache.arrow.flight.PutResult; import org.apache.arrow.flight.Result; import org.apache.arrow.flight.SchemaResult; +import org.apache.arrow.flight.SessionOptionValue; +import org.apache.arrow.flight.SetSessionOptionsRequest; +import org.apache.arrow.flight.SetSessionOptionsResult; import org.apache.arrow.flight.Ticket; import org.apache.arrow.flight.sql.FlightSqlProducer; import org.apache.arrow.flight.sql.SqlInfoBuilder; @@ -664,6 +667,22 @@ public SqlInfoBuilder getSqlInfoBuilder() { return sqlInfoBuilder; } + private final Map sessionOptions = new HashMap<>(); + + @Override + public void setSessionOptions( + final SetSessionOptionsRequest request, + final CallContext context, + final StreamListener listener) { + sessionOptions.putAll(request.getSessionOptions()); + listener.onNext(new SetSessionOptionsResult(Collections.emptyMap())); + listener.onCompleted(); + } + + public Map getSessionOptions() { + return sessionOptions; + } + private static final class TicketConversionUtils { private TicketConversionUtils() { // Prevent instantiation. From 94dfea8cb20a8e92efe5a188cc7fe5fd28702231 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Thu, 8 Jan 2026 14:11:06 +0100 Subject: [PATCH 241/408] GH-951: Fix CI completely, especially JNI on Windows 2022 and MacOS platforms (#925) This fixes #951 --- ci/scripts/jni_macos_build.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/scripts/jni_macos_build.sh b/ci/scripts/jni_macos_build.sh index 13c0675d38..65ab450666 100755 --- a/ci/scripts/jni_macos_build.sh +++ b/ci/scripts/jni_macos_build.sh @@ -77,7 +77,9 @@ cmake \ cmake --build "${build_dir}/cpp" --target install github_actions_group_end -export JAVA_JNI_CMAKE_ARGS="-DProtobuf_ROOT=${build_dir}/cpp/protobuf_ep-install" +JAVA_JNI_CMAKE_ARGS="-DProtobuf_ROOT=${build_dir}/cpp/_deps/protobuf-build" +JAVA_JNI_CMAKE_ARGS+=" -DProtobuf_SRC_ROOT_FOLDER=${build_dir}/cpp/_deps/protobuf-src" +export JAVA_JNI_CMAKE_ARGS "${source_dir}/ci/scripts/jni_build.sh" \ "${source_dir}" \ "${install_dir}" \ From 007744191764d93628bdbf29084cc77c3c6aac5a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Jan 2026 15:46:59 +0100 Subject: [PATCH 242/408] MINOR: Bump checker.framework.version from 3.52.0 to 3.52.1 (#927) Bumps `checker.framework.version` from 3.52.0 to 3.52.1. Updates `org.checkerframework:checker-qual` from 3.52.0 to 3.52.1
Release notes

Sourced from org.checkerframework:checker-qual's releases.

Checker Framework 3.52.1

Version 3.52.1 (2025-12-02)

User-visible changes:

Added Opt.ifPresentOrElse() method.

Closed issues: #7243, #7398.

Changelog

Sourced from org.checkerframework:checker-qual's changelog.

Version 3.52.1 (2025-12-02)

User-visible changes:

Added Opt.ifPresentOrElse() method.

Closed issues: #7243, #7398.

Commits
  • 72c0de5 new release 3.52.1
  • 0549f00 Remove link.
  • 85842ab Prep for release.
  • 294c7ca Fix the dataflow shaded jars that are published. (#7404)
  • cd7c953 Update cimg/base Docker tag to v2025.12 (#7403)
  • 026bd52 Link from the developer manual to "building from source" in the manual (#7385)
  • e086cba More signature annotations
  • 31ca5d3 Correct shaded dataflow jars. (#7402)
  • 08cc5d1 Update dependency com.amazonaws:aws-java-sdk-bom to v1.12.794 (#7401)
  • 4a22556 Nullness annotations for java.lang.classfile
  • Additional commits viewable in compare view

Updates `org.checkerframework:checker` from 3.52.0 to 3.52.1
Release notes

Sourced from org.checkerframework:checker's releases.

Checker Framework 3.52.1

Version 3.52.1 (2025-12-02)

User-visible changes:

Added Opt.ifPresentOrElse() method.

Closed issues: #7243, #7398.

Changelog

Sourced from org.checkerframework:checker's changelog.

Version 3.52.1 (2025-12-02)

User-visible changes:

Added Opt.ifPresentOrElse() method.

Closed issues: #7243, #7398.

Commits
  • 72c0de5 new release 3.52.1
  • 0549f00 Remove link.
  • 85842ab Prep for release.
  • 294c7ca Fix the dataflow shaded jars that are published. (#7404)
  • cd7c953 Update cimg/base Docker tag to v2025.12 (#7403)
  • 026bd52 Link from the developer manual to "building from source" in the manual (#7385)
  • e086cba More signature annotations
  • 31ca5d3 Correct shaded dataflow jars. (#7402)
  • 08cc5d1 Update dependency com.amazonaws:aws-java-sdk-bom to v1.12.794 (#7401)
  • 4a22556 Nullness annotations for java.lang.classfile
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 777f7d5cef..8cfd4eceff 100644 --- a/pom.xml +++ b/pom.xml @@ -110,7 +110,7 @@ under the License. 10.23.0 true 2.42.0 - 3.52.0 + 3.53.0 1.5.21 none -Xdoclint:none From 32ea946a99d1a0276fc61390eefd2b9b12a8fcf7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Jan 2026 17:25:06 +0100 Subject: [PATCH 243/408] MINOR: Bump org.jacoco:jacoco-maven-plugin from 0.8.13 to 0.8.14 (#924) Bumps [org.jacoco:jacoco-maven-plugin](https://github.com/jacoco/jacoco) from 0.8.13 to 0.8.14.
Release notes

Sourced from org.jacoco:jacoco-maven-plugin's releases.

0.8.14

New Features

  • JaCoCo now officially supports Java 25 (GitHub #1950).
  • Experimental support for Java 26 class files (GitHub #1870).
  • Branches added by the Kotlin compiler for default argument number 33 or higher are filtered out during generation of report (GitHub #1655).
  • Part of bytecode generated by the Kotlin compiler for elvis operator that follows safe call operator is filtered out during generation of report (GitHub #1814, #1954).
  • Part of bytecode generated by the Kotlin compiler for more cases of chained safe call operators is filtered out during generation of report (GitHub #1956).
  • Part of bytecode generated by the Kotlin compiler for invocations of suspendCoroutineUninterceptedOrReturn intrinsic is filtered out during generation of report (GitHub #1929).
  • Part of bytecode generated by the Kotlin compiler for suspending lambdas with parameters is filtered out during generation of report (GitHub #1945).
  • Part of bytecode generated by the Kotlin compiler for suspending functions and lambdas with suspension points that return inline value class is filtered out during generation of report (GitHub #1871).
  • Part of bytecode generated by the Kotlin Compose compiler plugin for pausable composition is filtered out during generation of report (GitHub #1911).
  • Methods generated by the Kotlin serialization compiler plugin are filtered out (GitHub #1885, #1970, #1971).

Fixed bugs

  • Fixed handling of implicit else clause of when with String subject in Kotlin (GitHub #1813, #1940).
  • Fixed handling of implicit default clause of switch by String in Java when compiled by ECJ (GitHub #1813, #1940). Fixed handling of exceptions in chains of safe call operators in Kotlin (GitHub #1819).

Non-functional Changes

  • JaCoCo now depends on ASM 9.9 (GitHub #1965).
Commits
  • 2eb2483 Prepare release v0.8.14
  • de76181 KotlinSerializableFilter should filter more methods (#1971)
  • 89c4bd5 Fix NPE in KotlinSerializableFilter (#1970)
  • 0981128 Migrate release staging to the Central Publisher Portal (#1968)
  • d07bc6b Add filter for bytecode generated by Kotlin serialization compiler plugin (#1...
  • 5e35fd5 Upgrade maven-dependency-plugin to 3.9.0 (#1966)
  • c2fe5cc Upgrade ASM to 9.9 (#1965)
  • b0f8e23 KotlinSafeCallOperatorFilter should filter "unoptimized" safe call followed b...
  • c7bd3f4 Upgrade spotless-maven-plugin to 3.0.0 (#1961)
  • faa289d KotlinSafeCallOperatorFilter should not be affected by presence of pseudo ins...
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.jacoco:jacoco-maven-plugin&package-manager=maven&previous-version=0.8.13&new-version=0.8.14)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8cfd4eceff..a695f91b1d 100644 --- a/pom.xml +++ b/pom.xml @@ -350,7 +350,7 @@ under the License. org.jacoco jacoco-maven-plugin - 0.8.13 + 0.8.14

... (truncated)

Commits
  • 2148f28 v2.11.7
  • e65fa80 #1614 investigating "duplicate" nullable annotations
  • d2ac4f1 Merge pull request #1616 from werli/fix-build-with-optional
  • 1470f17 Conditionally remove unnecessary cast for optional record wither methods
  • c56f082 #1611 #1579 advancing hacks and workarounds for type_use / nullable annotations
  • 95df0cd Custom nullable in nullableAnnotation should not use qualified notation
  • f7a662e #1610 derived arrays, nullable array cloning
  • 9001c82 #1612 false negative in test
  • eecbc5b #1612 fixing and refining no-arg constructors
  • b659a65 whatever to build
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.immutables:value&package-manager=maven&previous-version=2.10.1&new-version=2.11.7)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a695f91b1d..bd6e004424 100644 --- a/pom.xml +++ b/pom.xml @@ -312,7 +312,7 @@ under the License. org.immutables value - 2.10.1 + 2.12.1 From 30eb4cf8cae6b18c9eb934956230888fa2a804e6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 9 Jan 2026 11:40:31 +0100 Subject: [PATCH 245/408] MINOR: Bump com.google.api.grpc:proto-google-common-protos from 2.56.0 to 2.63.1 (#920) Bumps [com.google.api.grpc:proto-google-common-protos](https://github.com/googleapis/sdk-platform-java) from 2.56.0 to 2.63.1.
Release notes

Sourced from com.google.api.grpc:proto-google-common-protos's releases.

v2.63.0

2.63.0 (2025-10-16)

Features

Dependencies

  • Bump errorprone-annotations to v2.42.0 (8d6c1f9)
  • Bump guava to v33.5.0 (8d6c1f9)
  • Bump j2objc-annotations to v3.1 (8d6c1f9)
  • update google auth library dependencies to v1.40.0 (#3945) (1d74663)
  • Upgrade Google Http Java Client to v2.0.2 (#3946) (7fb4f15)

v2.62.3

2.62.3 (2025-10-02)

Bug Fixes

  • mtls: Fix EndpointContext's determineEndpoint logic to respect env var (#3912) (e5948d0)

v2.62.2

2.62.2 (2025-09-18)

Dependencies

v2.62.1

2.62.1 (2025-09-05)

Dependencies

v2.62.0

2.62.0 (2025-08-19)

... (truncated)

Changelog

Sourced from com.google.api.grpc:proto-google-common-protos's changelog.

Changelog

2.64.1 (2025-11-07)

Dependencies

2.64.0 (2025-10-31)

Features

  • [common-protos] Add Carousel widget (1e4a7e5)
  • librariangen: add generate package (#3952) (2f6c75d)
  • librariangen: generate grpc stubs and resource helpers (#3967) (452d703)

Dependencies

2.63.0 (2025-10-16)

Features

Dependencies

  • Bump errorprone-annotations to v2.42.0 (8d6c1f9)
  • Bump guava to v33.5.0 (8d6c1f9)
  • Bump j2objc-annotations to v3.1 (8d6c1f9)
  • update google auth library dependencies to v1.40.0 (#3945) (1d74663)
  • Upgrade Google Http Java Client to v2.0.2 (#3946) (7fb4f15)

2.62.3 (2025-10-02)

Bug Fixes

  • mtls: Fix EndpointContext's determineEndpoint logic to respect env var (#3912) (e5948d0)

... (truncated)

Commits
  • 4aaea1e chore(main): release 2.55.1 (#3695)
  • 2725744 deps: revert "deps: update arrow.version to v18.2.0" (#3694)
  • 3d06ab7 chore(main): release 2.55.1-SNAPSHOT (#3692)
  • a38020a chore(main): release 2.55.0 (#3669)
  • 8fd7b62 build(deps): update dependency com.google.cloud:google-cloud-shared-config to...
  • 2562a7d chore: update googleapis commit at Thu Feb 27 02:27:38 UTC 2025 (#3666)
  • 542d98d chore: add aliases to generate command options. (#3689)
  • 5192426 chore: add java 8 compatibility check (#3688)
  • 25d3101 chore: fix logback-classic version for testing (#3686)
  • 0932605 test: Reduce the LRO timeout value in Showcase tests (#3684)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.api.grpc:proto-google-common-protos&package-manager=maven&previous-version=2.56.0&new-version=2.63.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml index 24beac391e..b1f755844e 100644 --- a/flight/flight-core/pom.xml +++ b/flight/flight-core/pom.xml @@ -134,7 +134,7 @@ under the License. com.google.api.grpc proto-google-common-protos - 2.56.0 + 2.63.2 test From bd3a6ee4efbd637e814867637b6e7b7f8f09bcc8 Mon Sep 17 00:00:00 2001 From: Tamas Mate <50709850+tmater@users.noreply.github.com> Date: Fri, 9 Jan 2026 13:43:33 +0100 Subject: [PATCH 246/408] MINOR: Add private constructor to UuidType singleton (#945) Add private constructor to UuidType singleton. --- .../org/apache/arrow/c/RoundtripTest.java | 81 ++----------------- .../org/apache/arrow/vector/UuidVector.java | 4 +- .../arrow/vector/extension/UuidType.java | 2 + .../apache/arrow/vector/TestListVector.java | 10 +-- .../apache/arrow/vector/TestMapVector.java | 8 +- .../apache/arrow/vector/TestStructVector.java | 4 +- .../org/apache/arrow/vector/TestUuidType.java | 26 +++--- .../complex/impl/TestComplexCopier.java | 10 +-- .../complex/impl/TestPromotableWriter.java | 5 +- .../complex/writer/TestComplexWriter.java | 2 +- 10 files changed, 42 insertions(+), 110 deletions(-) diff --git a/c/src/test/java/org/apache/arrow/c/RoundtripTest.java b/c/src/test/java/org/apache/arrow/c/RoundtripTest.java index 010a305495..f6ff88571e 100644 --- a/c/src/test/java/org/apache/arrow/c/RoundtripTest.java +++ b/c/src/test/java/org/apache/arrow/c/RoundtripTest.java @@ -35,7 +35,6 @@ import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; -import org.apache.arrow.memory.util.hash.ArrowBufHasher; import org.apache.arrow.vector.BaseLargeVariableWidthVector; import org.apache.arrow.vector.BaseVariableWidthVector; import org.apache.arrow.vector.BigIntVector; @@ -44,7 +43,6 @@ import org.apache.arrow.vector.DateMilliVector; import org.apache.arrow.vector.DecimalVector; import org.apache.arrow.vector.DurationVector; -import org.apache.arrow.vector.ExtensionTypeVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.FixedSizeBinaryVector; import org.apache.arrow.vector.Float2Vector; @@ -74,6 +72,7 @@ import org.apache.arrow.vector.UInt2Vector; import org.apache.arrow.vector.UInt4Vector; import org.apache.arrow.vector.UInt8Vector; +import org.apache.arrow.vector.UuidVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; @@ -92,6 +91,7 @@ import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.complex.UnionVector; import org.apache.arrow.vector.complex.impl.UnionMapWriter; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.holders.IntervalDayHolder; import org.apache.arrow.vector.holders.NullableLargeVarBinaryHolder; import org.apache.arrow.vector.holders.NullableUInt4Holder; @@ -100,7 +100,6 @@ import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.ArrowType.ExtensionType; -import org.apache.arrow.vector.types.pojo.ExtensionTypeRegistry; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; @@ -810,9 +809,8 @@ public void testEmptyRunEndEncodedVector() { @Test public void testExtensionTypeVector() { - ExtensionTypeRegistry.register(new UuidType()); final Schema schema = - new Schema(Collections.singletonList(Field.nullable("a", new UuidType()))); + new Schema(Collections.singletonList(Field.nullable("a", UuidType.INSTANCE))); try (final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { // Fill with data UUID u1 = UUID.randomUUID(); @@ -830,13 +828,12 @@ public void testExtensionTypeVector() { assertEquals(root.getSchema(), importedRoot.getSchema()); final Field field = importedRoot.getSchema().getFields().get(0); - final UuidType expectedType = new UuidType(); assertEquals( field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_NAME), - expectedType.extensionName()); + UuidType.INSTANCE.extensionName()); assertEquals( field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_METADATA), - expectedType.serialize()); + UuidType.INSTANCE.serialize()); final UuidVector deserialized = (UuidVector) importedRoot.getFieldVectors().get(0); assertEquals(vector.getValueCount(), deserialized.getValueCount()); @@ -1115,72 +1112,4 @@ private VectorSchemaRoot createTestVSR() { return new VectorSchemaRoot(fields, vectors); } - - static class UuidType extends ExtensionType { - - @Override - public ArrowType storageType() { - return new ArrowType.FixedSizeBinary(16); - } - - @Override - public String extensionName() { - return "uuid"; - } - - @Override - public boolean extensionEquals(ExtensionType other) { - return other instanceof UuidType; - } - - @Override - public ArrowType deserialize(ArrowType storageType, String serializedData) { - if (!storageType.equals(storageType())) { - throw new UnsupportedOperationException( - "Cannot construct UuidType from underlying type " + storageType); - } - return new UuidType(); - } - - @Override - public String serialize() { - return ""; - } - - @Override - public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator) { - return new UuidVector(name, allocator, new FixedSizeBinaryVector(name, allocator, 16)); - } - } - - static class UuidVector extends ExtensionTypeVector { - - public UuidVector( - String name, BufferAllocator allocator, FixedSizeBinaryVector underlyingVector) { - super(name, allocator, underlyingVector); - } - - @Override - public UUID getObject(int index) { - final ByteBuffer bb = ByteBuffer.wrap(getUnderlyingVector().getObject(index)); - return new UUID(bb.getLong(), bb.getLong()); - } - - @Override - public int hashCode(int index) { - return hashCode(index, null); - } - - @Override - public int hashCode(int index, ArrowBufHasher hasher) { - return getUnderlyingVector().hashCode(index, hasher); - } - - public void set(int index, UUID uuid) { - ByteBuffer bb = ByteBuffer.allocate(16); - bb.putLong(uuid.getMostSignificantBits()); - bb.putLong(uuid.getLeastSignificantBits()); - getUnderlyingVector().set(index, bb.array()); - } - } } diff --git a/vector/src/main/java/org/apache/arrow/vector/UuidVector.java b/vector/src/main/java/org/apache/arrow/vector/UuidVector.java index c662a6e064..e0dadd1c67 100644 --- a/vector/src/main/java/org/apache/arrow/vector/UuidVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/UuidVector.java @@ -69,7 +69,7 @@ public class UuidVector extends ExtensionTypeVector public UuidVector( String name, BufferAllocator allocator, FixedSizeBinaryVector underlyingVector) { super(name, allocator, underlyingVector); - this.field = new Field(name, FieldType.nullable(new UuidType()), null); + this.field = new Field(name, FieldType.nullable(UuidType.INSTANCE), null); } /** @@ -99,7 +99,7 @@ public UuidVector( */ public UuidVector(String name, BufferAllocator allocator) { super(name, allocator, new FixedSizeBinaryVector(name, allocator, UUID_BYTE_WIDTH)); - this.field = new Field(name, FieldType.nullable(new UuidType()), null); + this.field = new Field(name, FieldType.nullable(UuidType.INSTANCE), null); } /** diff --git a/vector/src/main/java/org/apache/arrow/vector/extension/UuidType.java b/vector/src/main/java/org/apache/arrow/vector/extension/UuidType.java index f0f2636c82..cd29f930e1 100644 --- a/vector/src/main/java/org/apache/arrow/vector/extension/UuidType.java +++ b/vector/src/main/java/org/apache/arrow/vector/extension/UuidType.java @@ -63,6 +63,8 @@ public class UuidType extends ExtensionType { /** Storage type for UUID: FixedSizeBinary(16). */ public static final ArrowType STORAGE_TYPE = new ArrowType.FixedSizeBinary(UUID_BYTE_WIDTH); + private UuidType() {} + static { ExtensionTypeRegistry.register(INSTANCE); } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index 41a95a8d11..df3a609f53 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -1208,7 +1208,7 @@ public void testGetTransferPairWithField() { @Test public void testListVectorWithExtensionType() throws Exception { - final FieldType type = FieldType.nullable(new UuidType()); + final FieldType type = FieldType.nullable(UuidType.INSTANCE); try (final ListVector inVector = new ListVector("list", allocator, type, null)) { UnionListWriter writer = inVector.getWriter(); writer.allocate(); @@ -1216,7 +1216,7 @@ public void testListVectorWithExtensionType() throws Exception { UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); writer.startList(); - ExtensionWriter extensionWriter = writer.extension(new UuidType()); + ExtensionWriter extensionWriter = writer.extension(UuidType.INSTANCE); extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter.writeExtension(u1); extensionWriter.writeExtension(u2); @@ -1236,7 +1236,7 @@ public void testListVectorWithExtensionType() throws Exception { @Test public void testListVectorReaderForExtensionType() throws Exception { - final FieldType type = FieldType.nullable(new UuidType()); + final FieldType type = FieldType.nullable(UuidType.INSTANCE); try (final ListVector inVector = new ListVector("list", allocator, type, null)) { UnionListWriter writer = inVector.getWriter(); writer.allocate(); @@ -1244,7 +1244,7 @@ public void testListVectorReaderForExtensionType() throws Exception { UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); writer.startList(); - ExtensionWriter extensionWriter = writer.extension(new UuidType()); + ExtensionWriter extensionWriter = writer.extension(UuidType.INSTANCE); extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter.writeExtension(u1); extensionWriter.writeExtension(u2); @@ -1279,7 +1279,7 @@ public void testCopyFromForExtensionType() throws Exception { UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); writer.startList(); - ExtensionWriter extensionWriter = writer.extension(new UuidType()); + ExtensionWriter extensionWriter = writer.extension(UuidType.INSTANCE); extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter.writeExtension(u1); extensionWriter.writeExtension(u2); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java index df8f338f45..d9d2ca50dc 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java @@ -1284,13 +1284,13 @@ public void testMapVectorWithExtensionType() throws Exception { writer.startMap(); writer.startEntry(); writer.key().bigInt().writeBigInt(0); - ExtensionWriter extensionWriter = writer.value().extension(new UuidType()); + ExtensionWriter extensionWriter = writer.value().extension(UuidType.INSTANCE); extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter.writeExtension(u1); writer.endEntry(); writer.startEntry(); writer.key().bigInt().writeBigInt(1); - extensionWriter = writer.value().extension(new UuidType()); + extensionWriter = writer.value().extension(UuidType.INSTANCE); extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter.writeExtension(u2); writer.endEntry(); @@ -1326,13 +1326,13 @@ public void testCopyFromForExtensionType() throws Exception { writer.startMap(); writer.startEntry(); writer.key().bigInt().writeBigInt(0); - ExtensionWriter extensionWriter = writer.value().extension(new UuidType()); + ExtensionWriter extensionWriter = writer.value().extension(UuidType.INSTANCE); extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter.writeExtension(u1); writer.endEntry(); writer.startEntry(); writer.key().bigInt().writeBigInt(1); - extensionWriter = writer.value().extension(new UuidType()); + extensionWriter = writer.value().extension(UuidType.INSTANCE); extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter.writeExtension(u2); writer.endEntry(); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java b/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java index b8abfe1ef6..21ebeebc86 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java @@ -341,7 +341,7 @@ public void testGetTransferPairWithFieldAndCallBack() { @Test public void testStructVectorWithExtensionTypes() { - UuidType uuidType = new UuidType(); + UuidType uuidType = UuidType.INSTANCE; Field uuidField = new Field("struct_child", FieldType.nullable(uuidType), null); Field structField = new Field("struct", FieldType.nullable(new ArrowType.Struct()), List.of(uuidField)); @@ -353,7 +353,7 @@ public void testStructVectorWithExtensionTypes() { @Test public void testStructVectorTransferPairWithExtensionType() { - UuidType uuidType = new UuidType(); + UuidType uuidType = UuidType.INSTANCE; Field uuidField = new Field("uuid_child", FieldType.nullable(uuidType), null); Field structField = new Field("struct", FieldType.nullable(new ArrowType.Struct()), List.of(uuidField)); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestUuidType.java b/vector/src/test/java/org/apache/arrow/vector/TestUuidType.java index 9f7c65b82b..acf9dd6868 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestUuidType.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestUuidType.java @@ -75,21 +75,21 @@ void testConstants() { @Test void testStorageType() { - UuidType type = new UuidType(); + UuidType type = UuidType.INSTANCE; assertEquals(UuidType.STORAGE_TYPE, type.storageType()); assertInstanceOf(ArrowType.FixedSizeBinary.class, type.storageType()); } @Test void testExtensionName() { - UuidType type = new UuidType(); + UuidType type = UuidType.INSTANCE; assertEquals("arrow.uuid", type.extensionName()); } @Test void testExtensionEquals() { - UuidType type1 = new UuidType(); - UuidType type2 = new UuidType(); + UuidType type1 = UuidType.INSTANCE; + UuidType type2 = UuidType.INSTANCE; UuidType type3 = UuidType.INSTANCE; assertTrue(type1.extensionEquals(type2)); @@ -99,20 +99,20 @@ void testExtensionEquals() { @Test void testIsComplex() { - UuidType type = new UuidType(); + UuidType type = UuidType.INSTANCE; assertFalse(type.isComplex()); } @Test void testSerialize() { - UuidType type = new UuidType(); + UuidType type = UuidType.INSTANCE; String serialized = type.serialize(); assertEquals("", serialized); } @Test void testDeserializeValid() { - UuidType type = new UuidType(); + UuidType type = UuidType.INSTANCE; ArrowType storageType = new ArrowType.FixedSizeBinary(UuidType.UUID_BYTE_WIDTH); ArrowType deserialized = assertDoesNotThrow(() -> type.deserialize(storageType, "")); @@ -122,7 +122,7 @@ void testDeserializeValid() { @Test void testDeserializeInvalidStorageType() { - UuidType type = new UuidType(); + UuidType type = UuidType.INSTANCE; ArrowType wrongStorageType = new ArrowType.FixedSizeBinary(32); assertThrows(UnsupportedOperationException.class, () -> type.deserialize(wrongStorageType, "")); @@ -130,7 +130,7 @@ void testDeserializeInvalidStorageType() { @Test void testGetNewVector() { - UuidType type = new UuidType(); + UuidType type = UuidType.INSTANCE; try (FieldVector vector = type.getNewVector("uuid_field", FieldType.nullable(type), allocator)) { assertInstanceOf(UuidVector.class, vector); @@ -141,7 +141,7 @@ void testGetNewVector() { @Test void testVectorOperations() { - UuidType type = new UuidType(); + UuidType type = UuidType.INSTANCE; try (FieldVector vector = type.getNewVector("uuid_field", FieldType.nullable(type), allocator)) { UuidVector uuidVector = (UuidVector) vector; @@ -218,7 +218,7 @@ void testVectorIpcRoundTrip() throws IOException { @Test void testVectorByteArrayOperations() { - UuidType type = new UuidType(); + UuidType type = UuidType.INSTANCE; try (FieldVector vector = type.getNewVector("uuid_field", FieldType.nullable(type), allocator)) { UuidVector uuidVector = (UuidVector) vector; @@ -240,7 +240,7 @@ void testVectorByteArrayOperations() { @Test void testGetNewVectorWithCustomFieldType() { - UuidType type = new UuidType(); + UuidType type = UuidType.INSTANCE; FieldType fieldType = new FieldType(false, type, null); try (FieldVector vector = type.getNewVector("non_nullable_uuid", fieldType, allocator)) { @@ -262,7 +262,7 @@ void testSingleton() { @Test void testUnderlyingVector() { - UuidType type = new UuidType(); + UuidType type = UuidType.INSTANCE; try (FieldVector vector = type.getNewVector("uuid_field", FieldType.nullable(type), allocator)) { UuidVector uuidVector = (UuidVector) vector; diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java index 493a4b26ab..73c1cd3b74 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java @@ -860,7 +860,7 @@ public void testCopyListVectorWithExtensionType() { for (int i = 0; i < COUNT; i++) { listWriter.setPosition(i); listWriter.startList(); - ExtensionWriter extensionWriter = listWriter.extension(new UuidType()); + ExtensionWriter extensionWriter = listWriter.extension(UuidType.INSTANCE); extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter.writeExtension(UUID.randomUUID()); extensionWriter.writeExtension(UUID.randomUUID()); @@ -896,10 +896,10 @@ public void testCopyMapVectorWithExtensionType() { mapWriter.setPosition(i); mapWriter.startMap(); mapWriter.startEntry(); - ExtensionWriter extensionKeyWriter = mapWriter.key().extension(new UuidType()); + ExtensionWriter extensionKeyWriter = mapWriter.key().extension(UuidType.INSTANCE); extensionKeyWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionKeyWriter.writeExtension(UUID.randomUUID()); - ExtensionWriter extensionValueWriter = mapWriter.value().extension(new UuidType()); + ExtensionWriter extensionValueWriter = mapWriter.value().extension(UuidType.INSTANCE); extensionValueWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionValueWriter.writeExtension(UUID.randomUUID()); mapWriter.endEntry(); @@ -934,10 +934,10 @@ public void testCopyStructVectorWithExtensionType() { for (int i = 0; i < COUNT; i++) { structWriter.setPosition(i); structWriter.start(); - ExtensionWriter extensionWriter1 = structWriter.extension("timestamp1", new UuidType()); + ExtensionWriter extensionWriter1 = structWriter.extension("timestamp1", UuidType.INSTANCE); extensionWriter1.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter1.writeExtension(UUID.randomUUID()); - ExtensionWriter extensionWriter2 = structWriter.extension("timestamp2", new UuidType()); + ExtensionWriter extensionWriter2 = structWriter.extension("timestamp2", UuidType.INSTANCE); extensionWriter2.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter2.writeExtension(UUID.randomUUID()); structWriter.end(); diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java index a4594024fa..c71717a027 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java @@ -785,7 +785,7 @@ public void testExtensionType() throws Exception { try (final NonNullableStructVector container = NonNullableStructVector.empty(EMPTY_SCHEMA_PATH, allocator); final UuidVector v = - container.addOrGet("uuid", FieldType.nullable(new UuidType()), UuidVector.class); + container.addOrGet("uuid", FieldType.nullable(UuidType.INSTANCE), UuidVector.class); final PromotableWriter writer = new PromotableWriter(v, container)) { UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); @@ -810,7 +810,8 @@ public void testExtensionType() throws Exception { public void testExtensionTypeForList() throws Exception { try (final ListVector container = ListVector.empty(EMPTY_SCHEMA_PATH, allocator); final UuidVector v = - (UuidVector) container.addOrGetVector(FieldType.nullable(new UuidType())).getVector(); + (UuidVector) + container.addOrGetVector(FieldType.nullable(UuidType.INSTANCE)).getVector(); final PromotableWriter writer = new PromotableWriter(v, container)) { UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index 871a3cc461..3a8f3f8e6a 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -2510,7 +2510,7 @@ public void extensionWriterReader() throws Exception { StructWriter rootWriter = writer.rootAsStruct(); { - ExtensionWriter extensionWriter = rootWriter.extension("uuid1", new UuidType()); + ExtensionWriter extensionWriter = rootWriter.extension("uuid1", UuidType.INSTANCE); extensionWriter.setPosition(0); extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter.writeExtension(u1); From 7d4cf21bd6502fc650dfad4a6e9fbe0ae5cf4360 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 9 Jan 2026 15:22:30 +0100 Subject: [PATCH 247/408] MINOR: Bump io.netty:netty-bom from 4.1.119.Final to 4.2.7.Final (#887) Bumps [io.netty:netty-bom](https://github.com/netty/netty) from 4.1.119.Final to 4.2.7.Final.
Commits
  • 511cbac [maven-release-plugin] prepare release netty-4.2.7.Final
  • bf1cad6 Adjust plugin config to not publish testsuite artifacts
  • 690f56f [maven-release-plugin] rollback the release of netty-4.2.7.Final
  • 63b5232 [maven-release-plugin] prepare for next development iteration
  • 9f99dfd [maven-release-plugin] prepare release netty-4.2.7.Final
  • 551c32a Upgrade publishing plugin
  • a6660fe [maven-release-plugin] rollback the release of netty-4.2.7.Final
  • 297b7c1 [maven-release-plugin] prepare for next development iteration
  • 2c89a17 [maven-release-plugin] prepare release netty-4.2.7.Final
  • 1782e8c Merge commit from fork
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=io.netty:netty-bom&package-manager=maven&previous-version=4.1.119.Final&new-version=4.2.7.Final)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index bd6e004424..f3025e16ee 100644 --- a/pom.xml +++ b/pom.xml @@ -97,7 +97,7 @@ under the License. 5.12.2 2.0.17 33.4.8-jre - 4.1.127.Final + 4.2.9.Final 1.73.0 4.33.1 2.18.3 From 794277963b4c42cd019230ec096f354c2cb685f1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 10 Jan 2026 07:25:50 +0100 Subject: [PATCH 248/408] MINOR: Bump parquet.version from 1.15.2 to 1.16.0 (#913) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps `parquet.version` from 1.15.2 to 1.16.0. Updates `org.apache.parquet:parquet-avro` from 1.15.2 to 1.16.0
Release notes

Sourced from org.apache.parquet:parquet-avro's releases.

Apache Parquet Java 1.16.0

What's Changed

... (truncated)

Commits
  • 402c381 [maven-release-plugin] prepare release apache-parquet-1.16.0-rc2
  • 0e279ef Add comparator for UnknownLogicalType (#3292) (#3295)
  • f85f083 [maven-release-plugin] prepare for next development iteration
  • 36d1880 [maven-release-plugin] prepare release apache-parquet-1.16.0-rc1
  • 2d463ee [maven-release-plugin] prepare for next development iteration
  • 1e3d701 [maven-release-plugin] prepare release apache-parquet-1.16.0-rc0
  • 7ef2f91 bump parquet-plugins to 1.16.0 for release
  • 0d25e13 MINOR: Bump parquet-format to 2.12.0 (#3285)
  • 299b0ae MINOR: Bump thrift to 0.22.0 (#3229)
  • 36a5f9c Bump jackson.version from 2.19.0 to 2.19.2 (#3266)
  • Additional commits viewable in compare view

Updates `org.apache.parquet:parquet-hadoop` from 1.15.2 to 1.16.0
Release notes

Sourced from org.apache.parquet:parquet-hadoop's releases.

Apache Parquet Java 1.16.0

What's Changed

... (truncated)

Commits
  • 402c381 [maven-release-plugin] prepare release apache-parquet-1.16.0-rc2
  • 0e279ef Add comparator for UnknownLogicalType (#3292) (#3295)
  • f85f083 [maven-release-plugin] prepare for next development iteration
  • 36d1880 [maven-release-plugin] prepare release apache-parquet-1.16.0-rc1
  • 2d463ee [maven-release-plugin] prepare for next development iteration
  • 1e3d701 [maven-release-plugin] prepare release apache-parquet-1.16.0-rc0
  • 7ef2f91 bump parquet-plugins to 1.16.0 for release
  • 0d25e13 MINOR: Bump parquet-format to 2.12.0 (#3285)
  • 299b0ae MINOR: Bump thrift to 0.22.0 (#3229)
  • 36a5f9c Bump jackson.version from 2.19.0 to 2.19.2 (#3266)
  • Additional commits viewable in compare view

You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dataset/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataset/pom.xml b/dataset/pom.xml index 66233c3970..6bca75bdd0 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -32,7 +32,7 @@ under the License. ../../../cpp/release-build/ - 1.15.2 + 1.16.0 1.12.0 From a602e6a21e6a783dd1a23403ee7614ef6031516d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 10 Jan 2026 10:05:07 +0100 Subject: [PATCH 249/408] MINOR: Bump org.immutables:value-annotations from 2.10.1 to 2.11.7 (#917) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.immutables:value-annotations](https://github.com/immutables/immutables) from 2.10.1 to 2.11.7.
Release notes

Sourced from org.immutables:value-annotations's releases.

2.11.7

Maintenance & refinements release

Thank you for the bug reports and suggestions!

Issues

  • #1621 Version 2.11.7 tag not present on GitHub
  • #1611 Jspecify Nullable doesn't work properly with generics
  • #1612 Conflicting constructor on empty interfaces when allParameters = true, and privateNoArgConstructor = true/ protectedNoArgConstructor = true (edge case regression after #1604)
  • #1579 TYPE_USE Nullable annotation not respected in the builder for arrays (arrays/elements annotation mirrors are missing) (addressed with some source code parsing, which requires -sourcepath to be provided during compilation)

PRs

New Contributors

Full Changelog: https://github.com/immutables/immutables/compare/2.11.6...2.11.7

2.11.6

Maintenance & refinements release

Thank you for the bug reports and suggestions!

Issues

  • #1602 Avoid calling check/validation method twice when using plain public constructors (@Style(of = "new")
  • #1603 Fixed compilation error with staged builders and complex generics
  • #1604 parameterless constructor when there's no attributes, but allParameters=true or allMandatoryParameters=true

Full Changelog: https://github.com/immutables/immutables/compare/2.11.5...2.11.6

2.11.5

Maintenance & refinements release

Thank you for the bug reports and PRs!

Issues

  • #1602 @Check methods (returning void i.e. non-normalizing) now works from plain public constructors (@Style(of = "new")
  • #1583 Staged builder now works for "outside"/top-level class builders, including record builders (with *BuildStages class generated to hold stage interfaces)
  • #1598 fixed: @Data from org.immutables:datatype can be used as meta-annotation
  • #1433 additionalStrictContainerConstructor=false can be used to suppress redundant strict factory method (constructor) overload

PRs

New Contributors

... (truncated)

Commits
  • 2148f28 v2.11.7
  • e65fa80 #1614 investigating "duplicate" nullable annotations
  • d2ac4f1 Merge pull request #1616 from werli/fix-build-with-optional
  • 1470f17 Conditionally remove unnecessary cast for optional record wither methods
  • c56f082 #1611 #1579 advancing hacks and workarounds for type_use / nullable annotations
  • 95df0cd Custom nullable in nullableAnnotation should not use qualified notation
  • f7a662e #1610 derived arrays, nullable array cloning
  • 9001c82 #1612 false negative in test
  • eecbc5b #1612 fixing and refining no-arg constructors
  • b659a65 whatever to build
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.immutables:value-annotations&package-manager=maven&previous-version=2.10.1&new-version=2.11.7)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f3025e16ee..6d1181d1d3 100644 --- a/pom.xml +++ b/pom.xml @@ -181,7 +181,7 @@ under the License. org.immutables value-annotations - 2.10.1 + 2.12.1 provided From e620c4481b51b47be952918b1d7e1441b22f0b44 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jan 2026 14:13:57 +0100 Subject: [PATCH 250/408] MINOR: Bump logback.version from 1.5.21 to 1.5.24 (#962) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps `logback.version` from 1.5.21 to 1.5.24. Updates `ch.qos.logback:logback-classic` from 1.5.21 to 1.5.24
Release notes

Sourced from ch.qos.logback:logback-classic's releases.

Logback 1.5.24

2026-01-06 Release of logback version 1.5.24

• Added ExpressionPropertyCondition a PropertyCondition that can evaluate boolean expressions similar to Java. See the relevant documentation for further details.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit 62bc5fc245dd3a52f3dd45e232733f4cefb4806d associated with the tag v_1.5.24. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Logback 1.5.23

2025-12-21 Release of logback version 1.5.23

• In response to issues/959 file name collisions are detected at configuration time by analyzing the configuration file and no longer at run time. This avoids the ConcurrentModificationException reported in the issue.

• ZIP and XZ compression now use a BufferedOutputStream when writing to the compressed file. This issue was reported in issues/988.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit 0bcc3feb54a6d99caac70969ee5f8334aad1fbaf associated with the tag v_1.5.23. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Logback 1.5.22

2025-12-11 Release of logback version 1.5.22

• In order to prevent involuntary information leakage, Logback will no longer output the value of a substituted variable, if the variable name contains any of the case-insensitive strings "password", "secret" or "confidential". This problem was reported by Chintan Rohila in issues/986.

• Logback now takes the overridden toString() method of Throwable subclasses into account when printing stack traces. This issue was reported in LOGBACK-543 by Alvin Chee, with a fix provided in PR 404 by Brett Kail.

• Instead of limit-counting guard, Logback now uses a tumbling-window guard to rate limit internal error messages.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit 572379aabd2f672b49593e4020696c624541e5b0 associated with the tag v_1.5.22. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits
  • 62bc5fc prepare release 1.5.24
  • aac604d typo fix of local variable name
  • 8a6df9e ExpressionPropertyCondition constructor should be public
  • 95e588c minor changes in ExpressionPropertyCondition
  • 859f5a1 added ExpressionPropertyCondition capable of parsing logical expressions on p...
  • 348075a start work on 1.5.24-SNAPSHOT
  • 0bcc3fe prepare release 1.5.23
  • 4627dbd better to use BufferedOutputStream during ZIP and XZ compression, especially ...
  • 299f091 add collision test in presence of conditional processing
  • b446f3f In Context, remove collision map
  • Additional commits viewable in compare view

Updates `ch.qos.logback:logback-core` from 1.5.21 to 1.5.24
Release notes

Sourced from ch.qos.logback:logback-core's releases.

Logback 1.5.24

2026-01-06 Release of logback version 1.5.24

• Added ExpressionPropertyCondition a PropertyCondition that can evaluate boolean expressions similar to Java. See the relevant documentation for further details.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit 62bc5fc245dd3a52f3dd45e232733f4cefb4806d associated with the tag v_1.5.24. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Logback 1.5.23

2025-12-21 Release of logback version 1.5.23

• In response to issues/959 file name collisions are detected at configuration time by analyzing the configuration file and no longer at run time. This avoids the ConcurrentModificationException reported in the issue.

• ZIP and XZ compression now use a BufferedOutputStream when writing to the compressed file. This issue was reported in issues/988.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit 0bcc3feb54a6d99caac70969ee5f8334aad1fbaf associated with the tag v_1.5.23. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Logback 1.5.22

2025-12-11 Release of logback version 1.5.22

• In order to prevent involuntary information leakage, Logback will no longer output the value of a substituted variable, if the variable name contains any of the case-insensitive strings "password", "secret" or "confidential". This problem was reported by Chintan Rohila in issues/986.

• Logback now takes the overridden toString() method of Throwable subclasses into account when printing stack traces. This issue was reported in LOGBACK-543 by Alvin Chee, with a fix provided in PR 404 by Brett Kail.

• Instead of limit-counting guard, Logback now uses a tumbling-window guard to rate limit internal error messages.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit 572379aabd2f672b49593e4020696c624541e5b0 associated with the tag v_1.5.22. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits
  • 62bc5fc prepare release 1.5.24
  • aac604d typo fix of local variable name
  • 8a6df9e ExpressionPropertyCondition constructor should be public
  • 95e588c minor changes in ExpressionPropertyCondition
  • 859f5a1 added ExpressionPropertyCondition capable of parsing logical expressions on p...
  • 348075a start work on 1.5.24-SNAPSHOT
  • 0bcc3fe prepare release 1.5.23
  • 4627dbd better to use BufferedOutputStream during ZIP and XZ compression, especially ...
  • 299f091 add collision test in presence of conditional processing
  • b446f3f In Context, remove collision map
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6d1181d1d3..0659c43008 100644 --- a/pom.xml +++ b/pom.xml @@ -111,7 +111,7 @@ under the License. true 2.42.0 3.53.0 - 1.5.21 + 1.5.24 none -Xdoclint:none From 05292ac0d16a4735189683b43c2084dd2ee92e20 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jan 2026 14:14:29 +0100 Subject: [PATCH 251/408] MINOR: Bump org.codehaus.mojo:exec-maven-plugin from 3.5.0 to 3.6.3 (#959) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.codehaus.mojo:exec-maven-plugin](https://github.com/mojohaus/exec-maven-plugin) from 3.5.0 to 3.6.3.
Release notes

Sourced from org.codehaus.mojo:exec-maven-plugin's releases.

3.6.3

📝 Documentation updates

👻 Maintenance

📦 Dependency updates

3.6.2

🚀 New features and improvements

📦 Dependency updates

3.6.1

🐛 Bug Fixes

📦 Dependency updates

3.6.0

🚀 New features and improvements

🐛 Bug Fixes

... (truncated)

Commits
  • fe1fa8c [maven-release-plugin] prepare release 3.6.3
  • 5b3feca Bump asm.version from 9.9 to 9.9.1
  • efc7faa Bump org.apache.commons:commons-exec from 1.5.0 to 1.6.0
  • cdaf267 JUnit 5 best practices (#505)
  • f3f5997 Move ExecJavaMojoTest, ExecMojoTest to JUnit 5
  • 03b87b5 Document thread group isolation limitation in java goal (#503)
  • 7a66c3e Add support for JEP 512 for for package-private static main methods with and ...
  • a6d01ef Move to JUnit 5
  • 88d5961 [maven-release-plugin] prepare for next development iteration
  • 416fdf1 [maven-release-plugin] prepare release 3.6.2
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.codehaus.mojo:exec-maven-plugin&package-manager=maven&previous-version=3.5.0&new-version=3.6.3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0659c43008..2e734396ee 100644 --- a/pom.xml +++ b/pom.xml @@ -505,7 +505,7 @@ under the License. org.codehaus.mojo exec-maven-plugin - 3.5.0 + 3.6.3 org.codehaus.mojo From 3206fe558b21ac75e6754149a4ca5961d4d29cb7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jan 2026 14:14:51 +0100 Subject: [PATCH 252/408] MINOR: Bump org.apache.commons:commons-text from 1.13.1 to 1.15.0 (#956) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.apache.commons:commons-text](https://github.com/apache/commons-text) from 1.13.1 to 1.15.0.
Changelog

Sourced from org.apache.commons:commons-text's changelog.

Apache Commons Text 1.15.0 Release Notes

The Apache Commons Text team is pleased to announce the release of Apache Commons Text 1.15.0.

Apache Commons Text is a set of utility functions and reusable components for processing and manipulating text in a Java environment.

Release 1.15.0. This is a feature and maintenance release. Java 8 or later is required.

New features

  •  Add experimental CycloneDX VEX file
[#683](https://github.com/apache/commons-text/issues/683). Thanks to
Piotr P. Karwasz, Gary Gregory.
  • TEXT-235: Add Damerau-Levenshtein distance #687. Thanks to LorgeN, Gary Gregory.
  •  Add unit tests to increase coverage
[#719](https://github.com/apache/commons-text/issues/719). Thanks to
Michael Hausegger, Gary Gregory.

  •  Add new test for CharSequenceTranslator#with()
[#725](https://github.com/apache/commons-text/issues/725). Thanks to
Michael Hausegger, Gary Gregory.

  •  Add tests and assertions to
org.apache.commons.text.similarity to get to 100% code coverage
[#727](https://github.com/apache/commons-text/issues/727),
[#728](https://github.com/apache/commons-text/issues/728). Thanks to
Michael Hausegger.

Fixed Bugs

  •  Fix exception message typo in
XmlStringLookup.XmlStringLookup(Map, Path...). Thanks to Gary Gregory.
  • TEXT-236: Inserting at the end of a TextStringBuilder throws a StringIndexOutOfBoundsException. Thanks to Pierre Post, Sumit Bera, Alex Herbert, Gary Gregory.
  •  Fix TextStringBuilderTest.testAppendToCharBuffer() to use
proper argument type
[#724](https://github.com/apache/commons-text/issues/724). Thanks to
Michael Hausegger.

  •  Fix Apache RAT plugin console warnings. Thanks to Gary
Gregory.

  •  Fix site XML to use version 2.0.0 XML schema. Thanks to Gary
Gregory.

  •  Removed unreachable threshold verification code in
src/main/java/org/apache/commons/text/similarity
[#730](https://github.com/apache/commons-text/issues/730). Thanks to
Michael Hausegger.

  •  Enable secure processing for the XML parser in
XmlStringLookup in case the underlying JAXP implementation doesn't
[#729](https://github.com/apache/commons-text/issues/729). Thanks to 김민재
(minjas0507), Gary Gregory, Piotr Karwasz.

Changes

  •  Bump org.apache.commons:commons-parent from 85 to 93
[#704](https://github.com/apache/commons-text/issues/704),
[#723](https://github.com/apache/commons-text/issues/723),
[#726](https://github.com/apache/commons-text/issues/726). Thanks to
Gary Gregory.

  •  Bump commons.bytebuddy.version from 1.17.6 to 1.18.2
[#696](https://github.com/apache/commons-text/issues/696),
[#722](https://github.com/apache/commons-text/issues/722). Thanks to
Gary Gregory.

  •  Bump graalvm.version from 24.2.2 to 25.0.1
[#703](https://github.com/apache/commons-text/issues/703),
[#716](https://github.com/apache/commons-text/issues/716). Thanks to
Gary Gregory, Dependabot.

  •  Bump org.apache.commons:commons-lang3 from 3.18.0 to 3.20.0.
Thanks to Gary Gregory.

  •  Bump commons-io:commons-io from 2.20.0 to 2.21.0. Thanks to
Gary Gregory.

Historical list of changes: https://commons.apache.org/proper/commons-text/changes.html

For complete information on Apache Commons Text, including instructions on how to submit bug reports, patches, or suggestions for improvement, see the Apache Commons Text website:

https://commons.apache.org/proper/commons-text

Download page: https://commons.apache.org/proper/commons-text/download_text.cgi

... (truncated)

Commits
  • 04e9374 Prepare for the release candidate 1.15.0 RC1
  • 502c4c4 Prepare for the next release candidate
  • c6e17ec Use direct access
  • 58e1e12 Simplify XML FSP (#731)
  • b5052c9 Bump actions/setup-java from 5.0.0 to 5.1.0
  • 2e2d4bc Revert "Bump actions/setup-java from 5.0.0 to 5.1.0"
  • b0ddbd1 Bump actions/setup-java from 5.0.0 to 5.1.0
  • 1c2d382 Add tests with external DTD
  • ed3df4b Internal clean up
  • bb508f3 Bump actions/checkout from 6.0.0 to 6.0.1
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.commons:commons-text&package-manager=maven&previous-version=1.13.1&new-version=1.15.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql/pom.xml b/flight/flight-sql/pom.xml index 15d00e3e18..4175ff70d3 100644 --- a/flight/flight-sql/pom.xml +++ b/flight/flight-sql/pom.xml @@ -113,7 +113,7 @@ under the License. org.apache.commons commons-text - 1.13.1 + 1.15.0 test From 936a31a4e59f099fa422c0a4d6a4316941dcd841 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jan 2026 14:16:04 +0100 Subject: [PATCH 253/408] MINOR: Bump io.grpc:grpc-bom from 1.73.0 to 1.78.0 (#958) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [io.grpc:grpc-bom](https://github.com/grpc/grpc-java) from 1.73.0 to 1.78.0.
Release notes

Sourced from io.grpc:grpc-bom's releases.

V1.78.0

Bug Fixes

  • core: Fix shutdown failing accepted RPCs during channel startup (02e98a806). This fixes a race where RPCs could fail with "UNAVAILABLE: Channel shutdown invoked" even though they were created before channel.shutdown()
  • okhttp: Fix race condition overwriting MAX_CONCURRENT_STREAMS (#12548) (8d49dc1c9)
  • binder: Stop leaking this from BinderServerTransport's ctor (#12453) (89d77e062)
  • rls: Avoid missed config update from reentrancy (55ae1d054). This fixes a regression since 1.75.0 triggered by CdsLb being converted to XdsDepManager. Without this fix, a second channel to the same target may hang when starting, causing DEADLINE_EXCEEDED, and unhang when the control plane delivers an update (e.g., endpoint address update)

Improvements

  • xds: gRFC A88 - Changes to XdsClient Watcher APIs (#12446) (f385add31). We now have improved xDS error handling and this provides a clearer mechanism for the xDS server to report per-resource errors to the client, resulting in better error messages for debugging and faster detection of non-existent resources. This also improves the handling of all xDS-related data errors and the behavior of the xDS resource timer.
  • rls: Control plane channel monitor state and back off handling (#12460) (26c1c1341). Resets RLS request backoff timers when the Control plane channel state transitions to READY. Also when the backoff timer expires, instead of making a RLS request immediately, it just causes a picker update to allow making rpc again to the RLS target.
  • core: simplify DnsNameResolver.resolveAddresses() (4843256af)
  • netty: Run handshakeCompleteRunnable in success cases (283f1031f)
  • api,netty: Add custom header support for HTTP CONNECT proxy (bbc0aa369)
  • binder: Pre-factor out the guts of the BinderClientTransport handshake. (9313e87df)
  • compiler: Add RISC-V 64-bit architecture support to compiler build configuration (725ab22f3)
  • core: Release lock before closing shared resource (cb73f217e). Shared resources are internal to gRPC for sharing expensive objects across channels and servers, like threads. This reduces the chances of forming a deadlock, like seen with s2a in d50098f
  • Upgrade gson to 2.12.1 (6dab2ceab)
  • Upgrade dependencies (f36defa2d). proto-google-common-protos to 2.63.1, google-auth-library to 1.40.0, error-prone annotations to 2.44.0, guava to 33.5.0-android, opentelemetry to 1.56.0
  • compiler: Update maximum supported protobuf edition to EDITION_2024 (2f64092b8)
  • binder: Introduce server authorization strategy v2 (d9710725d). Adds support for android:isolatedProcess Services and moves all security checks to the handshake, making subsequent transactions more efficient.

New Features

  • compiler: Upgrade to C++ protobuf 33.1 (#12534) (58ae5f808).
  • util: Add gRFC A68 random subsetting LB (48a42889d). The policy uses the name random_subsetting_experimental. If it is working for you, tell us so we can gauge marking it stable. While the xDS portions haven’t yet landed, it is possible to use with xDS with JSON-style Structs as supported by gRFC A52
  • xds: Support for System Root Certs (#12499) (51611bad1). Most service mesh workloads use mTLS, as described in gRFC A29. However, there are cases where it is useful for applications to use normal TLS rather than using certificates for workload identity, such as when a mesh wants to move some workloads behind a reverse proxy. The xDS CertificateValidationContext message (see envoyproxy/envoy#34235) has a system_root_certs field. In the gRPC client, if this field is present and the ca_certificate_provider_instance field is unset, system root certificates will be used for validation. This implements gRFC A82.
  • xds: Support for GCP Authentication Filter (#12499) (51611bad1). In service mesh environments, there are cases where intermediate proxies make it impossible to rely on mTLS for end-to-end authentication. These cases can be addressed instead by the use of service account identity JWT tokens. The xDS GCP Authentication filter provides a mechanism for attaching such JWT tokens as gRPC call credentials on GCP. gRPC already supports a framework for xDS HTTP filters, as described in gRFC A39. This release supports the GCP Authentication filter under this framework as described in gRFC A83.
  • xds: Support for xDS-based authority rewriting (#12499) (51611bad1). gRPC supports getting routing configuration from an xDS server, as described in gRFCs A27 and A28. The xDS configuration can configure the client to rewrite the authority header on requests. This functionality can be useful in cases where the server is using the authority header to make decisions about how to process the request, such as when multiple hosts are handled via a reverse proxy. Note that this feature is solely about rewriting the authority header on data plane RPCs; it does not affect the authority used in the TLS handshake.
    As mentioned in gRFC A29, there are use-cases for gRPC that prohibit trusting the xDS server to control security-centric configuration. The authority rewriting feature falls under the same umbrella as mTLS configuration. As a result, the authority rewriting feature will only be enabled when the bootstrap config for the xDS server has trusted_xds_server in the server_features field.
  • xds: xDS based SNI setting and SAN validation (#12378) (0567531). When using xDS credentials make SNI for the Tls handshake to be configured via xDS, rather than use the channel authority as the SNI, and make SAN validation to be able to use the SNI sent when so instructed via xDS. Implements gRFC A101.

Documentation

  • api: Document gRFC A18 TCP_USER_TIMEOUT handling for keepalive (da7038782)
  • core: Fix AbstractClientStream Javadoc (28a6130e8)
  • examples: Document how to preserve META-INF/services in uber jars (97695d523)

Thanks to

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=io.grpc:grpc-bom&package-manager=maven&previous-version=1.73.0&new-version=1.78.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2e734396ee..a9eb5ebbdd 100644 --- a/pom.xml +++ b/pom.xml @@ -98,7 +98,7 @@ under the License. 2.0.17 33.4.8-jre 4.2.9.Final - 1.73.0 + 1.78.0 4.33.1 2.18.3 3.4.2 From 109063f7d970d717e2210fd294cef09f99d16706 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 Jan 2026 11:27:18 +0100 Subject: [PATCH 254/408] MINOR: Bump com.github.ben-manes.caffeine:caffeine from 3.2.0 to 3.2.3 (#960) Bumps [com.github.ben-manes.caffeine:caffeine](https://github.com/ben-manes/caffeine) from 3.2.0 to 3.2.3.
Release notes

Sourced from com.github.ben-manes.caffeine:caffeine's releases.

3.2.3

  • Fixed frequency tracking of weak keys to use the object's identity hash code (#1902)
  • Added support for underscores in CaffeineSpec when using numeric literals (#1890)
  • Improved the external api to no longer lock when querying for the maximum size or weighted size (#1897)
  • Added detection and recovery when a custom CompletableFuture is in an inconsistent state (quarkus#50513)

3.2.2

  • Fixed characteristics returned by Spliterators (#1883)

3.2.1

  • Fixed computeIfAbsent for an async cache's synchronous view to retry if incomplete
  • Improved CaffeineSpec when being reflectively constructed (#1839)
  • Improved the handling of negative durations with variable expiration
  • Fixed intermittent null after replacing a weak/soft value (#1820)
Commits
  • 5227a98 minor build touchups
  • cc3f37d reorganize into separate gradle test suites
  • 2299add Allow users to read the maximum size without locking (fixes #1897)
  • 6250b38 clarify policy javadoc and add corresponding test cases (fixes #1927)
  • c975fc0 upgrade error-prone static analyzer
  • d8e0a92 allow the project.version to be overridden by external builders
  • 0e46d22 detect if the user's future is inconsistent with the results
  • 1971428 use the assemble task for a full build without running the test suites
  • 782ac79 use the key reference with the frequency sketch (fixes #1902)
  • e0dd94b minor build clean up
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.github.ben-manes.caffeine:caffeine&package-manager=maven&previous-version=3.2.0&new-version=3.2.3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 965e071e72..8801ad8178 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -151,7 +151,7 @@ under the License. com.github.ben-manes.caffeine caffeine - 3.2.0 + 3.2.3 From 68451bfd55f6fff268a0e11dd4f6e4b3e2a025d2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Jan 2026 16:38:46 +0100 Subject: [PATCH 255/408] MINOR: Bump org.apache.avro:avro from 1.12.0 to 1.12.1 (#955) Bumps org.apache.avro:avro from 1.12.0 to 1.12.1. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.avro:avro&package-manager=maven&previous-version=1.12.0&new-version=1.12.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dataset/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dataset/pom.xml b/dataset/pom.xml index 6bca75bdd0..2d582268a6 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -33,7 +33,7 @@ under the License. ../../../cpp/release-build/ 1.16.0 - 1.12.0 + 1.12.1 diff --git a/pom.xml b/pom.xml index a9eb5ebbdd..d6fdfdd477 100644 --- a/pom.xml +++ b/pom.xml @@ -103,7 +103,7 @@ under the License. 2.18.3 3.4.2 25.2.10 - 1.12.0 + 1.12.1 5.17.0 2 From 9cdda52550e5d95b9868e5fda26d51465c8c258d Mon Sep 17 00:00:00 2001 From: Joana Hrotko Date: Thu, 15 Jan 2026 13:49:14 +0000 Subject: [PATCH 256/408] GH-891: Add ExtensionTypeWriterFactory to TransferPair (#892) ## What's Changed This PR simplifies extension type writer creation by moving from a factory-based pattern to a type-based pattern. Instead of passing `ExtensionTypeWriterFactory` instances through multiple API layers, extension types now provide their own writers via a new `getNewFieldWriter()` method on `ArrowType.ExtensionType`. - Added `getNewFieldWriter(ValueVector)` abstract method to `ArrowType.ExtensionType` - Removed `ExtensionTypeWriterFactory` interface and all implementations - Removed factory parameters from `ComplexCopier`, `PromotableWriter`, and `TransferPair` APIs - Updated `UnionWriter` to support extension types (previously threw `UnsupportedOperationException`) - Simplified extension type implementations (`UuidType`, `OpaqueType`) The factory pattern didn't scale well. Each new extension type required creating a separate factory class and passing it through multiple API layers. This was especially painful for external developers who had to maintain two classes per extension type and manage factory parameters everywhere. The new approach follows the same pattern as `MinorType`, where each type knows how to create its own writer. This reduces boilerplate, simplifies the API, and makes it easier to implement custom extension types outside arrow-java. ## Breaking Changes - `ExtensionTypeWriterFactory` has been removed - Extension types must now implement `getNewFieldWriter(ValueVector vector)` method - ExtensionHolders must implement `type()` which returns the `ExtensionType` for that Holder - (Writers are obtained directly from the extension type, not from a factory) ### Migration Guide - _Extension types must now implement `getNewFieldWriter(ValueVector vector)` method_ ```java public class UuidType extends ExtensionType { ... @Override public FieldWriter getNewFieldWriter(ValueVector vector) { return new UuidWriterImpl((UuidVector) vector); } ... } ``` - _ExtensionHolders must implement `type()` which returns the `ExtensionType` for that Holder_ ```java public class UuidHolder extends ExtensionHolder { ... @Override public ArrowType type() { return UuidType.INSTANCE; } ``` - How to use Extension Writers? **Before:** ```java writer.extension(UuidType.INSTANCE); writer.addExtensionTypeWriterFactory(extensionTypeWriterFactory); writer.writeExtension(value); ``` **After:** ```java writer.extension(UuidType.INSTANCE); writer.writeExtension(value); ``` - Also `copyAsValue` does not need to provide the factory anymore. Closes #891 . --- .../templates/AbstractFieldReader.java | 5 +- .../templates/AbstractFieldWriter.java | 11 +- .../src/main/codegen/templates/ArrowType.java | 6 + .../main/codegen/templates/BaseReader.java | 3 - .../main/codegen/templates/BaseWriter.java | 7 +- .../main/codegen/templates/ComplexCopier.java | 23 +--- .../main/codegen/templates/NullReader.java | 1 - .../codegen/templates/PromotableWriter.java | 14 +-- .../codegen/templates/UnionListWriter.java | 12 +- .../main/codegen/templates/UnionReader.java | 23 ++++ .../main/codegen/templates/UnionVector.java | 18 +++ .../main/codegen/templates/UnionWriter.java | 27 +++- .../apache/arrow/vector/BaseValueVector.java | 13 -- .../org/apache/arrow/vector/NullVector.java | 13 -- .../org/apache/arrow/vector/ValueVector.java | 25 ---- .../complex/AbstractContainerVector.java | 13 -- .../arrow/vector/complex/LargeListVector.java | 33 +---- .../vector/complex/LargeListViewVector.java | 15 --- .../arrow/vector/complex/ListVector.java | 33 +---- .../arrow/vector/complex/ListViewVector.java | 15 +-- .../complex/impl/AbstractBaseReader.java | 10 -- .../impl/ExtensionTypeWriterFactory.java | 38 ------ .../complex/impl/UnionExtensionWriter.java | 8 +- .../complex/impl/UnionLargeListReader.java | 4 - .../complex/impl/UuidWriterFactory.java | 45 ------- .../vector/complex/impl/UuidWriterImpl.java | 6 + .../arrow/vector/extension/OpaqueType.java | 7 ++ .../arrow/vector/extension/UuidType.java | 8 ++ .../arrow/vector/holders/ExtensionHolder.java | 4 + .../vector/holders/NullableUuidHolder.java | 7 ++ .../arrow/vector/holders/UuidHolder.java | 7 ++ .../arrow/vector/TestLargeListVector.java | 79 ++++++++++++ .../apache/arrow/vector/TestListVector.java | 89 ++++++++++++-- .../apache/arrow/vector/TestMapVector.java | 115 ++++++++++++++++-- .../apache/arrow/vector/TestStructVector.java | 10 +- .../apache/arrow/vector/TestUuidVector.java | 17 ++- .../complex/impl/TestComplexCopier.java | 23 ++-- .../complex/impl/TestPromotableWriter.java | 36 ++++-- .../complex/writer/TestComplexWriter.java | 22 +++- .../vector/types/pojo/TestExtensionType.java | 7 ++ 40 files changed, 493 insertions(+), 359 deletions(-) delete mode 100644 vector/src/main/java/org/apache/arrow/vector/complex/impl/ExtensionTypeWriterFactory.java delete mode 100644 vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java diff --git a/vector/src/main/codegen/templates/AbstractFieldReader.java b/vector/src/main/codegen/templates/AbstractFieldReader.java index c7c5b4d78d..556fb576ce 100644 --- a/vector/src/main/codegen/templates/AbstractFieldReader.java +++ b/vector/src/main/codegen/templates/AbstractFieldReader.java @@ -109,10 +109,6 @@ public void copyAsField(String name, ${name}Writer writer) { - public void copyAsValue(StructWriter writer, ExtensionTypeWriterFactory writerFactory) { - fail("CopyAsValue StructWriter"); - } - public void read(ExtensionHolder holder) { fail("Extension"); } @@ -147,4 +143,5 @@ public int size() { private void fail(String name) { throw new IllegalArgumentException(String.format("You tried to read a [%s] type when you are using a field reader of type [%s].", name, this.getClass().getSimpleName())); } + } diff --git a/vector/src/main/codegen/templates/AbstractFieldWriter.java b/vector/src/main/codegen/templates/AbstractFieldWriter.java index ae5b97faef..4b4a17d932 100644 --- a/vector/src/main/codegen/templates/AbstractFieldWriter.java +++ b/vector/src/main/codegen/templates/AbstractFieldWriter.java @@ -107,14 +107,17 @@ public void endEntry() { throw new IllegalStateException(String.format("You tried to end a map entry when you are using a ValueWriter of type %s.", this.getClass().getSimpleName())); } + @Override public void write(ExtensionHolder var1) { - this.fail("ExtensionType"); + this.fail("Cannot write ExtensionHolder"); } + @Override public void writeExtension(Object var1) { - this.fail("ExtensionType"); + this.fail("Cannot write extension object"); } - public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory var1) { - this.fail("ExtensionType"); + @Override + public void writeExtension(Object var1, ArrowType type) { + this.fail("Cannot write extension with type " + type); } <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> diff --git a/vector/src/main/codegen/templates/ArrowType.java b/vector/src/main/codegen/templates/ArrowType.java index fd35c1cd2b..b428f09155 100644 --- a/vector/src/main/codegen/templates/ArrowType.java +++ b/vector/src/main/codegen/templates/ArrowType.java @@ -27,8 +27,10 @@ import org.apache.arrow.flatbuf.Type; import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.writer.FieldWriter; import org.apache.arrow.vector.types.*; import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.ValueVector; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -331,6 +333,10 @@ public boolean equals(Object obj) { public T accept(ArrowTypeVisitor visitor) { return visitor.visit(this); } + + public FieldWriter getNewFieldWriter(ValueVector vector) { + throw new UnsupportedOperationException("WriterImpl not yet implemented."); + } } private static final int defaultDecimalBitWidth = 128; diff --git a/vector/src/main/codegen/templates/BaseReader.java b/vector/src/main/codegen/templates/BaseReader.java index 4c6f49ab9b..c52345af21 100644 --- a/vector/src/main/codegen/templates/BaseReader.java +++ b/vector/src/main/codegen/templates/BaseReader.java @@ -49,7 +49,6 @@ public interface RepeatedStructReader extends StructReader{ boolean next(); int size(); void copyAsValue(StructWriter writer); - void copyAsValue(StructWriter writer, ExtensionTypeWriterFactory writerFactory); } public interface ListReader extends BaseReader{ @@ -60,7 +59,6 @@ public interface RepeatedListReader extends ListReader{ boolean next(); int size(); void copyAsValue(ListWriter writer); - void copyAsValue(ListWriter writer, ExtensionTypeWriterFactory writerFactory); } public interface MapReader extends BaseReader{ @@ -71,7 +69,6 @@ public interface RepeatedMapReader extends MapReader{ boolean next(); int size(); void copyAsValue(MapWriter writer); - void copyAsValue(MapWriter writer, ExtensionTypeWriterFactory writerFactory); } public interface ScalarReader extends diff --git a/vector/src/main/codegen/templates/BaseWriter.java b/vector/src/main/codegen/templates/BaseWriter.java index 78da7fddc3..a4c98d7089 100644 --- a/vector/src/main/codegen/templates/BaseWriter.java +++ b/vector/src/main/codegen/templates/BaseWriter.java @@ -125,11 +125,12 @@ public interface ExtensionWriter extends BaseWriter { void writeExtension(Object value); /** - * Adds the given extension type factory. This factory allows configuring writer implementations for specific ExtensionTypeVector. + * Writes the given extension type value. * - * @param factory the extension type factory to add + * @param value the extension type value to write + * @param type of the extension */ - void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory factory); + void writeExtension(Object value, ArrowType type); } public interface ScalarWriter extends diff --git a/vector/src/main/codegen/templates/ComplexCopier.java b/vector/src/main/codegen/templates/ComplexCopier.java index 4df5478f48..6655f6c2a7 100644 --- a/vector/src/main/codegen/templates/ComplexCopier.java +++ b/vector/src/main/codegen/templates/ComplexCopier.java @@ -41,15 +41,8 @@ public class ComplexCopier { * @param input field to read from * @param output field to write to */ - public static void copy(FieldReader input, FieldWriter output) { - writeValue(input, output, null); - } - - public static void copy(FieldReader input, FieldWriter output, ExtensionTypeWriterFactory extensionTypeWriterFactory) { - writeValue(input, output, extensionTypeWriterFactory); - } + public static void copy(FieldReader reader, FieldWriter writer) { - private static void writeValue(FieldReader reader, FieldWriter writer, ExtensionTypeWriterFactory extensionTypeWriterFactory) { final MinorType mt = reader.getMinorType(); switch (mt) { @@ -65,7 +58,7 @@ private static void writeValue(FieldReader reader, FieldWriter writer, Extension FieldReader childReader = reader.reader(); FieldWriter childWriter = getListWriterForReader(childReader, writer); if (childReader.isSet()) { - writeValue(childReader, childWriter, extensionTypeWriterFactory); + copy(childReader, childWriter); } else { childWriter.writeNull(); } @@ -83,8 +76,8 @@ private static void writeValue(FieldReader reader, FieldWriter writer, Extension FieldReader structReader = reader.reader(); if (structReader.isSet()) { writer.startEntry(); - writeValue(mapReader.key(), getMapWriterForReader(mapReader.key(), writer.key()), extensionTypeWriterFactory); - writeValue(mapReader.value(), getMapWriterForReader(mapReader.value(), writer.value()), extensionTypeWriterFactory); + copy(mapReader.key(), getMapWriterForReader(mapReader.key(), writer.key())); + copy(mapReader.value(), getMapWriterForReader(mapReader.value(), writer.value())); writer.endEntry(); } else { writer.writeNull(); @@ -103,7 +96,7 @@ private static void writeValue(FieldReader reader, FieldWriter writer, Extension if (childReader.getMinorType() != Types.MinorType.NULL) { FieldWriter childWriter = getStructWriterForReader(childReader, writer, name); if (childReader.isSet()) { - writeValue(childReader, childWriter, extensionTypeWriterFactory); + copy(childReader, childWriter); } else { childWriter.writeNull(); } @@ -115,14 +108,10 @@ private static void writeValue(FieldReader reader, FieldWriter writer, Extension } break; case EXTENSIONTYPE: - if (extensionTypeWriterFactory == null) { - throw new IllegalArgumentException("Must provide ExtensionTypeWriterFactory"); - } if (reader.isSet()) { Object value = reader.readObject(); if (value != null) { - writer.addExtensionTypeWriterFactory(extensionTypeWriterFactory); - writer.writeExtension(value); + writer.writeExtension(value, reader.getField().getType()); } } else { writer.writeNull(); diff --git a/vector/src/main/codegen/templates/NullReader.java b/vector/src/main/codegen/templates/NullReader.java index 0529633478..88e6ea98ea 100644 --- a/vector/src/main/codegen/templates/NullReader.java +++ b/vector/src/main/codegen/templates/NullReader.java @@ -86,7 +86,6 @@ public void read(int arrayIndex, Nullable${name}Holder holder){ } - public void copyAsValue(StructWriter writer, ExtensionTypeWriterFactory writerFactory){} public void read(ExtensionHolder holder) { holder.isSet = 0; } diff --git a/vector/src/main/codegen/templates/PromotableWriter.java b/vector/src/main/codegen/templates/PromotableWriter.java index d22eb00b2c..11d34f72c9 100644 --- a/vector/src/main/codegen/templates/PromotableWriter.java +++ b/vector/src/main/codegen/templates/PromotableWriter.java @@ -286,7 +286,7 @@ protected void setWriter(ValueVector v) { writer = new UnionWriter((UnionVector) vector, nullableStructWriterFactory); break; case EXTENSIONTYPE: - writer = new UnionExtensionWriter((ExtensionTypeVector) vector); + writer = ((ExtensionType) vector.getField().getType()).getNewFieldWriter(vector); break; default: writer = type.getNewFieldWriter(vector); @@ -541,17 +541,13 @@ public void writeLargeVarChar(String value) { } @Override - public void writeExtension(Object value) { - getWriter(MinorType.EXTENSIONTYPE).writeExtension(value); + public void writeExtension(Object value, ArrowType arrowType) { + getWriter(MinorType.EXTENSIONTYPE, arrowType).writeExtension(value, arrowType); } @Override - public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory factory) { - getWriter(MinorType.EXTENSIONTYPE).addExtensionTypeWriterFactory(factory); - } - - public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory factory, ArrowType arrowType) { - getWriter(MinorType.EXTENSIONTYPE, arrowType).addExtensionTypeWriterFactory(factory); + public void write(ExtensionHolder holder) { + getWriter(MinorType.EXTENSIONTYPE, holder.type()).write(holder); } @Override diff --git a/vector/src/main/codegen/templates/UnionListWriter.java b/vector/src/main/codegen/templates/UnionListWriter.java index 3c41ac72b6..4b54739230 100644 --- a/vector/src/main/codegen/templates/UnionListWriter.java +++ b/vector/src/main/codegen/templates/UnionListWriter.java @@ -204,13 +204,13 @@ public MapWriter map(String name, boolean keysSorted) { @Override public ExtensionWriter extension(ArrowType arrowType) { - this.extensionType = arrowType; + extensionType = arrowType; return this; } + @Override public ExtensionWriter extension(String name, ArrowType arrowType) { - ExtensionWriter extensionWriter = writer.extension(name, arrowType); - return extensionWriter; + return writer.extension(name, arrowType); } <#if listName == "LargeList"> @@ -337,13 +337,13 @@ public void writeNull() { @Override public void writeExtension(Object value) { - writer.writeExtension(value); + writer.writeExtension(value, extensionType); writer.setPosition(writer.idx() + 1); } @Override - public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory var1) { - writer.addExtensionTypeWriterFactory(var1, extensionType); + public void writeExtension(Object value, ArrowType type) { + writeExtension(value); } public void write(ExtensionHolder var1) { diff --git a/vector/src/main/codegen/templates/UnionReader.java b/vector/src/main/codegen/templates/UnionReader.java index 96ad3e1b9b..0edae7ade0 100644 --- a/vector/src/main/codegen/templates/UnionReader.java +++ b/vector/src/main/codegen/templates/UnionReader.java @@ -79,6 +79,10 @@ public void read(int index, UnionHolder holder) { } private FieldReader getReaderForIndex(int index) { + return getReaderForIndex(index, null); + } + + private FieldReader getReaderForIndex(int index, ArrowType type) { int typeValue = data.getTypeValue(index); FieldReader reader = (FieldReader) readers[typeValue]; if (reader != null) { @@ -105,11 +109,26 @@ private FieldReader getReaderForIndex(int index) { + case EXTENSIONTYPE: + if(type == null) { + throw new RuntimeException("Cannot get Extension reader without an ArrowType"); + } + return (FieldReader) getExtension(type); default: throw new UnsupportedOperationException("Unsupported type: " + MinorType.values()[typeValue]); } } + private ExtensionReader extensionReader; + + private ExtensionReader getExtension(ArrowType type) { + if (extensionReader == null) { + extensionReader = data.getExtension(type).getReader(); + extensionReader.setPosition(idx()); + } + return extensionReader; + } + private SingleStructReaderImpl structReader; private StructReader getStruct() { @@ -240,4 +259,8 @@ public FieldReader reader() { public boolean next() { return getReaderForIndex(idx()).next(); } + + public void read(ExtensionHolder holder){ + getReaderForIndex(idx(), holder.type()).read(holder); + } } diff --git a/vector/src/main/codegen/templates/UnionVector.java b/vector/src/main/codegen/templates/UnionVector.java index 67efdf60f7..c706591966 100644 --- a/vector/src/main/codegen/templates/UnionVector.java +++ b/vector/src/main/codegen/templates/UnionVector.java @@ -379,6 +379,22 @@ public MapVector getMap(String name, ArrowType arrowType) { return mapVector; } + private ExtensionTypeVector extensionVector; + + public ExtensionTypeVector getExtension(ArrowType arrowType) { + if (extensionVector == null) { + int vectorCount = internalStruct.size(); + extensionVector = addOrGet(null, MinorType.EXTENSIONTYPE, arrowType, ExtensionTypeVector.class); + if (internalStruct.size() > vectorCount) { + extensionVector.allocateNew(); + if (callBack != null) { + callBack.doWork(); + } + } + } + return extensionVector; + } + public int getTypeValue(int index) { return typeBuffer.getByte(index * TYPE_WIDTH); } @@ -725,6 +741,8 @@ public ValueVector getVectorByType(int typeId, ArrowType arrowType) { return getListView(); case MAP: return getMap(name, arrowType); + case EXTENSIONTYPE: + return getExtension(arrowType); default: throw new UnsupportedOperationException("Cannot support type: " + MinorType.values()[typeId]); } diff --git a/vector/src/main/codegen/templates/UnionWriter.java b/vector/src/main/codegen/templates/UnionWriter.java index 272edab17c..0db699fd8c 100644 --- a/vector/src/main/codegen/templates/UnionWriter.java +++ b/vector/src/main/codegen/templates/UnionWriter.java @@ -28,6 +28,8 @@ package org.apache.arrow.vector.complex.impl; <#include "/@includes/vv_imports.ftl" /> +import java.util.HashMap; + import org.apache.arrow.vector.complex.writer.BaseWriter; import org.apache.arrow.vector.types.Types.MinorType; @@ -213,8 +215,31 @@ public MapWriter asMap(ArrowType arrowType) { return getMapWriter(arrowType); } + private java.util.Map extensionWriters = new HashMap<>(); + private ExtensionWriter getExtensionWriter(ArrowType arrowType) { - throw new UnsupportedOperationException("ExtensionTypes are not supported yet."); + ExtensionWriter w = extensionWriters.get(arrowType); + if (w == null) { + w = ((ExtensionType) arrowType).getNewFieldWriter(data.getExtension(arrowType)); + w.setPosition(idx()); + extensionWriters.put(arrowType, w); + } + return w; + } + + public void writeExtension(Object value, ArrowType type) { + data.setType(idx(), MinorType.EXTENSIONTYPE); + ExtensionWriter w = getExtensionWriter(type); + w.setPosition(idx()); + w.writeExtension(value); + } + + @Override + public void write(ExtensionHolder holder) { + data.setType(idx(), MinorType.EXTENSIONTYPE); + ExtensionWriter w = getExtensionWriter(holder.type()); + w.setPosition(idx()); + w.write(holder); } BaseWriter getWriter(MinorType minorType) { diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java index cc57cde29e..37dfa20616 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java @@ -22,7 +22,6 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.ReferenceManager; import org.apache.arrow.util.Preconditions; -import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.util.DataSizeRoundingUtil; import org.apache.arrow.vector.util.TransferPair; @@ -261,18 +260,6 @@ public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { throw new UnsupportedOperationException(); } - @Override - public void copyFrom( - int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { - throw new UnsupportedOperationException(); - } - - @Override - public void copyFromSafe( - int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { - throw new UnsupportedOperationException(); - } - /** * Transfer the validity buffer from `validityBuffer` to the target vector's `validityBuffer`. * Start at `startIndex` and copy `length` number of elements. If the starting index is 8 byte diff --git a/vector/src/main/java/org/apache/arrow/vector/NullVector.java b/vector/src/main/java/org/apache/arrow/vector/NullVector.java index 0d6dab2837..6bfe540d23 100644 --- a/vector/src/main/java/org/apache/arrow/vector/NullVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/NullVector.java @@ -27,7 +27,6 @@ import org.apache.arrow.memory.util.hash.ArrowBufHasher; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.compare.VectorVisitor; -import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.impl.NullReader; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.ipc.message.ArrowFieldNode; @@ -330,18 +329,6 @@ public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { throw new UnsupportedOperationException(); } - @Override - public void copyFrom( - int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { - throw new UnsupportedOperationException(); - } - - @Override - public void copyFromSafe( - int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { - throw new UnsupportedOperationException(); - } - @Override public String getName() { return this.getField().getName(); diff --git a/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/vector/src/main/java/org/apache/arrow/vector/ValueVector.java index e0628c2ee1..3a5058256c 100644 --- a/vector/src/main/java/org/apache/arrow/vector/ValueVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/ValueVector.java @@ -22,7 +22,6 @@ import org.apache.arrow.memory.OutOfMemoryException; import org.apache.arrow.memory.util.hash.ArrowBufHasher; import org.apache.arrow.vector.compare.VectorVisitor; -import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.Field; @@ -310,30 +309,6 @@ public interface ValueVector extends Closeable, Iterable { */ void copyFromSafe(int fromIndex, int thisIndex, ValueVector from); - /** - * Copy a cell value from a particular index in source vector to a particular position in this - * vector. - * - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - * @param writerFactory the extension type writer factory to use for copying extension type values - */ - void copyFrom( - int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory); - - /** - * Same as {@link #copyFrom(int, int, ValueVector)} except that it handles the case when the - * capacity of the vector needs to be expanded before copy. - * - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - * @param writerFactory the extension type writer factory to use for copying extension type values - */ - void copyFromSafe( - int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory); - /** * Accept a generic {@link VectorVisitor} and return the result. * diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java index 429f9884bb..a6a71cf1a4 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java @@ -21,7 +21,6 @@ import org.apache.arrow.vector.DensityAwareVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.ValueVector; -import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeList; @@ -152,18 +151,6 @@ public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { throw new UnsupportedOperationException(); } - @Override - public void copyFrom( - int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { - throw new UnsupportedOperationException(); - } - - @Override - public void copyFromSafe( - int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { - throw new UnsupportedOperationException(); - } - @Override public String getName() { return name; diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java index 48c8127e23..997b5a8b78 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java @@ -49,7 +49,6 @@ import org.apache.arrow.vector.ZeroVector; import org.apache.arrow.vector.compare.VectorVisitor; import org.apache.arrow.vector.complex.impl.ComplexCopier; -import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.impl.UnionLargeListReader; import org.apache.arrow.vector.complex.impl.UnionLargeListWriter; import org.apache.arrow.vector.complex.reader.FieldReader; @@ -483,42 +482,12 @@ public void copyFromSafe(int inIndex, int outIndex, ValueVector from) { */ @Override public void copyFrom(int inIndex, int outIndex, ValueVector from) { - copyFrom(inIndex, outIndex, from, null); - } - - /** - * Copy a cell value from a particular index in source vector to a particular position in this - * vector. - * - * @param inIndex position to copy from in source vector - * @param outIndex position to copy to in this vector - * @param from source vector - * @param writerFactory the extension type writer factory to use for copying extension type values - */ - @Override - public void copyFrom( - int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { Preconditions.checkArgument(this.getMinorType() == from.getMinorType()); FieldReader in = from.getReader(); in.setPosition(inIndex); UnionLargeListWriter out = getWriter(); out.setPosition(outIndex); - ComplexCopier.copy(in, out, writerFactory); - } - - /** - * Same as {@link #copyFrom(int, int, ValueVector)} except that it handles the case when the - * capacity of the vector needs to be expanded before copy. - * - * @param inIndex position to copy from in source vector - * @param outIndex position to copy to in this vector - * @param from source vector - * @param writerFactory the extension type writer factory to use for copying extension type values - */ - @Override - public void copyFromSafe( - int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { - copyFrom(inIndex, outIndex, from, writerFactory); + ComplexCopier.copy(in, out); } /** diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java index 992a664449..2da7eb057e 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java @@ -41,7 +41,6 @@ import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.ZeroVector; import org.apache.arrow.vector.compare.VectorVisitor; -import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.impl.UnionLargeListViewReader; import org.apache.arrow.vector.complex.impl.UnionLargeListViewWriter; import org.apache.arrow.vector.complex.impl.UnionListReader; @@ -347,20 +346,6 @@ public void copyFrom(int inIndex, int outIndex, ValueVector from) { "LargeListViewVector does not support copyFrom operation yet."); } - @Override - public void copyFromSafe( - int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { - throw new UnsupportedOperationException( - "LargeListViewVector does not support copyFromSafe operation yet."); - } - - @Override - public void copyFrom( - int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { - throw new UnsupportedOperationException( - "LargeListViewVector does not support copyFrom operation yet."); - } - @Override public FieldVector getDataVector() { return vector; diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 89549257c4..93a313ef4f 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -42,7 +42,6 @@ import org.apache.arrow.vector.ZeroVector; import org.apache.arrow.vector.compare.VectorVisitor; import org.apache.arrow.vector.complex.impl.ComplexCopier; -import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.impl.UnionListReader; import org.apache.arrow.vector.complex.impl.UnionListWriter; import org.apache.arrow.vector.complex.reader.FieldReader; @@ -401,42 +400,12 @@ public void copyFromSafe(int inIndex, int outIndex, ValueVector from) { */ @Override public void copyFrom(int inIndex, int outIndex, ValueVector from) { - copyFrom(inIndex, outIndex, from, null); - } - - /** - * Same as {@link #copyFrom(int, int, ValueVector)} except that it handles the case when the - * capacity of the vector needs to be expanded before copy. - * - * @param inIndex position to copy from in source vector - * @param outIndex position to copy to in this vector - * @param from source vector - * @param writerFactory the extension type writer factory to use for copying extension type values - */ - @Override - public void copyFromSafe( - int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { - copyFrom(inIndex, outIndex, from, writerFactory); - } - - /** - * Copy a cell value from a particular index in source vector to a particular position in this - * vector. - * - * @param inIndex position to copy from in source vector - * @param outIndex position to copy to in this vector - * @param from source vector - * @param writerFactory the extension type writer factory to use for copying extension type values - */ - @Override - public void copyFrom( - int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { Preconditions.checkArgument(this.getMinorType() == from.getMinorType()); FieldReader in = from.getReader(); in.setPosition(inIndex); FieldWriter out = getWriter(); out.setPosition(outIndex); - ComplexCopier.copy(in, out, writerFactory); + ComplexCopier.copy(in, out); } /** diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java index 2784240429..8711db5e0f 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java @@ -42,7 +42,6 @@ import org.apache.arrow.vector.ZeroVector; import org.apache.arrow.vector.compare.VectorVisitor; import org.apache.arrow.vector.complex.impl.ComplexCopier; -import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.impl.UnionListViewReader; import org.apache.arrow.vector.complex.impl.UnionListViewWriter; import org.apache.arrow.vector.complex.reader.FieldReader; @@ -339,12 +338,6 @@ public void copyFromSafe(int inIndex, int outIndex, ValueVector from) { copyFrom(inIndex, outIndex, from); } - @Override - public void copyFromSafe( - int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { - copyFrom(inIndex, outIndex, from, writerFactory); - } - @Override public OUT accept(VectorVisitor visitor, IN value) { return visitor.visit(this, value); @@ -352,18 +345,12 @@ public OUT accept(VectorVisitor visitor, IN value) { @Override public void copyFrom(int inIndex, int outIndex, ValueVector from) { - copyFrom(inIndex, outIndex, from, null); - } - - @Override - public void copyFrom( - int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { Preconditions.checkArgument(this.getMinorType() == from.getMinorType()); FieldReader in = from.getReader(); in.setPosition(inIndex); FieldWriter out = getWriter(); out.setPosition(outIndex); - ComplexCopier.copy(in, out, writerFactory); + ComplexCopier.copy(in, out); } @Override diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java index bf074ecb90..b2e95663f7 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java @@ -115,14 +115,4 @@ public void copyAsValue(ListWriter writer) { public void copyAsValue(MapWriter writer) { ComplexCopier.copy(this, (FieldWriter) writer); } - - @Override - public void copyAsValue(ListWriter writer, ExtensionTypeWriterFactory writerFactory) { - ComplexCopier.copy(this, (FieldWriter) writer, writerFactory); - } - - @Override - public void copyAsValue(MapWriter writer, ExtensionTypeWriterFactory writerFactory) { - ComplexCopier.copy(this, (FieldWriter) writer, writerFactory); - } } diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/ExtensionTypeWriterFactory.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/ExtensionTypeWriterFactory.java deleted file mode 100644 index a01d591555..0000000000 --- a/vector/src/main/java/org/apache/arrow/vector/complex/impl/ExtensionTypeWriterFactory.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.arrow.vector.complex.impl; - -import org.apache.arrow.vector.ExtensionTypeVector; -import org.apache.arrow.vector.complex.writer.FieldWriter; - -/** - * A factory interface for creating instances of {@link AbstractExtensionTypeWriter}. This factory - * allows configuring writer implementations for specific {@link ExtensionTypeVector}. - * - * @param the type of writer implementation for a specific {@link ExtensionTypeVector}. - */ -public interface ExtensionTypeWriterFactory { - - /** - * Returns an instance of the writer implementation for the given {@link ExtensionTypeVector}. - * - * @param vector the {@link ExtensionTypeVector} for which the writer implementation is to be - * returned. - * @return an instance of the writer implementation for the given {@link ExtensionTypeVector}. - */ - T getWriterImpl(ExtensionTypeVector vector); -} diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionExtensionWriter.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionExtensionWriter.java index 4219069cba..93796aa77e 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionExtensionWriter.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionExtensionWriter.java @@ -60,11 +60,6 @@ public void writeExtension(Object var1) { } @Override - public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory factory) { - this.writer = factory.getWriterImpl(vector); - this.writer.setPosition(idx()); - } - public void write(ExtensionHolder holder) { this.writer.write(holder); } @@ -79,6 +74,7 @@ public void setPosition(int index) { @Override public void writeNull() { - this.writer.writeNull(); + this.vector.setNull(getPosition()); + this.vector.setValueCount(getPosition() + 1); } } diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionLargeListReader.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionLargeListReader.java index a9104cb0d2..be236c3166 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionLargeListReader.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionLargeListReader.java @@ -105,8 +105,4 @@ public boolean next() { public void copyAsValue(UnionLargeListWriter writer) { ComplexCopier.copy(this, (FieldWriter) writer); } - - public void copyAsValue(UnionLargeListWriter writer, ExtensionTypeWriterFactory writerFactory) { - ComplexCopier.copy(this, (FieldWriter) writer, writerFactory); - } } diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java deleted file mode 100644 index 35988129cb..0000000000 --- a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.arrow.vector.complex.impl; - -import org.apache.arrow.vector.ExtensionTypeVector; -import org.apache.arrow.vector.UuidVector; - -/** - * Factory for creating {@link UuidWriterImpl} instances. - * - *

This factory is used to create writers for UUID extension type vectors. - * - * @see UuidWriterImpl - * @see org.apache.arrow.vector.extension.UuidType - */ -public class UuidWriterFactory implements ExtensionTypeWriterFactory { - - /** - * Creates a writer implementation for the given extension type vector. - * - * @param extensionTypeVector the vector to create a writer for - * @return a {@link UuidWriterImpl} if the vector is a {@link UuidVector}, null otherwise - */ - @Override - public AbstractFieldWriter getWriterImpl(ExtensionTypeVector extensionTypeVector) { - if (extensionTypeVector instanceof UuidVector) { - return new UuidWriterImpl((UuidVector) extensionTypeVector); - } - return null; - } -} diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java index 8a78add11c..ee3c79d5e3 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java @@ -21,6 +21,7 @@ import org.apache.arrow.vector.holders.ExtensionHolder; import org.apache.arrow.vector.holders.NullableUuidHolder; import org.apache.arrow.vector.holders.UuidHolder; +import org.apache.arrow.vector.types.pojo.ArrowType; /** * Writer implementation for {@link UuidVector}. @@ -56,6 +57,11 @@ public void writeExtension(Object value) { vector.setValueCount(getPosition() + 1); } + @Override + public void writeExtension(Object value, ArrowType type) { + writeExtension(value); + } + @Override public void write(ExtensionHolder holder) { if (holder instanceof UuidHolder) { diff --git a/vector/src/main/java/org/apache/arrow/vector/extension/OpaqueType.java b/vector/src/main/java/org/apache/arrow/vector/extension/OpaqueType.java index ca56214fda..780a4ee659 100644 --- a/vector/src/main/java/org/apache/arrow/vector/extension/OpaqueType.java +++ b/vector/src/main/java/org/apache/arrow/vector/extension/OpaqueType.java @@ -54,10 +54,12 @@ import org.apache.arrow.vector.TimeStampNanoVector; import org.apache.arrow.vector.TimeStampSecTZVector; import org.apache.arrow.vector.TimeStampSecVector; +import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.ViewVarBinaryVector; import org.apache.arrow.vector.ViewVarCharVector; +import org.apache.arrow.vector.complex.writer.FieldWriter; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.ExtensionTypeRegistry; @@ -177,6 +179,11 @@ public int hashCode() { return Objects.hash(super.hashCode(), storageType, typeName, vendorName); } + @Override + public FieldWriter getNewFieldWriter(ValueVector vector) { + throw new UnsupportedOperationException("WriterImpl not yet implemented."); + } + @Override public String toString() { return "OpaqueType(" diff --git a/vector/src/main/java/org/apache/arrow/vector/extension/UuidType.java b/vector/src/main/java/org/apache/arrow/vector/extension/UuidType.java index cd29f930e1..c249c6eda9 100644 --- a/vector/src/main/java/org/apache/arrow/vector/extension/UuidType.java +++ b/vector/src/main/java/org/apache/arrow/vector/extension/UuidType.java @@ -20,6 +20,9 @@ import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.FixedSizeBinaryVector; import org.apache.arrow.vector.UuidVector; +import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.complex.impl.UuidWriterImpl; +import org.apache.arrow.vector.complex.writer.FieldWriter; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.ArrowType.ExtensionType; import org.apache.arrow.vector.types.pojo.ExtensionTypeRegistry; @@ -108,4 +111,9 @@ public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocato return new UuidVector( name, fieldType, allocator, new FixedSizeBinaryVector(name, allocator, UUID_BYTE_WIDTH)); } + + @Override + public FieldWriter getNewFieldWriter(ValueVector vector) { + return new UuidWriterImpl((UuidVector) vector); + } } diff --git a/vector/src/main/java/org/apache/arrow/vector/holders/ExtensionHolder.java b/vector/src/main/java/org/apache/arrow/vector/holders/ExtensionHolder.java index fc7ed85878..4d3f767aef 100644 --- a/vector/src/main/java/org/apache/arrow/vector/holders/ExtensionHolder.java +++ b/vector/src/main/java/org/apache/arrow/vector/holders/ExtensionHolder.java @@ -16,7 +16,11 @@ */ package org.apache.arrow.vector.holders; +import org.apache.arrow.vector.types.pojo.ArrowType; + /** Base {@link ValueHolder} class for a {@link org.apache.arrow.vector.ExtensionTypeVector}. */ public abstract class ExtensionHolder implements ValueHolder { public int isSet; + + public abstract ArrowType type(); } diff --git a/vector/src/main/java/org/apache/arrow/vector/holders/NullableUuidHolder.java b/vector/src/main/java/org/apache/arrow/vector/holders/NullableUuidHolder.java index e5398d82cf..7fa50ca761 100644 --- a/vector/src/main/java/org/apache/arrow/vector/holders/NullableUuidHolder.java +++ b/vector/src/main/java/org/apache/arrow/vector/holders/NullableUuidHolder.java @@ -17,6 +17,8 @@ package org.apache.arrow.vector.holders; import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.vector.extension.UuidType; +import org.apache.arrow.vector.types.pojo.ArrowType; /** * Value holder for nullable UUID values. @@ -32,4 +34,9 @@ public class NullableUuidHolder extends ExtensionHolder { /** Buffer containing 16-byte UUID data. */ public ArrowBuf buffer; + + @Override + public ArrowType type() { + return UuidType.INSTANCE; + } } diff --git a/vector/src/main/java/org/apache/arrow/vector/holders/UuidHolder.java b/vector/src/main/java/org/apache/arrow/vector/holders/UuidHolder.java index 484e05c24b..8a0a66e435 100644 --- a/vector/src/main/java/org/apache/arrow/vector/holders/UuidHolder.java +++ b/vector/src/main/java/org/apache/arrow/vector/holders/UuidHolder.java @@ -17,6 +17,8 @@ package org.apache.arrow.vector.holders; import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.vector.extension.UuidType; +import org.apache.arrow.vector.types.pojo.ArrowType; /** * Value holder for non-nullable UUID values. @@ -35,4 +37,9 @@ public class UuidHolder extends ExtensionHolder { public UuidHolder() { this.isSet = 1; } + + @Override + public ArrowType type() { + return UuidType.INSTANCE; + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java index d5cbf925b2..759c84651d 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java @@ -26,18 +26,24 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.UUID; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.complex.BaseRepeatedValueVector; import org.apache.arrow.vector.complex.LargeListVector; import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.impl.UnionLargeListReader; import org.apache.arrow.vector.complex.impl.UnionLargeListWriter; import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.complex.writer.BaseWriter.ExtensionWriter; +import org.apache.arrow.vector.extension.UuidType; +import org.apache.arrow.vector.holders.UuidHolder; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.TransferPair; +import org.apache.arrow.vector.util.UuidUtility; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -1021,6 +1027,79 @@ public void testGetTransferPairWithField() throws Exception { } } + @Test + public void testCopyValueSafeForExtensionType() throws Exception { + try (LargeListVector inVector = LargeListVector.empty("input", allocator); + LargeListVector outVector = LargeListVector.empty("output", allocator)) { + UnionLargeListWriter writer = inVector.getWriter(); + writer.allocate(); + + // Create first list with UUIDs + writer.setPosition(0); + UUID u1 = UUID.randomUUID(); + UUID u2 = UUID.randomUUID(); + writer.startList(); + ExtensionWriter extensionWriter = writer.extension(UuidType.INSTANCE); + extensionWriter.writeExtension(u1); + extensionWriter.writeExtension(u2); + writer.endList(); + + // Create second list with UUIDs + writer.setPosition(1); + UUID u3 = UUID.randomUUID(); + UUID u4 = UUID.randomUUID(); + writer.startList(); + extensionWriter = writer.extension(UuidType.INSTANCE); + extensionWriter.writeExtension(u3); + extensionWriter.writeExtension(u4); + extensionWriter.writeNull(); + + writer.endList(); + writer.setValueCount(2); + + // Use copyFromSafe with ExtensionTypeWriterFactory + // This internally calls TransferImpl.copyValueSafe with ExtensionTypeWriterFactory + outVector.allocateNew(); + TransferPair tp = inVector.makeTransferPair(outVector); + tp.copyValueSafe(0, 0); + tp.copyValueSafe(1, 1); + outVector.setValueCount(2); + + // Verify first list + UnionLargeListReader reader = outVector.getReader(); + reader.setPosition(0); + assertTrue(reader.isSet(), "first list shouldn't be null"); + reader.next(); + FieldReader uuidReader = reader.reader(); + UuidHolder holder = new UuidHolder(); + uuidReader.read(holder); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + assertEquals(u1, actualUuid); + reader.next(); + uuidReader = reader.reader(); + uuidReader.read(holder); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + assertEquals(u2, actualUuid); + + // Verify second list + reader.setPosition(1); + assertTrue(reader.isSet(), "second list shouldn't be null"); + reader.next(); + uuidReader = reader.reader(); + uuidReader.read(holder); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + assertEquals(u3, actualUuid); + reader.next(); + uuidReader = reader.reader(); + uuidReader.read(holder); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + assertEquals(u4, actualUuid); + reader.next(); + uuidReader = reader.reader(); + assertFalse(uuidReader.isSet(), "third element should be null"); + } + } + private void writeIntValues(UnionLargeListWriter writer, int[] values) { writer.startList(); for (int v : values) { diff --git a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index df3a609f53..e96ac3027c 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -35,7 +35,6 @@ import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.impl.UnionListReader; import org.apache.arrow.vector.complex.impl.UnionListWriter; -import org.apache.arrow.vector.complex.impl.UuidWriterFactory; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.complex.writer.BaseWriter.ExtensionWriter; import org.apache.arrow.vector.extension.UuidType; @@ -1217,7 +1216,6 @@ public void testListVectorWithExtensionType() throws Exception { UUID u2 = UUID.randomUUID(); writer.startList(); ExtensionWriter extensionWriter = writer.extension(UuidType.INSTANCE); - extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter.writeExtension(u1); extensionWriter.writeExtension(u2); writer.endList(); @@ -1245,7 +1243,6 @@ public void testListVectorReaderForExtensionType() throws Exception { UUID u2 = UUID.randomUUID(); writer.startList(); ExtensionWriter extensionWriter = writer.extension(UuidType.INSTANCE); - extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter.writeExtension(u1); extensionWriter.writeExtension(u2); writer.endList(); @@ -1279,23 +1276,78 @@ public void testCopyFromForExtensionType() throws Exception { UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); writer.startList(); + + writer.extension(UuidType.INSTANCE).writeExtension(u1); + writer.writeExtension(u2); + writer.writeNull(); + writer.endList(); + + writer.setValueCount(3); + + // copy values from input to output + outVector.allocateNew(); + outVector.copyFrom(0, 0, inVector); + outVector.setValueCount(3); + + UnionListReader reader = outVector.getReader(); + assertTrue(reader.isSet(), "shouldn't be null"); + reader.setPosition(0); + reader.next(); + FieldReader uuidReader = reader.reader(); + UuidHolder holder = new UuidHolder(); + uuidReader.read(holder); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + assertEquals(u1, actualUuid); + reader.next(); + uuidReader = reader.reader(); + uuidReader.read(holder); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + assertEquals(u2, actualUuid); + } + } + + @Test + public void testCopyValueSafeForExtensionType() throws Exception { + try (ListVector inVector = ListVector.empty("input", allocator); + ListVector outVector = ListVector.empty("output", allocator)) { + UnionListWriter writer = inVector.getWriter(); + writer.allocate(); + + // Create first list with UUIDs + writer.setPosition(0); + UUID u1 = UUID.randomUUID(); + UUID u2 = UUID.randomUUID(); + writer.startList(); ExtensionWriter extensionWriter = writer.extension(UuidType.INSTANCE); - extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter.writeExtension(u1); extensionWriter.writeExtension(u2); - extensionWriter.writeNull(); writer.endList(); - writer.setValueCount(1); + // Create second list with UUIDs + writer.setPosition(1); + UUID u3 = UUID.randomUUID(); + UUID u4 = UUID.randomUUID(); + writer.startList(); + extensionWriter = writer.extension(UuidType.INSTANCE); + extensionWriter.writeExtension(u3); + extensionWriter.writeExtension(u4); + extensionWriter.writeNull(); - // copy values from input to output + writer.endList(); + writer.setValueCount(2); + + // Use TransferPair with ExtensionTypeWriterFactory + // This tests the new makeTransferPair API with writerFactory parameter outVector.allocateNew(); - outVector.copyFrom(0, 0, inVector, new UuidWriterFactory()); - outVector.setValueCount(1); + TransferPair transferPair = inVector.makeTransferPair(outVector); + transferPair.copyValueSafe(0, 0); + transferPair.copyValueSafe(1, 1); + outVector.setValueCount(2); + // Verify first list UnionListReader reader = outVector.getReader(); - assertTrue(reader.isSet(), "shouldn't be null"); reader.setPosition(0); + assertTrue(reader.isSet(), "first list shouldn't be null"); reader.next(); FieldReader uuidReader = reader.reader(); UuidHolder holder = new UuidHolder(); @@ -1307,6 +1359,23 @@ public void testCopyFromForExtensionType() throws Exception { uuidReader.read(holder); actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); assertEquals(u2, actualUuid); + + // Verify second list + reader.setPosition(1); + assertTrue(reader.isSet(), "second list shouldn't be null"); + reader.next(); + uuidReader = reader.reader(); + uuidReader.read(holder); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + assertEquals(u3, actualUuid); + reader.next(); + uuidReader = reader.reader(); + uuidReader.read(holder); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + assertEquals(u4, actualUuid); + reader.next(); + uuidReader = reader.reader(); + assertFalse(uuidReader.isSet(), "third element should be null"); } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java index d9d2ca50dc..bfac1237a4 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java @@ -35,7 +35,6 @@ import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.complex.impl.UnionMapReader; import org.apache.arrow.vector.complex.impl.UnionMapWriter; -import org.apache.arrow.vector.complex.impl.UuidWriterFactory; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.complex.writer.BaseWriter.ExtensionWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter; @@ -1285,14 +1284,12 @@ public void testMapVectorWithExtensionType() throws Exception { writer.startEntry(); writer.key().bigInt().writeBigInt(0); ExtensionWriter extensionWriter = writer.value().extension(UuidType.INSTANCE); - extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); - extensionWriter.writeExtension(u1); + extensionWriter.writeExtension(u1, UuidType.INSTANCE); writer.endEntry(); writer.startEntry(); writer.key().bigInt().writeBigInt(1); extensionWriter = writer.value().extension(UuidType.INSTANCE); - extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); - extensionWriter.writeExtension(u2); + extensionWriter.writeExtension(u2, UuidType.INSTANCE); writer.endEntry(); writer.endMap(); @@ -1327,20 +1324,17 @@ public void testCopyFromForExtensionType() throws Exception { writer.startEntry(); writer.key().bigInt().writeBigInt(0); ExtensionWriter extensionWriter = writer.value().extension(UuidType.INSTANCE); - extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); - extensionWriter.writeExtension(u1); + extensionWriter.writeExtension(u1, UuidType.INSTANCE); writer.endEntry(); writer.startEntry(); writer.key().bigInt().writeBigInt(1); - extensionWriter = writer.value().extension(UuidType.INSTANCE); - extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); - extensionWriter.writeExtension(u2); + extensionWriter.writeExtension(u2, UuidType.INSTANCE); writer.endEntry(); writer.endMap(); writer.setValueCount(1); outVector.allocateNew(); - outVector.copyFrom(0, 0, inVector, new UuidWriterFactory()); + outVector.copyFrom(0, 0, inVector); outVector.setValueCount(1); UnionMapReader mapReader = outVector.getReader(); @@ -1576,4 +1570,103 @@ public void testFixedSizeBinaryFirstInitialization() { assertArrayEquals(new byte[] {32, 21}, (byte[]) resultStruct.get(MapVector.VALUE_NAME)); } } + + @Test + public void testMapWithUuidKeyAndListUuidValue() throws Exception { + try (final MapVector mapVector = MapVector.empty("map", allocator, false)) { + mapVector.allocateNew(); + UnionMapWriter writer = mapVector.getWriter(); + + // Create test UUIDs + UUID key1 = UUID.randomUUID(); + UUID key2 = UUID.randomUUID(); + UUID value1a = UUID.randomUUID(); + UUID value1b = UUID.randomUUID(); + UUID value2a = UUID.randomUUID(); + UUID value2b = UUID.randomUUID(); + UUID value2c = UUID.randomUUID(); + + // Write first map entry: {key1 -> [value1a, value1b]} + writer.setPosition(0); + writer.startMap(); + + writer.startEntry(); + ExtensionWriter keyWriter = writer.key().extension(UuidType.INSTANCE); + keyWriter.writeExtension(key1, UuidType.INSTANCE); + ListWriter valueWriter = writer.value().list(); + valueWriter.startList(); + ExtensionWriter listItemWriter = valueWriter.extension(UuidType.INSTANCE); + listItemWriter.writeExtension(value1a, UuidType.INSTANCE); + listItemWriter = valueWriter.extension(UuidType.INSTANCE); + listItemWriter.writeExtension(value1b, UuidType.INSTANCE); + valueWriter.endList(); + writer.endEntry(); + + writer.startEntry(); + keyWriter = writer.key().extension(UuidType.INSTANCE); + keyWriter.writeExtension(key2, UuidType.INSTANCE); + valueWriter = writer.value().list(); + valueWriter.startList(); + listItemWriter = valueWriter.extension(UuidType.INSTANCE); + listItemWriter.writeExtension(value2a, UuidType.INSTANCE); + listItemWriter = valueWriter.extension(UuidType.INSTANCE); + listItemWriter.writeExtension(value2b, UuidType.INSTANCE); + listItemWriter = valueWriter.extension(UuidType.INSTANCE); + listItemWriter.writeExtension(value2c, UuidType.INSTANCE); + valueWriter.endList(); + writer.endEntry(); + + writer.endMap(); + writer.setValueCount(1); + + // Read and verify the data + UnionMapReader mapReader = mapVector.getReader(); + mapReader.setPosition(0); + + // Read first entry + mapReader.next(); + FieldReader keyReader = mapReader.key(); + UuidHolder keyHolder = new UuidHolder(); + keyReader.read(keyHolder); + UUID actualKey = UuidUtility.uuidFromArrowBuf(keyHolder.buffer, 0); + assertEquals(key1, actualKey); + + FieldReader valueReader = mapReader.value(); + assertTrue(valueReader.isSet()); + List listValue = (List) valueReader.readObject(); + assertEquals(2, listValue.size()); + + // Verify first list item - readObject() returns UUID objects for extension types + UUID actualValue1a = (UUID) listValue.get(0); + assertEquals(value1a, actualValue1a); + + // Verify second list item + UUID actualValue1b = (UUID) listValue.get(1); + assertEquals(value1b, actualValue1b); + + // Read second entry + mapReader.next(); + keyReader = mapReader.key(); + keyReader.read(keyHolder); + actualKey = UuidUtility.uuidFromArrowBuf(keyHolder.buffer, 0); + assertEquals(key2, actualKey); + + valueReader = mapReader.value(); + assertTrue(valueReader.isSet()); + listValue = (List) valueReader.readObject(); + assertEquals(3, listValue.size()); + + // Verify first list item - readObject() returns UUID objects for extension types + UUID actualValue2a = (UUID) listValue.get(0); + assertEquals(value2a, actualValue2a); + + // Verify second list item + UUID actualValue2b = (UUID) listValue.get(1); + assertEquals(value2b, actualValue2b); + + // Verify third list item + UUID actualValue2c = (UUID) listValue.get(2); + assertEquals(value2c, actualValue2c); + } + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java b/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java index 21ebeebc86..8c8a45f588 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java @@ -160,17 +160,23 @@ public void testGetPrimitiveVectors() { UnionVector unionVector = vector.addOrGetUnion("union"); unionVector.addVector(new BigIntVector("bigInt", allocator)); unionVector.addVector(new SmallIntVector("smallInt", allocator)); + unionVector.addVector(new UuidVector("uuid", allocator)); // add varchar vector vector.addOrGet( "varchar", FieldType.nullable(MinorType.VARCHAR.getType()), VarCharVector.class); + // add extension vector + vector.addOrGet("extension", FieldType.nullable(UuidType.INSTANCE), UuidVector.class); + List primitiveVectors = vector.getPrimitiveVectors(); - assertEquals(4, primitiveVectors.size()); + assertEquals(6, primitiveVectors.size()); assertEquals(MinorType.INT, primitiveVectors.get(0).getMinorType()); assertEquals(MinorType.BIGINT, primitiveVectors.get(1).getMinorType()); assertEquals(MinorType.SMALLINT, primitiveVectors.get(2).getMinorType()); - assertEquals(MinorType.VARCHAR, primitiveVectors.get(3).getMinorType()); + assertEquals(MinorType.EXTENSIONTYPE, primitiveVectors.get(3).getMinorType()); + assertEquals(MinorType.VARCHAR, primitiveVectors.get(4).getMinorType()); + assertEquals(MinorType.EXTENSIONTYPE, primitiveVectors.get(5).getMinorType()); } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestUuidVector.java b/vector/src/test/java/org/apache/arrow/vector/TestUuidVector.java index 3d70238ece..a3690461cf 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestUuidVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestUuidVector.java @@ -33,6 +33,7 @@ import org.apache.arrow.vector.holders.ExtensionHolder; import org.apache.arrow.vector.holders.NullableUuidHolder; import org.apache.arrow.vector.holders.UuidHolder; +import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.util.UuidUtility; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -358,7 +359,13 @@ void testReaderReadWithUnsupportedHolder() throws Exception { reader.setPosition(0); // Create a mock unsupported holder - ExtensionHolder unsupportedHolder = new ExtensionHolder() {}; + ExtensionHolder unsupportedHolder = + new ExtensionHolder() { + @Override + public ArrowType type() { + return null; + } + }; IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> reader.read(unsupportedHolder)); @@ -377,7 +384,13 @@ void testReaderReadWithArrayIndexUnsupportedHolder() throws Exception { UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); // Create a mock unsupported holder - ExtensionHolder unsupportedHolder = new ExtensionHolder() {}; + ExtensionHolder unsupportedHolder = + new ExtensionHolder() { + @Override + public ArrowType type() { + return null; + } + }; IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> reader.read(0, unsupportedHolder)); diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java index 73c1cd3b74..b2a8cf9ba4 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java @@ -861,7 +861,6 @@ public void testCopyListVectorWithExtensionType() { listWriter.setPosition(i); listWriter.startList(); ExtensionWriter extensionWriter = listWriter.extension(UuidType.INSTANCE); - extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter.writeExtension(UUID.randomUUID()); extensionWriter.writeExtension(UUID.randomUUID()); listWriter.endList(); @@ -874,7 +873,7 @@ public void testCopyListVectorWithExtensionType() { for (int i = 0; i < COUNT; i++) { in.setPosition(i); out.setPosition(i); - ComplexCopier.copy(in, out, new UuidWriterFactory()); + ComplexCopier.copy(in, out); } to.setValueCount(COUNT); @@ -897,11 +896,9 @@ public void testCopyMapVectorWithExtensionType() { mapWriter.startMap(); mapWriter.startEntry(); ExtensionWriter extensionKeyWriter = mapWriter.key().extension(UuidType.INSTANCE); - extensionKeyWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); - extensionKeyWriter.writeExtension(UUID.randomUUID()); + extensionKeyWriter.writeExtension(UUID.randomUUID(), UuidType.INSTANCE); ExtensionWriter extensionValueWriter = mapWriter.value().extension(UuidType.INSTANCE); - extensionValueWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); - extensionValueWriter.writeExtension(UUID.randomUUID()); + extensionValueWriter.writeExtension(UUID.randomUUID(), UuidType.INSTANCE); mapWriter.endEntry(); mapWriter.endMap(); } @@ -914,7 +911,7 @@ public void testCopyMapVectorWithExtensionType() { for (int i = 0; i < COUNT; i++) { in.setPosition(i); out.setPosition(i); - ComplexCopier.copy(in, out, new UuidWriterFactory()); + ComplexCopier.copy(in, out); } to.setValueCount(COUNT); @@ -934,12 +931,10 @@ public void testCopyStructVectorWithExtensionType() { for (int i = 0; i < COUNT; i++) { structWriter.setPosition(i); structWriter.start(); - ExtensionWriter extensionWriter1 = structWriter.extension("timestamp1", UuidType.INSTANCE); - extensionWriter1.addExtensionTypeWriterFactory(new UuidWriterFactory()); - extensionWriter1.writeExtension(UUID.randomUUID()); - ExtensionWriter extensionWriter2 = structWriter.extension("timestamp2", UuidType.INSTANCE); - extensionWriter2.addExtensionTypeWriterFactory(new UuidWriterFactory()); - extensionWriter2.writeExtension(UUID.randomUUID()); + ExtensionWriter extensionWriter1 = structWriter.extension("uuid1", UuidType.INSTANCE); + extensionWriter1.writeExtension(UUID.randomUUID(), UuidType.INSTANCE); + ExtensionWriter extensionWriter2 = structWriter.extension("uuid2", UuidType.INSTANCE); + extensionWriter2.writeExtension(UUID.randomUUID(), UuidType.INSTANCE); structWriter.end(); } @@ -951,7 +946,7 @@ public void testCopyStructVectorWithExtensionType() { for (int i = 0; i < COUNT; i++) { in.setPosition(i); out.setPosition(i); - ComplexCopier.copy(in, out, new UuidWriterFactory()); + ComplexCopier.copy(in, out); } to.setValueCount(COUNT); diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java index c71717a027..5b6d65d6ba 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java @@ -31,6 +31,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.DecimalVector; import org.apache.arrow.vector.DirtyRootAllocator; +import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.LargeVarBinaryVector; import org.apache.arrow.vector.LargeVarCharVector; import org.apache.arrow.vector.UuidVector; @@ -49,6 +50,7 @@ import org.apache.arrow.vector.holders.NullableTimeStampMilliTZHolder; import org.apache.arrow.vector.holders.TimeStampMilliTZHolder; import org.apache.arrow.vector.holders.UnionHolder; +import org.apache.arrow.vector.holders.UuidHolder; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -57,6 +59,7 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.DecimalUtility; import org.apache.arrow.vector.util.Text; +import org.apache.arrow.vector.util.UuidUtility; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -100,7 +103,6 @@ public void testPromoteToUnion() throws Exception { writer.integer("A").writeInt(10); // we don't write anything in 3 - writer.setPosition(4); writer.integer("A").writeInt(100); @@ -130,9 +132,21 @@ public void testPromoteToUnion() throws Exception { binHolder.buffer = buf; writer.fixedSizeBinary("A", 4).write(binHolder); + writer.setPosition(9); + UUID uuid = UUID.randomUUID(); + writer.extension("A", UuidType.INSTANCE).writeExtension(uuid, UuidType.INSTANCE); + writer.end(); + + writer.setPosition(10); + UUID uuid2 = UUID.randomUUID(); + UuidHolder uuidHolder = new UuidHolder(); + uuidHolder.buffer = allocator.buffer(UuidType.UUID_BYTE_WIDTH); + uuidHolder.buffer.setBytes(0, UuidUtility.getBytesFromUUID(uuid2)); + writer.extension("A", UuidType.INSTANCE).write(uuidHolder); writer.end(); + allocator.releaseBytes(UuidType.UUID_BYTE_WIDTH); - container.setValueCount(9); + container.setValueCount(11); final UnionVector uv = v.getChild("A", UnionVector.class); @@ -169,6 +183,12 @@ public void testPromoteToUnion() throws Exception { .order(ByteOrder.nativeOrder()) .getInt()); + assertFalse(uv.isNull(9), "9 shouldn't be null"); + assertEquals(uuid, uv.getObject(9)); + + assertFalse(uv.isNull(10), "10 shouldn't be null"); + assertEquals(uuid2, uv.getObject(10)); + container.clear(); container.allocateNew(); @@ -791,12 +811,11 @@ public void testExtensionType() throws Exception { UUID u2 = UUID.randomUUID(); container.allocateNew(); container.setValueCount(1); - writer.addExtensionTypeWriterFactory(new UuidWriterFactory()); writer.setPosition(0); - writer.writeExtension(u1); + writer.writeExtension(u1, UuidType.INSTANCE); writer.setPosition(1); - writer.writeExtension(u2); + writer.writeExtension(u2, UuidType.INSTANCE); container.setValueCount(2); @@ -817,16 +836,15 @@ public void testExtensionTypeForList() throws Exception { UUID u2 = UUID.randomUUID(); container.allocateNew(); container.setValueCount(1); - writer.addExtensionTypeWriterFactory(new UuidWriterFactory()); writer.setPosition(0); - writer.writeExtension(u1); + writer.writeExtension(u1, UuidType.INSTANCE); writer.setPosition(1); - writer.writeExtension(u2); + writer.writeExtension(u2, UuidType.INSTANCE); container.setValueCount(2); - UuidVector uuidVector = (UuidVector) container.getDataVector(); + FieldVector uuidVector = container.getDataVector(); assertEquals(u1, uuidVector.getObject(0)); assertEquals(u2, uuidVector.getObject(1)); } diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index 3a8f3f8e6a..b131bf07e2 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -66,7 +66,6 @@ import org.apache.arrow.vector.complex.impl.UnionMapReader; import org.apache.arrow.vector.complex.impl.UnionReader; import org.apache.arrow.vector.complex.impl.UnionWriter; -import org.apache.arrow.vector.complex.impl.UuidWriterFactory; import org.apache.arrow.vector.complex.reader.BaseReader.StructReader; import org.apache.arrow.vector.complex.reader.BigIntReader; import org.apache.arrow.vector.complex.reader.FieldReader; @@ -87,6 +86,7 @@ import org.apache.arrow.vector.holders.NullableFixedSizeBinaryHolder; import org.apache.arrow.vector.holders.NullableTimeStampMilliTZHolder; import org.apache.arrow.vector.holders.NullableTimeStampNanoTZHolder; +import org.apache.arrow.vector.holders.NullableUuidHolder; import org.apache.arrow.vector.holders.TimeStampMilliTZHolder; import org.apache.arrow.vector.holders.UuidHolder; import org.apache.arrow.vector.types.TimeUnit; @@ -1106,6 +1106,13 @@ public void simpleUnion() throws Exception { new UnionVector("union", allocator, /* field type */ null, /* call-back */ null); UnionWriter unionWriter = new UnionWriter(vector); unionWriter.allocate(); + + UUID uuid = UUID.randomUUID(); + ByteBuffer bb = ByteBuffer.allocate(16); + bb.putLong(uuid.getMostSignificantBits()); + bb.putLong(uuid.getLeastSignificantBits()); + byte[] uuidByte = bb.array(); + for (int i = 0; i < COUNT; i++) { unionWriter.setPosition(i); if (i % 5 == 0) { @@ -1128,6 +1135,12 @@ public void simpleUnion() throws Exception { holder.buffer = buf; unionWriter.write(holder); bufs.add(buf); + } else if (i % 5 == 4) { + UuidHolder holder = new UuidHolder(); + holder.buffer = allocator.buffer(UuidType.UUID_BYTE_WIDTH); + holder.buffer.setBytes(0, uuidByte); + unionWriter.write(holder); + allocator.releaseBytes(UuidType.UUID_BYTE_WIDTH); } else { unionWriter.writeFloat4((float) i); } @@ -1153,6 +1166,10 @@ public void simpleUnion() throws Exception { unionReader.read(holder); assertEquals(i, holder.buffer.getInt(0)); assertEquals(4, holder.byteWidth); + } else if (i % 5 == 4) { + NullableUuidHolder holder = new NullableUuidHolder(); + unionReader.read(holder); + assertEquals(UuidUtility.uuidFromArrowBuf(holder.buffer, 0), uuid); } else { assertEquals((float) i, unionReader.readFloat(), 1e-12); } @@ -2512,8 +2529,7 @@ public void extensionWriterReader() throws Exception { { ExtensionWriter extensionWriter = rootWriter.extension("uuid1", UuidType.INSTANCE); extensionWriter.setPosition(0); - extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); - extensionWriter.writeExtension(u1); + extensionWriter.writeExtension(u1, UuidType.INSTANCE); } // read StructReader rootReader = new SingleStructReaderImpl(parent).reader("root"); diff --git a/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java b/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java index 2ac4045aa2..ae5ac0726c 100644 --- a/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java +++ b/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java @@ -44,10 +44,12 @@ import org.apache.arrow.vector.Float4Vector; import org.apache.arrow.vector.UuidVector; import org.apache.arrow.vector.ValueIterableVector; +import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.compare.Range; import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.complex.StructVector; +import org.apache.arrow.vector.complex.writer.FieldWriter; import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.ipc.ArrowFileReader; import org.apache.arrow.vector.ipc.ArrowFileWriter; @@ -333,6 +335,11 @@ public String serialize() { public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator) { return new LocationVector(name, allocator); } + + @Override + public FieldWriter getNewFieldWriter(ValueVector vector) { + throw new UnsupportedOperationException("Not yet implemented."); + } } public static class LocationVector extends ExtensionTypeVector From 349d402a61733084399cc791710e251097b87ea3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Sat, 17 Jan 2026 05:00:44 +0000 Subject: [PATCH 257/408] GH-964: Fix IndexOutOfBoundsException in Array.getResultSet() for JDBC clients (#965) ## What's Changed - Fixed JDBC specification in ArrowFlightJdbcArray.getResultSet() that caused IndexOutOfBoundsException in JDBC clients like DBeaver when reading array columns - The method returned a single-column ResultSet containing only array values, but JDBC spec requires a 2-column format - Not it returns two columns: - Column 1 (INDEX): 1-based element indices per JDBC specification - Column 2: The actual array element values Closes #964. --- .../driver/jdbc/ArrowFlightJdbcArray.java | 19 +++++++++++++++---- .../driver/jdbc/ArrowFlightJdbcArrayTest.java | 4 ++-- ...stractArrowFlightJdbcListAccessorTest.java | 7 ++++++- .../ArrowFlightJdbcMapVectorAccessorTest.java | 16 ++++++++-------- 4 files changed, 31 insertions(+), 15 deletions(-) diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcArray.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcArray.java index 9b9eba51e5..f3d76ace92 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcArray.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcArray.java @@ -26,6 +26,7 @@ import org.apache.arrow.driver.jdbc.utils.SqlTypes; import org.apache.arrow.memory.util.LargeMemoryUtil; import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -135,12 +136,22 @@ public ResultSet getResultSet(long index, int count) throws SQLException { private static ResultSet getResultSetNoBoundariesCheck( ValueVector dataVector, long start, long count) throws SQLException { + int intStart = LargeMemoryUtil.checkedCastToInt(start); + int intCount = LargeMemoryUtil.checkedCastToInt(count); + + // Create an index vector with 1-based indices (per JDBC spec) to return with value vector + IntVector indexVector = new IntVector("INDEX", dataVector.getAllocator()); + indexVector.allocateNew(intCount); + for (int i = 0; i < intCount; i++) { + indexVector.set(i, i + 1); + } + indexVector.setValueCount(intCount); + TransferPair transferPair = dataVector.getTransferPair(dataVector.getAllocator()); - transferPair.splitAndTransfer( - LargeMemoryUtil.checkedCastToInt(start), LargeMemoryUtil.checkedCastToInt(count)); - FieldVector vectorSlice = (FieldVector) transferPair.getTo(); + transferPair.splitAndTransfer(intStart, intCount); + FieldVector valueVector = (FieldVector) transferPair.getTo(); - VectorSchemaRoot vectorSchemaRoot = VectorSchemaRoot.of(vectorSlice); + VectorSchemaRoot vectorSchemaRoot = VectorSchemaRoot.of(indexVector, valueVector); return ArrowFlightJdbcVectorSchemaRootResultSet.fromVectorSchemaRoot(vectorSchemaRoot); } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcArrayTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcArrayTest.java index 06d101724c..cb6abacb2f 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcArrayTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcArrayTest.java @@ -129,7 +129,7 @@ public void testShouldGetResultSetReturnValidResultSet() throws SQLException { try (ResultSet resultSet = arrowFlightJdbcArray.getResultSet()) { int count = 0; while (resultSet.next()) { - assertEquals((Object) resultSet.getInt(1), dataVector.getObject(count)); + assertEquals((Object) resultSet.getInt(2), dataVector.getObject(count)); count++; } } @@ -142,7 +142,7 @@ public void testShouldGetResultSetReturnValidResultSetWithOffsets() throws SQLEx try (ResultSet resultSet = arrowFlightJdbcArray.getResultSet(3, 5)) { int count = 0; while (resultSet.next()) { - assertEquals((Object) resultSet.getInt(1), dataVector.getObject(count + 3)); + assertEquals((Object) resultSet.getInt(2), dataVector.getObject(count + 3)); count++; } assertEquals(5, count); diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/AbstractArrowFlightJdbcListAccessorTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/AbstractArrowFlightJdbcListAccessorTest.java index ad689837e2..c5eb6e34ef 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/AbstractArrowFlightJdbcListAccessorTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/AbstractArrowFlightJdbcListAccessorTest.java @@ -191,7 +191,12 @@ public void testShouldGetArrayGetResultSetReturnValidResultSet( try (ResultSet rs = array.getResultSet()) { int count = 0; while (rs.next()) { - final int value = rs.getInt(1); + // Column 1: 1-based index (per JDBC spec) + final int index = rs.getInt(1); + assertThat(index, equalTo(count + 1)); + + // Column 2: actual value (per JDBC spec) + final int value = rs.getInt(2); assertThat(value, equalTo(currentRow * count)); count++; } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/ArrowFlightJdbcMapVectorAccessorTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/ArrowFlightJdbcMapVectorAccessorTest.java index 696e5afb71..f2d1725fd8 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/ArrowFlightJdbcMapVectorAccessorTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/ArrowFlightJdbcMapVectorAccessorTest.java @@ -153,15 +153,15 @@ public void testShouldGetArrayReturnValidArray() throws SQLException { try (ResultSet resultSet = array.getResultSet()) { assertTrue(resultSet.next()); - Map entry = resultSet.getObject(1, Map.class); + Map entry = resultSet.getObject(2, Map.class); assertEquals(1, entry.get("key")); assertEquals(11, entry.get("value")); assertTrue(resultSet.next()); - entry = resultSet.getObject(1, Map.class); + entry = resultSet.getObject(2, Map.class); assertEquals(2, entry.get("key")); assertEquals(22, entry.get("value")); assertTrue(resultSet.next()); - entry = resultSet.getObject(1, Map.class); + entry = resultSet.getObject(2, Map.class); assertEquals(3, entry.get("key")); assertEquals(33, entry.get("value")); assertFalse(resultSet.next()); @@ -173,7 +173,7 @@ public void testShouldGetArrayReturnValidArray() throws SQLException { assertFalse(accessor.wasNull()); try (ResultSet resultSet = array.getResultSet()) { assertTrue(resultSet.next()); - Map entry = resultSet.getObject(1, Map.class); + Map entry = resultSet.getObject(2, Map.class); assertEquals(2, entry.get("key")); assertNull(entry.get("value")); assertFalse(resultSet.next()); @@ -185,19 +185,19 @@ public void testShouldGetArrayReturnValidArray() throws SQLException { assertFalse(accessor.wasNull()); try (ResultSet resultSet = array.getResultSet()) { assertTrue(resultSet.next()); - Map entry = resultSet.getObject(1, Map.class); + Map entry = resultSet.getObject(2, Map.class); assertEquals(0, entry.get("key")); assertEquals(2000, entry.get("value")); assertTrue(resultSet.next()); - entry = resultSet.getObject(1, Map.class); + entry = resultSet.getObject(2, Map.class); assertEquals(1, entry.get("key")); assertEquals(2001, entry.get("value")); assertTrue(resultSet.next()); - entry = resultSet.getObject(1, Map.class); + entry = resultSet.getObject(2, Map.class); assertEquals(2, entry.get("key")); assertEquals(2002, entry.get("value")); assertTrue(resultSet.next()); - entry = resultSet.getObject(1, Map.class); + entry = resultSet.getObject(2, Map.class); assertEquals(3, entry.get("key")); assertEquals(2003, entry.get("value")); assertFalse(resultSet.next()); From 6eab884d7d868863a283435c29e7bbd912138379 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jan 2026 09:33:31 +0100 Subject: [PATCH 258/408] MINOR: Bump org.bouncycastle:bcpkix-jdk18on from 1.82 to 1.83 (#969) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.bouncycastle:bcpkix-jdk18on](https://github.com/bcgit/bc-java) from 1.82 to 1.83.

Changelog

Sourced from org.bouncycastle:bcpkix-jdk18on's changelog.

2.1.1 Version Release: 1.84 Date:      TBD

2.2.1 Version Release: 1.83 Date:      2025, November 27th.

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.bouncycastle:bcpkix-jdk18on&package-manager=maven&previous-version=1.82&new-version=1.83)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 8801ad8178..d84352e2da 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -140,7 +140,7 @@ under the License. org.bouncycastle bcpkix-jdk18on - 1.82 + 1.83 From 1e8608a5b3205eff9d41dac11cf0d2a9f334be83 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jan 2026 09:34:59 +0100 Subject: [PATCH 259/408] MINOR: Bump logback.version from 1.5.24 to 1.5.25 (#975) Bumps `logback.version` from 1.5.24 to 1.5.25. Updates `ch.qos.logback:logback-classic` from 1.5.24 to 1.5.25
Commits
  • f426e00 prepare release of 1.5.25
  • d28931f restrict object creation to expected supertype
  • aa264f7 test default variable values in appender-ref ref attribute
  • 8fb403a adjust copyright year
  • b294a12 check optionList in start()
  • b65040a Add EpochConverter for milliseconds/seconds since epoch (related to issue #96...
  • 0690174 cla for Duncan Jauncey
  • 71dc2af Removed email address for Tony.
  • 1f97ae1 check for undeclared by referenced appenders
  • b07355e Move the artifact version checking code to VersionUtil in logback-core.
  • Additional commits viewable in compare view

Updates `ch.qos.logback:logback-core` from 1.5.24 to 1.5.25
Commits
  • f426e00 prepare release of 1.5.25
  • d28931f restrict object creation to expected supertype
  • aa264f7 test default variable values in appender-ref ref attribute
  • 8fb403a adjust copyright year
  • b294a12 check optionList in start()
  • b65040a Add EpochConverter for milliseconds/seconds since epoch (related to issue #96...
  • 0690174 cla for Duncan Jauncey
  • 71dc2af Removed email address for Tony.
  • 1f97ae1 check for undeclared by referenced appenders
  • b07355e Move the artifact version checking code to VersionUtil in logback-core.
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d6fdfdd477..64e4a612ec 100644 --- a/pom.xml +++ b/pom.xml @@ -111,7 +111,7 @@ under the License. true 2.42.0 3.53.0 - 1.5.24 + 1.5.25 none -Xdoclint:none From 3d44a1c2e71d5f38980cc1030e5652f5f62b942d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jan 2026 09:46:51 +0100 Subject: [PATCH 260/408] MINOR: Bump com.fasterxml.jackson:jackson-bom from 2.18.3 to 2.21.0 (#973) Bumps [com.fasterxml.jackson:jackson-bom](https://github.com/FasterXML/jackson-bom) from 2.18.3 to 2.21.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.fasterxml.jackson:jackson-bom&package-manager=maven&previous-version=2.18.3&new-version=2.21.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 64e4a612ec..b3141d5cd8 100644 --- a/pom.xml +++ b/pom.xml @@ -100,7 +100,7 @@ under the License. 4.2.9.Final 1.78.0 4.33.1 - 2.18.3 + 2.21.0 3.4.2 25.2.10 1.12.1 From f36777c09246532090c01ca2b5127bb80fd703fb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jan 2026 09:48:12 +0100 Subject: [PATCH 261/408] MINOR: Bump parquet.version from 1.16.0 to 1.17.0 (#968) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps `parquet.version` from 1.16.0 to 1.17.0. Updates `org.apache.parquet:parquet-avro` from 1.16.0 to 1.17.0
Release notes

Sourced from org.apache.parquet:parquet-avro's releases.

Apache Parquet 1.17.0

What's Changed

New Contributors

... (truncated)

Commits
  • fac0c74 [maven-release-plugin] prepare release apache-parquet-1.17.0-rc0
  • a8ead9d Bump protobuf.version from 4.33.1 to 4.33.2 (#3373)
  • 0ecd799 Allow reading dictionary encoded boolean (#3370)
  • 46218f2 Bump commons-io:commons-io from 2.18.0 to 2.21.0 (#3369)
  • 7ec3284 Exclude package-info.class from shaded fastutil (#3322)
  • 7453be4 Bump com.google.guava:guava from 33.4.0-jre to 33.5.0-jre (#3366)
  • 893ef11 Bump easymock 5.6.0 to support Java 25 (#3363)
  • 6b2940c Remove unused parquet-thrift dependencies (#3323)
  • 5040a63 Bump protobuf.version from 3.25.6 to 4.30.2 (#3182)
  • 2ccc243 MINOR: parquet-avro tests should not debug to stderr (#3329)
  • Additional commits viewable in compare view

Updates `org.apache.parquet:parquet-hadoop` from 1.16.0 to 1.17.0
Release notes

Sourced from org.apache.parquet:parquet-hadoop's releases.

Apache Parquet 1.17.0

What's Changed

New Contributors

... (truncated)

Commits
  • fac0c74 [maven-release-plugin] prepare release apache-parquet-1.17.0-rc0
  • a8ead9d Bump protobuf.version from 4.33.1 to 4.33.2 (#3373)
  • 0ecd799 Allow reading dictionary encoded boolean (#3370)
  • 46218f2 Bump commons-io:commons-io from 2.18.0 to 2.21.0 (#3369)
  • 7ec3284 Exclude package-info.class from shaded fastutil (#3322)
  • 7453be4 Bump com.google.guava:guava from 33.4.0-jre to 33.5.0-jre (#3366)
  • 893ef11 Bump easymock 5.6.0 to support Java 25 (#3363)
  • 6b2940c Remove unused parquet-thrift dependencies (#3323)
  • 5040a63 Bump protobuf.version from 3.25.6 to 4.30.2 (#3182)
  • 2ccc243 MINOR: parquet-avro tests should not debug to stderr (#3329)
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dataset/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataset/pom.xml b/dataset/pom.xml index 2d582268a6..fcf54e785f 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -32,7 +32,7 @@ under the License. ../../../cpp/release-build/ - 1.16.0 + 1.17.0 1.12.1 From f9ab35c5628ab19af8df6c1146ac495edf917219 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jan 2026 13:40:23 +0100 Subject: [PATCH 262/408] MINOR: Bump commons-io:commons-io from 2.19.0 to 2.21.0 (#974) Bumps [commons-io:commons-io](https://github.com/apache/commons-io) from 2.19.0 to 2.21.0.
Changelog

Sourced from commons-io:commons-io's changelog.

Apache Commons IO 2.21.0 Release Notes

The Apache Commons IO team is pleased to announce the release of Apache Commons IO 2.21.0.

Introduction

The Apache Commons IO library contains utility classes, stream implementations, file filters, file comparators, endian transformation classes, and much more.

Version 2.21.0: Java 8 or later is required.

New features

o FileUtils#byteCountToDisplaySize() supports Zettabyte, Yottabyte, Ronnabyte and Quettabyte #763. Thanks to strangelookingnerd, Gary Gregory. o Add org.apache.commons.io.FileUtils.ONE_RB #763. Thanks to strangelookingnerd, Gary Gregory. o Add org.apache.commons.io.FileUtils.ONE_QB #763. Thanks to strangelookingnerd, Gary Gregory. o Add org.apache.commons.io.output.ProxyOutputStream.writeRepeat(byte[], int, int, long). Thanks to Gary Gregory. o Add org.apache.commons.io.output.ProxyOutputStream.writeRepeat(byte[], long). Thanks to Gary Gregory. o Add org.apache.commons.io.output.ProxyOutputStream.writeRepeat(int, long). Thanks to Gary Gregory. o Add length unit support in FileSystem limits. Thanks to Piotr P. Karwasz. o Add IOUtils.toByteArray(InputStream, int, int) for safer chunked reading with size validation. Thanks to Piotr P. Karwasz. o Add org.apache.commons.io.file.PathUtils.getPath(String, String). Thanks to Gary Gregory. o Add org.apache.commons.io.channels.ByteArraySeekableByteChannel. Thanks to Gary Gregory. o Add IOIterable.asIterable(). Thanks to Gary Gregory. o Add NIO channel support to AbstractStreamBuilder. Thanks to Piotr P. Karwasz. o Add CloseShieldChannel to close-shielded NIO Channels #786. Thanks to Piotr P. Karwasz. o Added IOUtils.checkFromIndexSize as a Java 8 backport of Objects.checkFromIndexSize #790. Thanks to Piotr P. Karwasz.

Fixed Bugs

o When testing on Java 21 and up, enable -XX:+EnableDynamicAgentLoading. Thanks to Gary Gregory. o When testing on Java 24 and up, don't fail FileUtilsListFilesTest for a different behavior in the JRE. Thanks to Gary Gregory. o ValidatingObjectInputStream does not validate dynamic proxy interfaces. Thanks to Stanislav Fort, Gary Gregory. o BoundedInputStream.getRemaining() now reports Long.MAX_VALUE instead of 0 when no limit is set. Thanks to Piotr P. Karwasz. o BoundedInputStream.available() correctly accounts for the maximum read limit. Thanks to Piotr P. Karwasz. o Deprecate IOUtils.readFully(InputStream, int) in favor of toByteArray(InputStream, int). Thanks to Gary Gregory, Piotr P. Karwasz. o IOUtils.toByteArray(InputStream) now throws IOException on byte array overflow. Thanks to Piotr P. Karwasz. o Javadoc general improvements. Thanks to Gary Gregory, Piotr P. Karwasz. o IOUtils.toByteArray() now throws EOFException when not enough data is available #796. Thanks to Piotr P. Karwasz. o Fix IOUtils.skip() usage in concurrent scenarios. Thanks to Piotr P. Karwasz. o [javadoc] Fix XmlStreamReader Javadoc to indicate the correct class that is built #806. Thanks to J Hawkins.

Changes

o Bump org.apache.commons:commons-parent from 85 to 91 #774, #783, #808. Thanks to Gary Gregory, Dependabot.

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=commons-io:commons-io&package-manager=maven&previous-version=2.19.0&new-version=2.21.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dataset/pom.xml | 2 +- flight/flight-sql-jdbc-core/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dataset/pom.xml b/dataset/pom.xml index fcf54e785f..3a8f048628 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -156,7 +156,7 @@ under the License. commons-io commons-io - 2.19.0 + 2.21.0 test diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index d84352e2da..bb191ca9ed 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -105,7 +105,7 @@ under the License. commons-io commons-io - 2.19.0 + 2.21.0 test From a74728d490a8307b926f0539eeb582220496ce58 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jan 2026 14:06:30 +0100 Subject: [PATCH 263/408] MINOR: Bump com.gradle:develocity-maven-extension from 2.0 to 2.3.1 (#976) Bumps com.gradle:develocity-maven-extension from 2.0 to 2.3.1. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.gradle:develocity-maven-extension&package-manager=maven&previous-version=2.0&new-version=2.3.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .mvn/extensions.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index 943140738d..b136e95f43 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -23,7 +23,7 @@ com.gradle develocity-maven-extension - 2.0 + 2.3.1 com.gradle From db9fff8638e907012b4fb4585723ebbc6514ab20 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jan 2026 14:26:37 +0100 Subject: [PATCH 264/408] MINOR: Bump org.apache.orc:orc-core from 2.2.1 to 2.2.2 (#971) Bumps org.apache.orc:orc-core from 2.2.1 to 2.2.2. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.orc:orc-core&package-manager=maven&previous-version=2.2.1&new-version=2.2.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- adapter/orc/pom.xml | 2 +- dataset/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/adapter/orc/pom.xml b/adapter/orc/pom.xml index ef72b2de65..89d45e155c 100644 --- a/adapter/orc/pom.xml +++ b/adapter/orc/pom.xml @@ -61,7 +61,7 @@ under the License. org.apache.orc orc-core - 2.2.1 + 2.2.2 test diff --git a/dataset/pom.xml b/dataset/pom.xml index 3a8f048628..686a234358 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -130,7 +130,7 @@ under the License. org.apache.orc orc-core - 2.2.1 + 2.2.2 test From 71c418c2fde3c49d4cfa4ec564514d04f66f71bf Mon Sep 17 00:00:00 2001 From: Joana Hrotko Date: Mon, 19 Jan 2026 13:41:06 +0000 Subject: [PATCH 265/408] GH-948: Use buffer indexing for UUID vector (#949) ## What's Changed The current UUID vector implementation creates new buffer slices when reading values through holders, which has several drawbacks: - Memory overhead: Each slice creates a new ArrowBuf object - Performance impact: Buffer slicing is slower than direct buffer indexing - Inconsistency: Other fixed-width types (like Decimal) use buffer indexing with a `start` offset field ### Proposed Changes 1. Add `start` field to UUID holders to track buffer offsets: - `UuidHolder`: Add `public int start = 0;` - `NullableUuidHolder`: Add `public int start = 0;` 2. Update `UuidVector` to use buffer indexing 3. Update readers and writers ### Related Work - Original UUID extension type implementation: GH-825 (#903) Closes #948 --- .../arrow/vector/UuidVectorBenchmarks.java | 134 +++++++ .../org/apache/arrow/vector/UuidVector.java | 115 +++--- .../impl/NullableUuidHolderReaderImpl.java | 123 +++++++ .../vector/complex/impl/UuidReaderImpl.java | 8 +- .../vector/complex/impl/UuidWriterImpl.java | 9 +- .../vector/holders/NullableUuidHolder.java | 3 + .../arrow/vector/holders/UuidHolder.java | 3 + .../arrow/vector/TestLargeListVector.java | 12 +- .../apache/arrow/vector/TestListVector.java | 24 +- .../apache/arrow/vector/TestMapVector.java | 20 +- .../org/apache/arrow/vector/TestUuidType.java | 3 +- .../apache/arrow/vector/TestUuidVector.java | 334 ++++++++++++++++-- .../complex/writer/TestComplexWriter.java | 4 +- 13 files changed, 649 insertions(+), 143 deletions(-) create mode 100644 performance/src/main/java/org/apache/arrow/vector/UuidVectorBenchmarks.java create mode 100644 vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableUuidHolderReaderImpl.java diff --git a/performance/src/main/java/org/apache/arrow/vector/UuidVectorBenchmarks.java b/performance/src/main/java/org/apache/arrow/vector/UuidVectorBenchmarks.java new file mode 100644 index 0000000000..b5f87e7a75 --- /dev/null +++ b/performance/src/main/java/org/apache/arrow/vector/UuidVectorBenchmarks.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector; + +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.complex.impl.UuidWriterImpl; +import org.apache.arrow.vector.holders.NullableUuidHolder; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.profile.GCProfiler; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +/** Benchmarks for {@link UuidVector}. */ +@State(Scope.Benchmark) +public class UuidVectorBenchmarks { + // checkstyle:off: MissingJavadocMethod + + private static final int VECTOR_LENGTH = 10_000; + + private static final int ALLOCATOR_CAPACITY = 1024 * 1024; + + private BufferAllocator allocator; + + private UuidVector vector; + + private UUID[] testUuids; + + @Setup + public void prepare() { + allocator = new RootAllocator(ALLOCATOR_CAPACITY); + vector = new UuidVector("vector", allocator); + vector.allocateNew(VECTOR_LENGTH); + vector.setValueCount(VECTOR_LENGTH); + + // Pre-generate UUIDs for consistent benchmarking + testUuids = new UUID[VECTOR_LENGTH]; + for (int i = 0; i < VECTOR_LENGTH; i++) { + testUuids[i] = new UUID(i, i * 2L); + } + } + + @TearDown + public void tearDown() { + vector.close(); + allocator.close(); + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + public void setWithHolder() { + NullableUuidHolder holder = new NullableUuidHolder(); + for (int i = 0; i < VECTOR_LENGTH; i++) { + vector.get(i, holder); + vector.setSafe(i, holder); + } + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + public void setUuidDirectly() { + for (int i = 0; i < VECTOR_LENGTH; i++) { + vector.setSafe(i, testUuids[i]); + } + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + public void setWithWriter() { + UuidWriterImpl writer = new UuidWriterImpl(vector); + for (int i = 0; i < VECTOR_LENGTH; i++) { + writer.writeExtension(testUuids[i]); + } + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + public void getWithUuidHolder() { + NullableUuidHolder holder = new NullableUuidHolder(); + for (int i = 0; i < VECTOR_LENGTH; i++) { + vector.get(i, holder); + } + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + public void getUuidDirectly() { + for (int i = 0; i < VECTOR_LENGTH; i++) { + UUID uuid = vector.getObject(i); + } + } + + public static void main(String[] args) throws RunnerException { + Options opt = + new OptionsBuilder() + .include(UuidVectorBenchmarks.class.getSimpleName()) + .forks(1) + .addProfiler(GCProfiler.class) + .build(); + + new Runner(opt).run(); + } + // checkstyle:on: MissingJavadocMethod +} diff --git a/vector/src/main/java/org/apache/arrow/vector/UuidVector.java b/vector/src/main/java/org/apache/arrow/vector/UuidVector.java index e0dadd1c67..e1e61a5a2e 100644 --- a/vector/src/main/java/org/apache/arrow/vector/UuidVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/UuidVector.java @@ -23,7 +23,9 @@ import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.util.ArrowBufPointer; +import org.apache.arrow.memory.util.ByteFunctionHelpers; import org.apache.arrow.memory.util.hash.ArrowBufHasher; +import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.complex.impl.UuidReaderImpl; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.extension.UuidType; @@ -132,7 +134,8 @@ public int hashCode(int index) { @Override public int hashCode(int index, ArrowBufHasher hasher) { - return getUnderlyingVector().hashCode(index, hasher); + int start = this.getStartOffset(index); + return ByteFunctionHelpers.hash(hasher, this.getDataBuffer(), start, start + UUID_BYTE_WIDTH); } /** @@ -145,21 +148,6 @@ public int isSet(int index) { return getUnderlyingVector().isSet(index); } - /** - * Gets the UUID value at the given index as an ArrowBuf. - * - * @param index the index to retrieve - * @return a buffer slice containing the 16-byte UUID - * @throws IllegalStateException if the value at the index is null and null checking is enabled - */ - public ArrowBuf get(int index) throws IllegalStateException { - if (NullCheckingForGet.NULL_CHECKING_ENABLED && this.isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } else { - return getBufferSlicePostNullCheck(index); - } - } - /** * Reads the UUID value at the given index into a NullableUuidHolder. * @@ -167,23 +155,24 @@ public ArrowBuf get(int index) throws IllegalStateException { * @param holder the holder to populate with the UUID data */ public void get(int index, NullableUuidHolder holder) { - if (NullCheckingForGet.NULL_CHECKING_ENABLED && this.isSet(index) == 0) { + Preconditions.checkArgument(index >= 0, "Cannot get negative index in UUID vector."); + if (isSet(index) == 0) { holder.isSet = 0; - } else { - holder.isSet = 1; - holder.buffer = getBufferSlicePostNullCheck(index); + return; } + holder.isSet = 1; + holder.buffer = getDataBuffer(); + holder.start = getStartOffset(index); } /** - * Reads the UUID value at the given index into a UuidHolder. + * Calculates the byte offset for a given index in the data buffer. * - * @param index the index to read from - * @param holder the holder to populate with the UUID data + * @param index the index of the UUID value + * @return the byte offset in the data buffer */ - public void get(int index, UuidHolder holder) { - holder.isSet = 1; - holder.buffer = getBufferSlicePostNullCheck(index); + public final int getStartOffset(int index) { + return index * UUID_BYTE_WIDTH; } /** @@ -207,7 +196,7 @@ public void set(int index, UUID value) { * @param holder the holder containing the UUID data */ public void set(int index, UuidHolder holder) { - this.set(index, holder.isSet, holder.buffer); + this.set(index, holder.buffer, holder.start); } /** @@ -217,28 +206,11 @@ public void set(int index, UuidHolder holder) { * @param holder the holder containing the UUID data */ public void set(int index, NullableUuidHolder holder) { - this.set(index, holder.isSet, holder.buffer); - } - - /** - * Sets the UUID value at the given index with explicit null flag. - * - * @param index the index to set - * @param isSet 1 if the value is set, 0 if null - * @param buffer the buffer containing the 16-byte UUID data - */ - public void set(int index, int isSet, ArrowBuf buffer) { - getUnderlyingVector().set(index, isSet, buffer); - } - - /** - * Sets the UUID value at the given index from an ArrowBuf. - * - * @param index the index to set - * @param value the buffer containing the 16-byte UUID data - */ - public void set(int index, ArrowBuf value) { - getUnderlyingVector().set(index, value); + if (holder.isSet == 0) { + getUnderlyingVector().setNull(index); + } else { + this.set(index, holder.buffer, holder.start); + } } /** @@ -249,10 +221,12 @@ public void set(int index, ArrowBuf value) { * @param sourceOffset the offset in the source buffer where the UUID data starts */ public void set(int index, ArrowBuf source, int sourceOffset) { - // Copy bytes from source buffer to target vector data buffer - ArrowBuf dataBuffer = getUnderlyingVector().getDataBuffer(); - dataBuffer.setBytes((long) index * UUID_BYTE_WIDTH, source, sourceOffset, UUID_BYTE_WIDTH); - getUnderlyingVector().setIndexDefined(index); + Preconditions.checkNotNull(source, "Cannot set UUID vector, the source buffer is null."); + + BitVectorHelper.setBit(getUnderlyingVector().getValidityBuffer(), index); + getUnderlyingVector() + .getDataBuffer() + .setBytes((long) index * UUID_BYTE_WIDTH, source, sourceOffset, UUID_BYTE_WIDTH); } /** @@ -286,10 +260,10 @@ public void setSafe(int index, UUID value) { * @param holder the holder containing the UUID data, or null to set a null value */ public void setSafe(int index, NullableUuidHolder holder) { - if (holder != null) { - getUnderlyingVector().setSafe(index, holder.isSet, holder.buffer); - } else { + if (holder == null || holder.isSet == 0) { getUnderlyingVector().setNull(index); + } else { + this.setSafe(index, holder.buffer, holder.start); } } @@ -297,14 +271,23 @@ public void setSafe(int index, NullableUuidHolder holder) { * Sets the UUID value at the given index from a UuidHolder, expanding capacity if needed. * * @param index the index to set - * @param holder the holder containing the UUID data, or null to set a null value + * @param holder the holder containing the UUID data */ public void setSafe(int index, UuidHolder holder) { - if (holder != null) { - getUnderlyingVector().setSafe(index, holder.isSet, holder.buffer); - } else { - getUnderlyingVector().setNull(index); - } + this.setSafe(index, holder.buffer, holder.start); + } + + /** + * Sets the UUID value at the given index by copying from a source buffer, expanding capacity if + * needed. + * + * @param index the index to set + * @param buffer the source buffer to copy from + * @param start the offset in the source buffer where the UUID data starts + */ + public void setSafe(int index, ArrowBuf buffer, int start) { + getUnderlyingVector().handleSafe(index); + this.set(index, buffer, start); } /** @@ -400,15 +383,9 @@ public TransferPair getTransferPair(BufferAllocator allocator) { return getTransferPair(this.getField().getName(), allocator); } - private ArrowBuf getBufferSlicePostNullCheck(int index) { - return getUnderlyingVector() - .getDataBuffer() - .slice((long) index * UUID_BYTE_WIDTH, UUID_BYTE_WIDTH); - } - @Override public int getTypeWidth() { - return getUnderlyingVector().getTypeWidth(); + return UUID_BYTE_WIDTH; } /** {@link TransferPair} for {@link UuidVector}. */ diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableUuidHolderReaderImpl.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableUuidHolderReaderImpl.java new file mode 100644 index 0000000000..7a5312f6ed --- /dev/null +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableUuidHolderReaderImpl.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.complex.impl; + +import org.apache.arrow.vector.holders.ExtensionHolder; +import org.apache.arrow.vector.holders.NullableUuidHolder; +import org.apache.arrow.vector.holders.UuidHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.util.UuidUtility; + +/** + * Reader implementation for reading UUID values from a {@link NullableUuidHolder}. + * + *

This reader wraps a single UUID holder value and provides methods to read from it. Unlike + * {@link UuidReaderImpl} which reads from a vector, this reader operates on a holder instance. + * + * @see NullableUuidHolder + * @see UuidReaderImpl + */ +public class NullableUuidHolderReaderImpl extends AbstractFieldReader { + private final NullableUuidHolder holder; + + /** + * Constructs a reader for the given UUID holder. + * + * @param holder the UUID holder to read from + */ + public NullableUuidHolderReaderImpl(NullableUuidHolder holder) { + this.holder = holder; + } + + @Override + public int size() { + throw new UnsupportedOperationException( + "size() is not supported on NullableUuidHolderReaderImpl. " + + "This reader wraps a single UUID holder value, not a collection. " + + "Use UuidReaderImpl for vector-based UUID reading."); + } + + @Override + public boolean next() { + throw new UnsupportedOperationException( + "next() is not supported on NullableUuidHolderReaderImpl. " + + "This reader wraps a single UUID holder value, not an iterator. " + + "Use UuidReaderImpl for vector-based UUID reading."); + } + + @Override + public void setPosition(int index) { + throw new UnsupportedOperationException( + "setPosition() is not supported on NullableUuidHolderReaderImpl. " + + "This reader wraps a single UUID holder value, not a vector. " + + "Use UuidReaderImpl for vector-based UUID reading."); + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.EXTENSIONTYPE; + } + + @Override + public boolean isSet() { + return holder.isSet == 1; + } + + @Override + public void read(ExtensionHolder h) { + if (h instanceof NullableUuidHolder) { + NullableUuidHolder nullableHolder = (NullableUuidHolder) h; + nullableHolder.buffer = this.holder.buffer; + nullableHolder.isSet = this.holder.isSet; + nullableHolder.start = this.holder.start; + } else if (h instanceof UuidHolder) { + UuidHolder uuidHolder = (UuidHolder) h; + uuidHolder.buffer = this.holder.buffer; + uuidHolder.start = this.holder.start; + } else { + throw new IllegalArgumentException( + "Unsupported holder type: " + + h.getClass().getName() + + ". " + + "Only NullableUuidHolder and UuidHolder are supported for UUID values. " + + "Provided holder type cannot be used to read UUID data."); + } + } + + @Override + public Object readObject() { + if (!isSet()) { + return null; + } + // Convert UUID bytes to Java UUID object + try { + return UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); + } catch (Exception e) { + throw new RuntimeException( + String.format( + "Failed to read UUID from buffer. Invalid Arrow buffer state: " + + "capacity=%d, readableBytes=%d, readerIndex=%d, writerIndex=%d, refCnt=%d. " + + "The buffer must contain exactly 16 bytes of valid UUID data.", + holder.buffer.capacity(), + holder.buffer.readableBytes(), + holder.buffer.readerIndex(), + holder.buffer.writerIndex(), + holder.buffer.refCnt()), + e); + } + } +} diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java index bb35b960d3..bb7ae13e5b 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java @@ -63,9 +63,7 @@ public boolean isSet() { @Override public void read(ExtensionHolder holder) { - if (holder instanceof UuidHolder) { - vector.get(idx(), (UuidHolder) holder); - } else if (holder instanceof NullableUuidHolder) { + if (holder instanceof NullableUuidHolder) { vector.get(idx(), (NullableUuidHolder) holder); } else { throw new IllegalArgumentException( @@ -75,9 +73,7 @@ public void read(ExtensionHolder holder) { @Override public void read(int arrayIndex, ExtensionHolder holder) { - if (holder instanceof UuidHolder) { - vector.get(arrayIndex, (UuidHolder) holder); - } else if (holder instanceof NullableUuidHolder) { + if (holder instanceof NullableUuidHolder) { vector.get(arrayIndex, (NullableUuidHolder) holder); } else { throw new IllegalArgumentException( diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java index ee3c79d5e3..944b7e2e62 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java @@ -51,8 +51,15 @@ public void writeExtension(Object value) { vector.setSafe(getPosition(), (ArrowBuf) value); } else if (value instanceof java.util.UUID) { vector.setSafe(getPosition(), (java.util.UUID) value); + } else if (value instanceof ExtensionHolder) { + write((ExtensionHolder) value); } else { - throw new IllegalArgumentException("Unsupported value type for UUID: " + value.getClass()); + throw new IllegalArgumentException( + "Unsupported value type for UUID: " + + value.getClass().getName() + + ". " + + "Supported types are: byte[] (16 bytes), ArrowBuf (16 bytes), or java.util.UUID. " + + "Convert your value to one of these types before writing."); } vector.setValueCount(getPosition() + 1); } diff --git a/vector/src/main/java/org/apache/arrow/vector/holders/NullableUuidHolder.java b/vector/src/main/java/org/apache/arrow/vector/holders/NullableUuidHolder.java index 7fa50ca761..6a2b4ff604 100644 --- a/vector/src/main/java/org/apache/arrow/vector/holders/NullableUuidHolder.java +++ b/vector/src/main/java/org/apache/arrow/vector/holders/NullableUuidHolder.java @@ -35,6 +35,9 @@ public class NullableUuidHolder extends ExtensionHolder { /** Buffer containing 16-byte UUID data. */ public ArrowBuf buffer; + /** Offset in the buffer where the UUID data starts. */ + public int start = 0; + @Override public ArrowType type() { return UuidType.INSTANCE; diff --git a/vector/src/main/java/org/apache/arrow/vector/holders/UuidHolder.java b/vector/src/main/java/org/apache/arrow/vector/holders/UuidHolder.java index 8a0a66e435..9ec0305f30 100644 --- a/vector/src/main/java/org/apache/arrow/vector/holders/UuidHolder.java +++ b/vector/src/main/java/org/apache/arrow/vector/holders/UuidHolder.java @@ -33,6 +33,9 @@ public class UuidHolder extends ExtensionHolder { /** Buffer containing 16-byte UUID data. */ public ArrowBuf buffer; + /** Offset in the buffer where the UUID data starts. */ + public int start = 0; + /** Constructs a UuidHolder with isSet = 1. */ public UuidHolder() { this.isSet = 1; diff --git a/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java index 759c84651d..ccc0d3e176 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java @@ -37,7 +37,7 @@ import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.complex.writer.BaseWriter.ExtensionWriter; import org.apache.arrow.vector.extension.UuidType; -import org.apache.arrow.vector.holders.UuidHolder; +import org.apache.arrow.vector.holders.NullableUuidHolder; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; @@ -1071,14 +1071,14 @@ public void testCopyValueSafeForExtensionType() throws Exception { assertTrue(reader.isSet(), "first list shouldn't be null"); reader.next(); FieldReader uuidReader = reader.reader(); - UuidHolder holder = new UuidHolder(); + NullableUuidHolder holder = new NullableUuidHolder(); uuidReader.read(holder); - UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u1, actualUuid); reader.next(); uuidReader = reader.reader(); uuidReader.read(holder); - actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u2, actualUuid); // Verify second list @@ -1087,12 +1087,12 @@ public void testCopyValueSafeForExtensionType() throws Exception { reader.next(); uuidReader = reader.reader(); uuidReader.read(holder); - actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u3, actualUuid); reader.next(); uuidReader = reader.reader(); uuidReader.read(holder); - actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u4, actualUuid); reader.next(); uuidReader = reader.reader(); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index e96ac3027c..1fe4c59f63 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -40,8 +40,8 @@ import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.holders.DurationHolder; import org.apache.arrow.vector.holders.FixedSizeBinaryHolder; +import org.apache.arrow.vector.holders.NullableUuidHolder; import org.apache.arrow.vector.holders.TimeStampMilliTZHolder; -import org.apache.arrow.vector.holders.UuidHolder; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -1254,14 +1254,14 @@ public void testListVectorReaderForExtensionType() throws Exception { reader.setPosition(0); reader.next(); FieldReader uuidReader = reader.reader(); - UuidHolder holder = new UuidHolder(); + NullableUuidHolder holder = new NullableUuidHolder(); uuidReader.read(holder); - UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u1, actualUuid); reader.next(); uuidReader = reader.reader(); uuidReader.read(holder); - actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u2, actualUuid); } } @@ -1294,14 +1294,14 @@ public void testCopyFromForExtensionType() throws Exception { reader.setPosition(0); reader.next(); FieldReader uuidReader = reader.reader(); - UuidHolder holder = new UuidHolder(); + NullableUuidHolder holder = new NullableUuidHolder(); uuidReader.read(holder); - UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u1, actualUuid); reader.next(); uuidReader = reader.reader(); uuidReader.read(holder); - actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u2, actualUuid); } } @@ -1350,14 +1350,14 @@ public void testCopyValueSafeForExtensionType() throws Exception { assertTrue(reader.isSet(), "first list shouldn't be null"); reader.next(); FieldReader uuidReader = reader.reader(); - UuidHolder holder = new UuidHolder(); + NullableUuidHolder holder = new NullableUuidHolder(); uuidReader.read(holder); - UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u1, actualUuid); reader.next(); uuidReader = reader.reader(); uuidReader.read(holder); - actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u2, actualUuid); // Verify second list @@ -1366,12 +1366,12 @@ public void testCopyValueSafeForExtensionType() throws Exception { reader.next(); uuidReader = reader.reader(); uuidReader.read(holder); - actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u3, actualUuid); reader.next(); uuidReader = reader.reader(); uuidReader.read(holder); - actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u4, actualUuid); reader.next(); uuidReader = reader.reader(); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java index bfac1237a4..274d2973bd 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java @@ -42,7 +42,7 @@ import org.apache.arrow.vector.complex.writer.FieldWriter; import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.holders.FixedSizeBinaryHolder; -import org.apache.arrow.vector.holders.UuidHolder; +import org.apache.arrow.vector.holders.NullableUuidHolder; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; @@ -1299,14 +1299,14 @@ public void testMapVectorWithExtensionType() throws Exception { mapReader.setPosition(0); mapReader.next(); FieldReader uuidReader = mapReader.value(); - UuidHolder holder = new UuidHolder(); + NullableUuidHolder holder = new NullableUuidHolder(); uuidReader.read(holder); - UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u1, actualUuid); mapReader.next(); uuidReader = mapReader.value(); uuidReader.read(holder); - actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u2, actualUuid); } } @@ -1341,14 +1341,14 @@ public void testCopyFromForExtensionType() throws Exception { mapReader.setPosition(0); mapReader.next(); FieldReader uuidReader = mapReader.value(); - UuidHolder holder = new UuidHolder(); + NullableUuidHolder holder = new NullableUuidHolder(); uuidReader.read(holder); - UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u1, actualUuid); mapReader.next(); uuidReader = mapReader.value(); uuidReader.read(holder); - actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u2, actualUuid); } } @@ -1626,9 +1626,9 @@ public void testMapWithUuidKeyAndListUuidValue() throws Exception { // Read first entry mapReader.next(); FieldReader keyReader = mapReader.key(); - UuidHolder keyHolder = new UuidHolder(); + NullableUuidHolder keyHolder = new NullableUuidHolder(); keyReader.read(keyHolder); - UUID actualKey = UuidUtility.uuidFromArrowBuf(keyHolder.buffer, 0); + UUID actualKey = UuidUtility.uuidFromArrowBuf(keyHolder.buffer, keyHolder.start); assertEquals(key1, actualKey); FieldReader valueReader = mapReader.value(); @@ -1648,7 +1648,7 @@ public void testMapWithUuidKeyAndListUuidValue() throws Exception { mapReader.next(); keyReader = mapReader.key(); keyReader.read(keyHolder); - actualKey = UuidUtility.uuidFromArrowBuf(keyHolder.buffer, 0); + actualKey = UuidUtility.uuidFromArrowBuf(keyHolder.buffer, keyHolder.start); assertEquals(key2, actualKey); valueReader = mapReader.value(); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestUuidType.java b/vector/src/test/java/org/apache/arrow/vector/TestUuidType.java index acf9dd6868..99045d1cba 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestUuidType.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestUuidType.java @@ -233,7 +233,8 @@ void testVectorByteArrayOperations() { // Verify the bytes match byte[] actualBytes = new byte[UuidType.UUID_BYTE_WIDTH]; - uuidVector.get(0).getBytes(0, actualBytes); + int offset = uuidVector.getStartOffset(0); + uuidVector.getDataBuffer().getBytes(offset, actualBytes); assertArrayEquals(uuidBytes, actualBytes); } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestUuidVector.java b/vector/src/test/java/org/apache/arrow/vector/TestUuidVector.java index a3690461cf..b5dd12d89c 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestUuidVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestUuidVector.java @@ -27,6 +27,7 @@ import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.complex.impl.NullableUuidHolderReaderImpl; import org.apache.arrow.vector.complex.impl.UuidReaderImpl; import org.apache.arrow.vector.complex.impl.UuidWriterImpl; import org.apache.arrow.vector.extension.UuidType; @@ -136,8 +137,8 @@ void testWriteExtensionWithUnsupportedType() throws Exception { IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> writer.writeExtension("invalid-type")); - assertEquals( - "Unsupported value type for UUID: class java.lang.String", exception.getMessage()); + assertTrue( + exception.getMessage().contains("Unsupported value type for UUID: java.lang.String")); } } @@ -235,9 +236,9 @@ void testReaderCopyAsValueExtensionVector() throws Exception { UuidReaderImpl reader = (UuidReaderImpl) vectorForRead.getReader(); reader.copyAsValue(writer); UuidReaderImpl reader2 = (UuidReaderImpl) vector.getReader(); - UuidHolder holder = new UuidHolder(); + NullableUuidHolder holder = new NullableUuidHolder(); reader2.read(0, holder); - UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(uuid, actualUuid); } } @@ -252,10 +253,10 @@ void testReaderReadWithUuidHolder() throws Exception { UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); reader.setPosition(0); - UuidHolder holder = new UuidHolder(); + NullableUuidHolder holder = new NullableUuidHolder(); reader.read(holder); - UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(uuid, actualUuid); assertEquals(1, holder.isSet); } @@ -274,7 +275,7 @@ void testReaderReadWithNullableUuidHolder() throws Exception { NullableUuidHolder holder = new NullableUuidHolder(); reader.read(holder); - UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(uuid, actualUuid); assertEquals(1, holder.isSet); } @@ -310,10 +311,10 @@ void testReaderReadWithArrayIndexUuidHolder() throws Exception { UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); - UuidHolder holder = new UuidHolder(); + NullableUuidHolder holder = new NullableUuidHolder(); reader.read(1, holder); - UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(uuid2, actualUuid); assertEquals(1, holder.isSet); } @@ -334,7 +335,7 @@ void testReaderReadWithArrayIndexNullableUuidHolder() throws Exception { NullableUuidHolder holder1 = new NullableUuidHolder(); reader.read(0, holder1); - assertEquals(uuid1, UuidUtility.uuidFromArrowBuf(holder1.buffer, 0)); + assertEquals(uuid1, UuidUtility.uuidFromArrowBuf(holder1.buffer, holder1.start)); assertEquals(1, holder1.isSet); NullableUuidHolder holder2 = new NullableUuidHolder(); @@ -343,7 +344,7 @@ void testReaderReadWithArrayIndexNullableUuidHolder() throws Exception { NullableUuidHolder holder3 = new NullableUuidHolder(); reader.read(2, holder3); - assertEquals(uuid2, UuidUtility.uuidFromArrowBuf(holder3.buffer, 0)); + assertEquals(uuid2, UuidUtility.uuidFromArrowBuf(holder3.buffer, holder3.start)); assertEquals(1, holder3.isSet); } } @@ -374,31 +375,6 @@ public ArrowType type() { } } - @Test - void testReaderReadWithArrayIndexUnsupportedHolder() throws Exception { - try (UuidVector vector = new UuidVector("test", allocator)) { - UUID uuid = UUID.randomUUID(); - vector.setSafe(0, uuid); - vector.setValueCount(1); - - UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); - - // Create a mock unsupported holder - ExtensionHolder unsupportedHolder = - new ExtensionHolder() { - @Override - public ArrowType type() { - return null; - } - }; - - IllegalArgumentException exception = - assertThrows(IllegalArgumentException.class, () -> reader.read(0, unsupportedHolder)); - - assertTrue(exception.getMessage().contains("Unsupported holder type for UuidReader")); - } - } - @Test void testReaderIsSet() throws Exception { try (UuidVector vector = new UuidVector("test", allocator)) { @@ -461,4 +437,290 @@ void testReaderGetField() throws Exception { assertEquals("test", reader.getField().getName()); } } + + @Test + void testHolderStartOffsetWithMultipleValues() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid1 = UUID.randomUUID(); + UUID uuid2 = UUID.randomUUID(); + UUID uuid3 = UUID.randomUUID(); + + vector.setSafe(0, uuid1); + vector.setSafe(1, uuid2); + vector.setSafe(2, uuid3); + vector.setValueCount(3); + + // Test UuidHolder with different indices + NullableUuidHolder holder = new NullableUuidHolder(); + vector.get(0, holder); + assertEquals(0, holder.start); + assertEquals(uuid1, UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start)); + + vector.get(1, holder); + assertEquals(16, holder.start); // UUID_BYTE_WIDTH = 16 + assertEquals(uuid2, UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start)); + + vector.get(2, holder); + assertEquals(32, holder.start); // 2 * UUID_BYTE_WIDTH = 32 + assertEquals(uuid3, UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start)); + } + } + + @Test + void testNullableHolderStartOffsetWithMultipleValues() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid1 = UUID.randomUUID(); + UUID uuid2 = UUID.randomUUID(); + + vector.setSafe(0, uuid1); + vector.setNull(1); + vector.setSafe(2, uuid2); + vector.setValueCount(3); + + // Test NullableUuidHolder with different indices + NullableUuidHolder holder1 = new NullableUuidHolder(); + vector.get(0, holder1); + assertEquals(0, holder1.start); + assertEquals(1, holder1.isSet); + assertEquals(uuid1, UuidUtility.uuidFromArrowBuf(holder1.buffer, holder1.start)); + + NullableUuidHolder holder2 = new NullableUuidHolder(); + vector.get(1, holder2); + assertEquals(0, holder2.isSet); + + NullableUuidHolder holder3 = new NullableUuidHolder(); + vector.get(2, holder3); + assertEquals(32, holder3.start); // 2 * UUID_BYTE_WIDTH = 32 + assertEquals(1, holder3.isSet); + assertEquals(uuid2, UuidUtility.uuidFromArrowBuf(holder3.buffer, holder3.start)); + + // Verify all holders share the same buffer + assertEquals(holder1.buffer, holder3.buffer); + } + } + + @Test + void testSetFromHolderWithStartOffset() throws Exception { + try (UuidVector sourceVector = new UuidVector("source", allocator); + UuidVector targetVector = new UuidVector("target", allocator)) { + UUID uuid1 = UUID.randomUUID(); + UUID uuid2 = UUID.randomUUID(); + + sourceVector.setSafe(0, uuid1); + sourceVector.setSafe(1, uuid2); + sourceVector.setValueCount(3); + + // Get holder from index 1 (should have start = 16) + NullableUuidHolder holder = new NullableUuidHolder(); + sourceVector.get(1, holder); + assertEquals(16, holder.start); + + // Set target vector using holder with non-zero start offset + targetVector.setSafe(0, holder); + targetVector.setValueCount(1); + + // Verify the value was copied correctly + assertEquals(uuid2, targetVector.getObject(0)); + } + } + + @Test + void testSetFromNullableHolderWithStartOffset() throws Exception { + try (UuidVector sourceVector = new UuidVector("source", allocator); + UuidVector targetVector = new UuidVector("target", allocator)) { + UUID uuid1 = UUID.randomUUID(); + UUID uuid2 = UUID.randomUUID(); + + sourceVector.setSafe(0, uuid1); + sourceVector.setNull(1); + sourceVector.setSafe(2, uuid2); + sourceVector.setValueCount(3); + + // Get holder from index 2 (should have start = 32) + NullableUuidHolder holder = new NullableUuidHolder(); + sourceVector.get(2, holder); + assertEquals(32, holder.start); + assertEquals(1, holder.isSet); + + // Set target vector using holder with non-zero start offset + targetVector.setSafe(0, holder); + targetVector.setValueCount(1); + + // Verify the value was copied correctly + assertEquals(uuid2, targetVector.getObject(0)); + + // Test with null holder + NullableUuidHolder nullHolder = new NullableUuidHolder(); + sourceVector.get(1, nullHolder); + assertEquals(0, nullHolder.isSet); + + targetVector.setSafe(1, nullHolder); + targetVector.setValueCount(2); + assertTrue(targetVector.isNull(1)); + } + } + + @Test + void testGetStartOffset() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + vector.allocateNew(10); + + // Test getStartOffset for various indices + assertEquals(0, vector.getStartOffset(0)); + assertEquals(16, vector.getStartOffset(1)); + assertEquals(32, vector.getStartOffset(2)); + assertEquals(48, vector.getStartOffset(3)); + assertEquals(160, vector.getStartOffset(10)); + } + } + + @Test + void testReaderWithStartOffsetMultipleReads() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid1 = UUID.randomUUID(); + UUID uuid2 = UUID.randomUUID(); + UUID uuid3 = UUID.randomUUID(); + + vector.setSafe(0, uuid1); + vector.setSafe(1, uuid2); + vector.setSafe(2, uuid3); + vector.setValueCount(3); + + UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); + NullableUuidHolder holder = new NullableUuidHolder(); + + // Read from different positions and verify start offset + reader.read(0, holder); + assertEquals(0, holder.start); + assertEquals(uuid1, UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start)); + + reader.read(1, holder); + assertEquals(16, holder.start); + assertEquals(uuid2, UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start)); + + reader.read(2, holder); + assertEquals(32, holder.start); + assertEquals(uuid3, UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start)); + } + } + + @Test + void testWriterWithExtensionHolder() throws Exception { + try (UuidVector sourceVector = new UuidVector("source", allocator); + UuidVector targetVector = new UuidVector("target", allocator)) { + UUID uuid = UUID.randomUUID(); + sourceVector.setSafe(0, uuid); + sourceVector.setValueCount(1); + + // Get holder from source + NullableUuidHolder holder = new NullableUuidHolder(); + sourceVector.get(0, holder); + + // Write using UuidWriterImpl with ExtensionHolder + UuidWriterImpl writer = new UuidWriterImpl(targetVector); + writer.setPosition(0); + writer.writeExtension(holder); + + assertEquals(uuid, targetVector.getObject(0)); + } + } + + @Test + void testNullableUuidHolderReaderImpl() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid = UUID.randomUUID(); + vector.setSafe(0, uuid); + vector.setValueCount(1); + + // Get holder from vector + NullableUuidHolder sourceHolder = new NullableUuidHolder(); + vector.get(0, sourceHolder); + assertEquals(1, sourceHolder.isSet); + assertEquals(0, sourceHolder.start); + + // Create reader from holder + NullableUuidHolderReaderImpl reader = new NullableUuidHolderReaderImpl(sourceHolder); + assertTrue(reader.isSet()); + assertEquals(uuid, reader.readObject()); + + // Read into another holder + NullableUuidHolder targetHolder = new NullableUuidHolder(); + reader.read(targetHolder); + assertEquals(1, targetHolder.isSet); + assertEquals(0, targetHolder.start); + assertEquals(uuid, UuidUtility.uuidFromArrowBuf(targetHolder.buffer, targetHolder.start)); + } + } + + @Test + void testNullableUuidHolderReaderImplWithNull() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + vector.setNull(0); + vector.setValueCount(1); + + // Get null holder from vector + NullableUuidHolder sourceHolder = new NullableUuidHolder(); + vector.get(0, sourceHolder); + assertEquals(0, sourceHolder.isSet); + + // Create reader from null holder + NullableUuidHolderReaderImpl reader = new NullableUuidHolderReaderImpl(sourceHolder); + assertFalse(reader.isSet()); + assertNull(reader.readObject()); + + // Read into another holder + NullableUuidHolder targetHolder = new NullableUuidHolder(); + reader.read(targetHolder); + assertEquals(0, targetHolder.isSet); + } + } + + @Test + void testNullableUuidHolderReaderImplReadIntoUuidHolder() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid = UUID.randomUUID(); + vector.setSafe(0, uuid); + vector.setValueCount(1); + + // Get holder from vector + NullableUuidHolder sourceHolder = new NullableUuidHolder(); + vector.get(0, sourceHolder); + + // Create reader from holder + NullableUuidHolderReaderImpl reader = new NullableUuidHolderReaderImpl(sourceHolder); + + // Read into UuidHolder (non-nullable) + UuidHolder targetHolder = new UuidHolder(); + reader.read(targetHolder); + assertEquals(0, targetHolder.start); + assertEquals(uuid, UuidUtility.uuidFromArrowBuf(targetHolder.buffer, targetHolder.start)); + } + } + + @Test + void testNullableUuidHolderReaderImplWithNonZeroStart() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid1 = UUID.randomUUID(); + UUID uuid2 = UUID.randomUUID(); + vector.setSafe(0, uuid1); + vector.setSafe(1, uuid2); + vector.setValueCount(2); + + // Get holder from index 1 (start = 16) + NullableUuidHolder sourceHolder = new NullableUuidHolder(); + vector.get(1, sourceHolder); + assertEquals(1, sourceHolder.isSet); + assertEquals(16, sourceHolder.start); + + // Create reader from holder + NullableUuidHolderReaderImpl reader = new NullableUuidHolderReaderImpl(sourceHolder); + assertEquals(uuid2, reader.readObject()); + + // Read into another holder and verify start is preserved + NullableUuidHolder targetHolder = new NullableUuidHolder(); + reader.read(targetHolder); + assertEquals(16, targetHolder.start); + assertEquals(uuid2, UuidUtility.uuidFromArrowBuf(targetHolder.buffer, targetHolder.start)); + } + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index b131bf07e2..80d03cae6d 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -1169,7 +1169,7 @@ public void simpleUnion() throws Exception { } else if (i % 5 == 4) { NullableUuidHolder holder = new NullableUuidHolder(); unionReader.read(holder); - assertEquals(UuidUtility.uuidFromArrowBuf(holder.buffer, 0), uuid); + assertEquals(UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start), uuid); } else { assertEquals((float) i, unionReader.readFloat(), 1e-12); } @@ -2536,7 +2536,7 @@ public void extensionWriterReader() throws Exception { { FieldReader uuidReader = rootReader.reader("uuid1"); uuidReader.setPosition(0); - UuidHolder uuidHolder = new UuidHolder(); + NullableUuidHolder uuidHolder = new NullableUuidHolder(); uuidReader.read(uuidHolder); UUID actualUuid = UuidUtility.uuidFromArrowBuf(uuidHolder.buffer, 0); assertEquals(u1, actualUuid); From a1d83179cf6d3cce4660f6f0bf8e7f75867e87bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Tue, 20 Jan 2026 13:57:15 +0000 Subject: [PATCH 266/408] GH-929: Add UUID support in JDBC driver (#930) ## What's Changed This PR adds UUID support to the Arrow Flight SQL JDBC driver, enabling JDBC applications to work with UUID data types when connecting to Flight SQL servers that use Arrow's canonical `arrow.uuid` extension type. ### Key Implementation Details - Added `ArrowFlightJdbcUuidVectorAccessor` to handle reading UUID values from `UuidVector` - `getObject()` returns `java.util.UUID` directly - `getString()` returns the standard hyphenated UUID format (e.g., "550e8400-e29b-41d4-a716-446655440000") - `getBytes()` returns the 16-byte binary representation - Added `UuidAvaticaParameterConverter` to handle parameter binding for UUID columns - Supports binding `java.util.UUID` objects directly via `setObject()` - Supports binding UUID string representations via `setString()` - Supports binding 16-byte arrays via `setBytes()` - UUID extension type maps to `java.sql.Types.OTHER`. - Updated `SqlTypes` to recognize `UuidType` and return appropriate SQL type ID **Examples** ``` java try (Statement stmt = conn.createStatement(); ResultSet rs = stmt.executeQuery("SELECT id, session_id FROM sessions")) { while (rs.next()) { int id = rs.getInt("id"); // getObject() returns java.util.UUID directly UUID sessionId = rs.getObject("session_id", UUID.class); // getString() returns hyphenated format: "550e8400-e29b-41d4-a716-..." String sessionIdStr = rs.getString("session_id"); System.out.printf("ID: %d, UUID: %s%n", id, sessionId); } } // Use PreparedStatement to bind UUID parameters String sql = "SELECT * FROM sessions WHERE session_id = ?"; try (PreparedStatement pstmt = conn.prepareStatement(sql)) { UUID targetId = UUID.fromString("550e8400-e29b-41d4-a716-446655440000"); // Bind UUID directly with setObject() pstmt.setObject(1, targetId); // Or bind as string: pstmt.setString(1, targetId.toString()); try (ResultSet rs = pstmt.executeQuery()) { if (rs.next()) { System.out.println("Found: " + rs.getObject("session_id")); } } } ``` Closes #929. --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Joana Hrotko --- docs/source/jdbc.rst | 8 +- .../driver/jdbc/ArrowDatabaseMetadata.java | 5 + .../ArrowFlightJdbcAccessorFactory.java | 5 + .../ArrowFlightJdbcUuidVectorAccessor.java | 88 ++++++++ .../impl/UuidAvaticaParameterConverter.java | 103 ++++++++++ .../jdbc/utils/AvaticaParameterBinder.java | 14 ++ .../arrow/driver/jdbc/utils/ConvertUtils.java | 14 ++ .../arrow/driver/jdbc/utils/SqlTypes.java | 5 + .../arrow/driver/jdbc/ResultSetTest.java | 176 ++++++++++++++++ .../ArrowFlightJdbcAccessorFactoryTest.java | 13 ++ ...ArrowFlightJdbcUuidVectorAccessorTest.java | 188 ++++++++++++++++++ .../UuidAvaticaParameterConverterTest.java | 160 +++++++++++++++ .../jdbc/utils/CoreMockedSqlProducers.java | 122 ++++++++++++ .../utils/RootAllocatorTestExtension.java | 22 ++ .../arrow/driver/jdbc/utils/SqlTypesTest.java | 5 + 15 files changed, 927 insertions(+), 1 deletion(-) create mode 100644 flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/binary/ArrowFlightJdbcUuidVectorAccessor.java create mode 100644 flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/UuidAvaticaParameterConverter.java create mode 100644 flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/binary/ArrowFlightJdbcUuidVectorAccessorTest.java create mode 100644 flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/converter/impl/UuidAvaticaParameterConverterTest.java diff --git a/docs/source/jdbc.rst b/docs/source/jdbc.rst index c0477cb06d..a4c95dbf00 100644 --- a/docs/source/jdbc.rst +++ b/docs/source/jdbc.rst @@ -213,7 +213,8 @@ Type Mapping ------------ The Arrow to JDBC type mapping can be obtained at runtime via -a method on ColumnBinder. +a method on ColumnBinder. The Flight SQL JDBC driver follows the same +mapping, with additional support for the UUID extension type noted below. +----------------------------+----------------------------+-------+ | Arrow Type | JDBC Type | Notes | @@ -232,6 +233,8 @@ a method on ColumnBinder. +----------------------------+----------------------------+-------+ | FixedSizeBinary | BINARY (setBytes) | | +----------------------------+----------------------------+-------+ +| Uuid (extension) | OTHER (setObject) | \(3) | ++----------------------------+----------------------------+-------+ | Float32 | REAL (setFloat) | | +----------------------------+----------------------------+-------+ | Int8 | TINYINT (setByte) | | @@ -276,3 +279,6 @@ a method on ColumnBinder. `_, which will lead to the driver using the "default timezone" (that of the Java VM). +* \(3) For the Flight SQL JDBC driver, the Arrow UUID extension type + (``arrow.uuid``) maps to JDBC ``OTHER`` and is surfaced as + ``java.util.UUID`` values. diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java index 7185ddfe01..502270e1cd 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java @@ -75,10 +75,12 @@ import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.ipc.ReadChannel; import org.apache.arrow.vector.ipc.message.MessageSerializer; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ExtensionTypeRegistry; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.Text; @@ -164,6 +166,9 @@ public class ArrowDatabaseMetadata extends AvaticaDatabaseMetaData { LONGNVARCHAR, SqlSupportsConvert.SQL_CONVERT_LONGVARCHAR_VALUE); sqlTypesToFlightEnumConvertTypes.put(DATE, SqlSupportsConvert.SQL_CONVERT_DATE_VALUE); sqlTypesToFlightEnumConvertTypes.put(TIMESTAMP, SqlSupportsConvert.SQL_CONVERT_TIMESTAMP_VALUE); + + // Register the UUID extension type so it is always available for the driver + ExtensionTypeRegistry.register(UuidType.INSTANCE); } ArrowDatabaseMetadata(final AvaticaConnection connection) { diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactory.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactory.java index bbfe88a78a..8362eb7627 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactory.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactory.java @@ -19,6 +19,7 @@ import java.util.function.IntSupplier; import org.apache.arrow.driver.jdbc.accessor.impl.ArrowFlightJdbcNullVectorAccessor; import org.apache.arrow.driver.jdbc.accessor.impl.binary.ArrowFlightJdbcBinaryVectorAccessor; +import org.apache.arrow.driver.jdbc.accessor.impl.binary.ArrowFlightJdbcUuidVectorAccessor; import org.apache.arrow.driver.jdbc.accessor.impl.calendar.ArrowFlightJdbcDateVectorAccessor; import org.apache.arrow.driver.jdbc.accessor.impl.calendar.ArrowFlightJdbcDurationVectorAccessor; import org.apache.arrow.driver.jdbc.accessor.impl.calendar.ArrowFlightJdbcIntervalVectorAccessor; @@ -65,6 +66,7 @@ import org.apache.arrow.vector.UInt2Vector; import org.apache.arrow.vector.UInt4Vector; import org.apache.arrow.vector.UInt8Vector; +import org.apache.arrow.vector.UuidVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; @@ -138,6 +140,9 @@ public static ArrowFlightJdbcAccessor createAccessor( } else if (vector instanceof LargeVarBinaryVector) { return new ArrowFlightJdbcBinaryVectorAccessor( (LargeVarBinaryVector) vector, getCurrentRow, setCursorWasNull); + } else if (vector instanceof UuidVector) { + return new ArrowFlightJdbcUuidVectorAccessor( + (UuidVector) vector, getCurrentRow, setCursorWasNull); } else if (vector instanceof FixedSizeBinaryVector) { return new ArrowFlightJdbcBinaryVectorAccessor( (FixedSizeBinaryVector) vector, getCurrentRow, setCursorWasNull); diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/binary/ArrowFlightJdbcUuidVectorAccessor.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/binary/ArrowFlightJdbcUuidVectorAccessor.java new file mode 100644 index 0000000000..4bdbcbb63d --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/binary/ArrowFlightJdbcUuidVectorAccessor.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.accessor.impl.binary; + +import java.util.UUID; +import java.util.function.IntSupplier; +import org.apache.arrow.driver.jdbc.accessor.ArrowFlightJdbcAccessor; +import org.apache.arrow.driver.jdbc.accessor.ArrowFlightJdbcAccessorFactory; +import org.apache.arrow.vector.UuidVector; +import org.apache.arrow.vector.util.UuidUtility; + +/** + * Accessor for the Arrow UUID extension type ({@link UuidVector}). + * + *

This accessor provides JDBC-compatible access to UUID values stored in Arrow's canonical UUID + * extension type ('arrow.uuid'). It follows PostgreSQL JDBC driver conventions: + * + *

    + *
  • {@link #getObject()} returns {@link java.util.UUID} + *
  • {@link #getString()} returns the hyphenated string format (e.g., + * "550e8400-e29b-41d4-a716-446655440000") + *
  • {@link #getBytes()} returns the 16-byte binary representation + *
+ */ +public class ArrowFlightJdbcUuidVectorAccessor extends ArrowFlightJdbcAccessor { + + private final UuidVector vector; + + /** + * Creates a new accessor for a UUID vector. + * + * @param vector the UUID vector to access + * @param currentRowSupplier supplier for the current row index + * @param setCursorWasNull consumer to set the wasNull flag + */ + public ArrowFlightJdbcUuidVectorAccessor( + UuidVector vector, + IntSupplier currentRowSupplier, + ArrowFlightJdbcAccessorFactory.WasNullConsumer setCursorWasNull) { + super(currentRowSupplier, setCursorWasNull); + this.vector = vector; + } + + @Override + public Object getObject() { + UUID uuid = vector.getObject(getCurrentRow()); + this.wasNull = uuid == null; + this.wasNullConsumer.setWasNull(this.wasNull); + return uuid; + } + + @Override + public Class getObjectClass() { + return UUID.class; + } + + @Override + public String getString() { + UUID uuid = (UUID) getObject(); + if (uuid == null) { + return null; + } + return uuid.toString(); + } + + @Override + public byte[] getBytes() { + UUID uuid = (UUID) getObject(); + if (uuid == null) { + return null; + } + return UuidUtility.getBytesFromUUID(uuid); + } +} diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/UuidAvaticaParameterConverter.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/UuidAvaticaParameterConverter.java new file mode 100644 index 0000000000..b2157890cf --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/UuidAvaticaParameterConverter.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.converter.impl; + +import static org.apache.arrow.driver.jdbc.utils.SqlTypes.getSqlTypeIdFromArrowType; +import static org.apache.arrow.driver.jdbc.utils.SqlTypes.getSqlTypeNameFromArrowType; + +import java.nio.ByteBuffer; +import java.util.UUID; +import org.apache.arrow.driver.jdbc.converter.AvaticaParameterConverter; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.UuidVector; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.util.UuidUtility; +import org.apache.calcite.avatica.AvaticaParameter; +import org.apache.calcite.avatica.remote.TypedValue; +import org.apache.calcite.avatica.util.ByteString; + +/** + * AvaticaParameterConverter for UUID Arrow extension type. + * + *

Handles conversion of UUID values from JDBC parameters to Arrow's UUID extension type. Accepts + * both {@link UUID} objects and String representations of UUIDs. + */ +public class UuidAvaticaParameterConverter implements AvaticaParameterConverter { + + public UuidAvaticaParameterConverter() {} + + @Override + public boolean bindParameter(FieldVector vector, TypedValue typedValue, int index) { + if (!(vector instanceof UuidVector)) { + return false; + } + + UuidVector uuidVector = (UuidVector) vector; + Object value = typedValue.toJdbc(null); + + if (value == null) { + uuidVector.setNull(index); + return true; + } + + UUID uuid; + if (value instanceof UUID) { + uuid = (UUID) value; + } else if (value instanceof String) { + uuid = UUID.fromString((String) value); + } else if (value instanceof byte[]) { + byte[] bytes = (byte[]) value; + if (bytes.length != 16) { + throw new IllegalArgumentException("UUID byte array must be 16 bytes, got " + bytes.length); + } + uuid = uuidFromBytes(bytes); + } else if (value instanceof ByteString) { + byte[] bytes = ((ByteString) value).getBytes(); + if (bytes.length != 16) { + throw new IllegalArgumentException("UUID byte array must be 16 bytes, got " + bytes.length); + } + uuid = uuidFromBytes(bytes); + } else { + throw new IllegalArgumentException( + "Cannot convert " + value.getClass().getName() + " to UUID"); + } + + uuidVector.setSafe(index, UuidUtility.getBytesFromUUID(uuid)); + return true; + } + + @Override + public AvaticaParameter createParameter(Field field) { + final String name = field.getName(); + final int jdbcType = getSqlTypeIdFromArrowType(field.getType()); + final String typeName = getSqlTypeNameFromArrowType(field.getType()); + final String className = UUID.class.getCanonicalName(); + return new AvaticaParameter(false, 0, 0, jdbcType, typeName, className, name); + } + + private static UUID uuidFromBytes(byte[] bytes) { + final long mostSignificantBits; + final long leastSignificantBits; + ByteBuffer bb = ByteBuffer.wrap(bytes); + // Reads the first eight bytes + mostSignificantBits = bb.getLong(); + // Reads the first eight bytes at this buffer's current + leastSignificantBits = bb.getLong(); + + return new UUID(mostSignificantBits, leastSignificantBits); + } +} diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java index 8c98ee4077..8f40d6698e 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java @@ -41,10 +41,14 @@ import org.apache.arrow.driver.jdbc.converter.impl.UnionAvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.Utf8AvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.Utf8ViewAvaticaParameterConverter; +import org.apache.arrow.driver.jdbc.converter.impl.UuidAvaticaParameterConverter; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeVisitor; +import org.apache.arrow.vector.types.pojo.ArrowType.ExtensionType; import org.apache.calcite.avatica.remote.TypedValue; import org.checkerframework.checker.nullness.qual.Nullable; @@ -290,5 +294,15 @@ public Boolean visit(ArrowType.RunEndEncoded type) { throw new UnsupportedOperationException( "No Avatica parameter binder implemented for type " + type); } + + @Override + public Boolean visit(ExtensionType type) { + if (type instanceof UuidType) { + return new UuidAvaticaParameterConverter().bindParameter(vector, typedValue, index); + } + + // fallback to default implementation + return ArrowTypeVisitor.super.visit(type); + } } } diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java index 5dd4c69c73..dd51ee5361 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java @@ -43,8 +43,12 @@ import org.apache.arrow.driver.jdbc.converter.impl.UnionAvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.Utf8AvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.Utf8ViewAvaticaParameterConverter; +import org.apache.arrow.driver.jdbc.converter.impl.UuidAvaticaParameterConverter; import org.apache.arrow.flight.sql.FlightSqlColumnMetadata; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeVisitor; +import org.apache.arrow.vector.types.pojo.ArrowType.ExtensionType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.calcite.avatica.AvaticaParameter; import org.apache.calcite.avatica.ColumnMetaData; @@ -294,5 +298,15 @@ public AvaticaParameter visit(ArrowType.RunEndEncoded type) { throw new UnsupportedOperationException( "No Avatica parameter binder implemented for type " + type); } + + @Override + public AvaticaParameter visit(ExtensionType type) { + if (type instanceof UuidType) { + return new UuidAvaticaParameterConverter().createParameter(field); + } + + // fallback to default implementation + return ArrowTypeVisitor.super.visit(type); + } } } diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/SqlTypes.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/SqlTypes.java index 5ba3957f8b..7982d5bc73 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/SqlTypes.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/SqlTypes.java @@ -20,11 +20,13 @@ import java.sql.Types; import java.util.HashMap; import java.util.Map; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.types.pojo.ArrowType; /** SQL Types utility functions. */ public class SqlTypes { + private static final Map typeIdToName = new HashMap<>(); static { @@ -110,6 +112,9 @@ public static int getSqlTypeIdFromArrowType(ArrowType arrowType) { case BinaryView: return Types.VARBINARY; case FixedSizeBinary: + if (arrowType instanceof UuidType) { + return Types.OTHER; + } return Types.BINARY; case LargeBinary: return Types.LONGVARBINARY; diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java index 569b5495fe..3a5a39be3d 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java @@ -22,8 +22,10 @@ import static org.hamcrest.CoreMatchers.allOf; import static org.hamcrest.CoreMatchers.anyOf; import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.*; @@ -31,7 +33,9 @@ import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.PreparedStatement; import java.sql.ResultSet; +import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.SQLTimeoutException; import java.sql.Statement; @@ -42,6 +46,7 @@ import java.util.List; import java.util.Random; import java.util.Set; +import java.util.UUID; import java.util.concurrent.CountDownLatch; import org.apache.arrow.driver.jdbc.utils.CoreMockedSqlProducers; import org.apache.arrow.driver.jdbc.utils.FallbackFlightSqlProducer; @@ -61,6 +66,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.arrow.vector.util.UuidUtility; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -795,4 +801,174 @@ public void testResultSetAppMetadata() throws Exception { "foo".getBytes(StandardCharsets.UTF_8)); } } + + @Test + public void testSelectQueryWithUuidColumn() throws SQLException { + // Expectations + final int expectedRowCount = 4; + final UUID[] expectedUuids = + new UUID[] { + CoreMockedSqlProducers.UUID_1, + CoreMockedSqlProducers.UUID_2, + CoreMockedSqlProducers.UUID_3, + null + }; + + final Integer[] expectedIds = new Integer[] {1, 2, 3, 4}; + + final List actualUuids = new ArrayList<>(expectedRowCount); + final List actualIds = new ArrayList<>(expectedRowCount); + + // Query + int actualRowCount = 0; + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(CoreMockedSqlProducers.UUID_SQL_CMD)) { + for (; resultSet.next(); actualRowCount++) { + actualIds.add((Integer) resultSet.getObject("id")); + actualUuids.add((UUID) resultSet.getObject("uuid_col")); + } + } + + // Assertions + int finalActualRowCount = actualRowCount; + assertAll( + "UUID ResultSet values are as expected", + () -> assertThat(finalActualRowCount, is(equalTo(expectedRowCount))), + () -> assertThat(actualIds.toArray(new Integer[0]), is(expectedIds)), + () -> assertThat(actualUuids.toArray(new UUID[0]), is(expectedUuids))); + } + + @Test + public void testGetObjectReturnsUuid() throws SQLException { + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(CoreMockedSqlProducers.UUID_SQL_CMD)) { + resultSet.next(); + Object result = resultSet.getObject("uuid_col"); + assertThat(result, instanceOf(UUID.class)); + assertThat(result, is(CoreMockedSqlProducers.UUID_1)); + } + } + + @Test + public void testGetObjectByIndexReturnsUuid() throws SQLException { + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(CoreMockedSqlProducers.UUID_SQL_CMD)) { + resultSet.next(); + Object result = resultSet.getObject(2); + assertThat(result, instanceOf(UUID.class)); + assertThat(result, is(CoreMockedSqlProducers.UUID_1)); + } + } + + @Test + public void testGetStringReturnsHyphenatedFormat() throws SQLException { + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(CoreMockedSqlProducers.UUID_SQL_CMD)) { + resultSet.next(); + String result = resultSet.getString("uuid_col"); + assertThat(result, is(CoreMockedSqlProducers.UUID_1.toString())); + } + } + + @Test + public void testGetBytesReturns16ByteArray() throws SQLException { + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(CoreMockedSqlProducers.UUID_SQL_CMD)) { + resultSet.next(); + byte[] result = resultSet.getBytes("uuid_col"); + assertThat(result.length, is(16)); + assertThat(result, is(UuidUtility.getBytesFromUUID(CoreMockedSqlProducers.UUID_1))); + } + } + + @Test + public void testNullUuidHandling() throws SQLException { + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(CoreMockedSqlProducers.UUID_SQL_CMD)) { + // Skip to row 4 which has NULL UUID + resultSet.next(); // row 1 + resultSet.next(); // row 2 + resultSet.next(); // row 3 + resultSet.next(); // row 4 (NULL UUID) + + Object objResult = resultSet.getObject("uuid_col"); + assertThat(objResult, nullValue()); + assertThat(resultSet.wasNull(), is(true)); + + String strResult = resultSet.getString("uuid_col"); + assertThat(strResult, nullValue()); + assertThat(resultSet.wasNull(), is(true)); + + byte[] bytesResult = resultSet.getBytes("uuid_col"); + assertThat(bytesResult, nullValue()); + assertThat(resultSet.wasNull(), is(true)); + } + } + + @Test + public void testMultipleUuidRows() throws SQLException { + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(CoreMockedSqlProducers.UUID_SQL_CMD)) { + resultSet.next(); + assertThat(resultSet.getObject("uuid_col"), is(CoreMockedSqlProducers.UUID_1)); + + resultSet.next(); + assertThat(resultSet.getObject("uuid_col"), is(CoreMockedSqlProducers.UUID_2)); + + resultSet.next(); + assertThat(resultSet.getObject("uuid_col"), is(CoreMockedSqlProducers.UUID_3)); + + resultSet.next(); + assertThat(resultSet.getObject("uuid_col"), nullValue()); + } + } + + @Test + public void testUuidExtensionTypeInSchema() throws SQLException { + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(CoreMockedSqlProducers.UUID_SQL_CMD)) { + ResultSetMetaData metaData = resultSet.getMetaData(); + + assertThat(metaData.getColumnCount(), is(2)); + assertThat(metaData.getColumnName(1), is("id")); + assertThat(metaData.getColumnName(2), is("uuid_col")); + + assertThat(metaData.getColumnType(2), is(java.sql.Types.OTHER)); + } + } + + @Test + public void testPreparedStatementWithUuidParameter() throws SQLException { + try (PreparedStatement pstmt = + connection.prepareStatement(CoreMockedSqlProducers.UUID_PREPARED_SELECT_SQL_CMD)) { + pstmt.setObject(1, CoreMockedSqlProducers.UUID_1); + try (ResultSet rs = pstmt.executeQuery()) { + rs.next(); + assertThat(rs.getObject("uuid_col"), is(CoreMockedSqlProducers.UUID_1)); + } + } + } + + @Test + public void testPreparedStatementWithUuidStringParameter() throws SQLException { + try (PreparedStatement pstmt = + connection.prepareStatement(CoreMockedSqlProducers.UUID_PREPARED_SELECT_SQL_CMD)) { + pstmt.setString(1, CoreMockedSqlProducers.UUID_1.toString()); + try (ResultSet rs = pstmt.executeQuery()) { + rs.next(); + assertThat(rs.getObject("uuid_col"), is(CoreMockedSqlProducers.UUID_1)); + } + } + } + + @Test + public void testPreparedStatementUpdateWithUuid() throws SQLException { + try (PreparedStatement pstmt = + connection.prepareStatement(CoreMockedSqlProducers.UUID_PREPARED_UPDATE_SQL_CMD)) { + pstmt.setObject(1, CoreMockedSqlProducers.UUID_3); + pstmt.setInt(2, 1); + int updated = pstmt.executeUpdate(); + assertThat(updated, is(1)); + } + } } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactoryTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactoryTest.java index 8b39041f0c..1fbd2f86a9 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactoryTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactoryTest.java @@ -16,10 +16,12 @@ */ package org.apache.arrow.driver.jdbc.accessor; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.function.IntSupplier; import org.apache.arrow.driver.jdbc.accessor.impl.binary.ArrowFlightJdbcBinaryVectorAccessor; +import org.apache.arrow.driver.jdbc.accessor.impl.binary.ArrowFlightJdbcUuidVectorAccessor; import org.apache.arrow.driver.jdbc.accessor.impl.calendar.ArrowFlightJdbcDateVectorAccessor; import org.apache.arrow.driver.jdbc.accessor.impl.calendar.ArrowFlightJdbcDurationVectorAccessor; import org.apache.arrow.driver.jdbc.accessor.impl.calendar.ArrowFlightJdbcIntervalVectorAccessor; @@ -497,4 +499,15 @@ public void createAccessorForMapVector() { assertTrue(accessor instanceof ArrowFlightJdbcMapVectorAccessor); } } + + @Test + public void createAccessorForUuidVector() { + try (ValueVector valueVector = rootAllocatorTestExtension.createUuidVector()) { + ArrowFlightJdbcAccessor accessor = + ArrowFlightJdbcAccessorFactory.createAccessor( + valueVector, GET_CURRENT_ROW, (boolean wasNull) -> {}); + + assertInstanceOf(ArrowFlightJdbcUuidVectorAccessor.class, accessor); + } + } } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/binary/ArrowFlightJdbcUuidVectorAccessorTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/binary/ArrowFlightJdbcUuidVectorAccessorTest.java new file mode 100644 index 0000000000..b7f341240c --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/binary/ArrowFlightJdbcUuidVectorAccessorTest.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.accessor.impl.binary; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.MatcherAssert.assertThat; + +import java.util.UUID; +import org.apache.arrow.driver.jdbc.accessor.ArrowFlightJdbcAccessorFactory; +import org.apache.arrow.driver.jdbc.utils.RootAllocatorTestExtension; +import org.apache.arrow.vector.UuidVector; +import org.apache.arrow.vector.util.UuidUtility; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * Tests for {@link ArrowFlightJdbcUuidVectorAccessor}. + * + *

Verifies that the accessor correctly handles UUID values from Arrow's UUID extension type, + * following PostgreSQL JDBC driver conventions. + */ +public class ArrowFlightJdbcUuidVectorAccessorTest { + + @RegisterExtension + public static RootAllocatorTestExtension rootAllocatorTestExtension = + new RootAllocatorTestExtension(); + + private static final UUID UUID_1 = UUID.fromString("550e8400-e29b-41d4-a716-446655440000"); + private static final UUID UUID_2 = UUID.fromString("6ba7b810-9dad-11d1-80b4-00c04fd430c8"); + private static final UUID UUID_3 = UUID.fromString("f47ac10b-58cc-4372-a567-0e02b2c3d479"); + + private UuidVector vector; + private ArrowFlightJdbcUuidVectorAccessor accessor; + private boolean wasNullCalled; + private boolean wasNullValue; + + @BeforeEach + public void setUp() { + vector = rootAllocatorTestExtension.createUuidVector(); + wasNullCalled = false; + wasNullValue = false; + ArrowFlightJdbcAccessorFactory.WasNullConsumer wasNullConsumer = + (wasNull) -> { + wasNullCalled = true; + wasNullValue = wasNull; + }; + accessor = new ArrowFlightJdbcUuidVectorAccessor(vector, () -> 0, wasNullConsumer); + } + + @AfterEach + public void tearDown() { + vector.close(); + } + + @Test + public void testGetObjectReturnsUuid() { + accessor = new ArrowFlightJdbcUuidVectorAccessor(vector, () -> 0, (wasNull) -> {}); + Object result = accessor.getObject(); + assertThat(result, is(UUID_1)); + assertThat(accessor.wasNull(), is(false)); + } + + @Test + public void testGetObjectReturnsCorrectUuidForEachRow() { + accessor = new ArrowFlightJdbcUuidVectorAccessor(vector, () -> 0, (wasNull) -> {}); + assertThat(accessor.getObject(), is(UUID_1)); + + accessor = new ArrowFlightJdbcUuidVectorAccessor(vector, () -> 1, (wasNull) -> {}); + assertThat(accessor.getObject(), is(UUID_2)); + + accessor = new ArrowFlightJdbcUuidVectorAccessor(vector, () -> 2, (wasNull) -> {}); + assertThat(accessor.getObject(), is(UUID_3)); + } + + @Test + public void testGetObjectReturnsNullForNullValue() { + vector.reset(); + vector.allocateNew(1); + vector.setNull(0); + vector.setValueCount(1); + + accessor = new ArrowFlightJdbcUuidVectorAccessor(vector, () -> 0, (wasNull) -> {}); + Object result = accessor.getObject(); + assertThat(result, nullValue()); + assertThat(accessor.wasNull(), is(true)); + } + + @Test + public void testGetObjectClassReturnsUuidClass() { + assertThat(accessor.getObjectClass(), equalTo(UUID.class)); + } + + @Test + public void testGetStringReturnsHyphenatedFormat() { + accessor = new ArrowFlightJdbcUuidVectorAccessor(vector, () -> 0, (wasNull) -> {}); + String result = accessor.getString(); + assertThat(result, is("550e8400-e29b-41d4-a716-446655440000")); + assertThat(accessor.wasNull(), is(false)); + } + + @Test + public void testGetStringReturnsNullForNullValue() { + vector.reset(); + vector.allocateNew(1); + vector.setNull(0); + vector.setValueCount(1); + + accessor = new ArrowFlightJdbcUuidVectorAccessor(vector, () -> 0, (wasNull) -> {}); + String result = accessor.getString(); + assertThat(result, nullValue()); + assertThat(accessor.wasNull(), is(true)); + } + + @Test + public void testGetBytesReturns16ByteArray() { + accessor = new ArrowFlightJdbcUuidVectorAccessor(vector, () -> 0, (wasNull) -> {}); + byte[] result = accessor.getBytes(); + assertThat(result.length, is(16)); + assertThat(result, is(UuidUtility.getBytesFromUUID(UUID_1))); + assertThat(accessor.wasNull(), is(false)); + } + + @Test + public void testGetBytesReturnsNullForNullValue() { + vector.reset(); + vector.allocateNew(1); + vector.setNull(0); + vector.setValueCount(1); + + accessor = new ArrowFlightJdbcUuidVectorAccessor(vector, () -> 0, (wasNull) -> {}); + byte[] result = accessor.getBytes(); + assertThat(result, nullValue()); + assertThat(accessor.wasNull(), is(true)); + } + + @Test + public void testWasNullConsumerIsCalled() { + accessor = + new ArrowFlightJdbcUuidVectorAccessor( + vector, + () -> 0, + (wasNull) -> { + wasNullCalled = true; + wasNullValue = wasNull; + }); + accessor.getObject(); + assertThat(wasNullCalled, is(true)); + assertThat(wasNullValue, is(false)); + } + + @Test + public void testWasNullConsumerIsCalledWithTrueForNull() { + vector.reset(); + vector.allocateNew(1); + vector.setNull(0); + vector.setValueCount(1); + + accessor = + new ArrowFlightJdbcUuidVectorAccessor( + vector, + () -> 0, + (wasNull) -> { + wasNullCalled = true; + wasNullValue = wasNull; + }); + accessor.getObject(); + assertThat(wasNullCalled, is(true)); + assertThat(wasNullValue, is(true)); + } +} diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/converter/impl/UuidAvaticaParameterConverterTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/converter/impl/UuidAvaticaParameterConverterTest.java new file mode 100644 index 0000000000..07751f0abc --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/converter/impl/UuidAvaticaParameterConverterTest.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.converter.impl; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.sql.Types; +import java.util.UUID; +import org.apache.arrow.driver.jdbc.utils.RootAllocatorTestExtension; +import org.apache.arrow.vector.UuidVector; +import org.apache.arrow.vector.extension.UuidType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.UuidUtility; +import org.apache.calcite.avatica.AvaticaParameter; +import org.apache.calcite.avatica.ColumnMetaData; +import org.apache.calcite.avatica.remote.TypedValue; +import org.apache.calcite.avatica.util.ByteString; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * Tests for {@link UuidAvaticaParameterConverter}. + * + *

Verifies that the converter correctly handles UUID parameter binding from JDBC to Arrow's UUID + * extension type. + */ +public class UuidAvaticaParameterConverterTest { + + @RegisterExtension + public static RootAllocatorTestExtension rootAllocatorTestExtension = + new RootAllocatorTestExtension(); + + private static final UUID TEST_UUID = UUID.fromString("550e8400-e29b-41d4-a716-446655440000"); + + private UuidVector vector; + private UuidAvaticaParameterConverter converter; + + @BeforeEach + public void setUp() { + vector = new UuidVector("uuid_param", rootAllocatorTestExtension.getRootAllocator()); + vector.allocateNew(5); + converter = new UuidAvaticaParameterConverter(); + } + + @AfterEach + public void tearDown() { + vector.close(); + } + + @Test + public void testBindParameterWithUuidObject() { + TypedValue typedValue = TypedValue.ofLocal(ColumnMetaData.Rep.OBJECT, TEST_UUID); + + boolean result = converter.bindParameter(vector, typedValue, 0); + + assertTrue(result); + assertThat(vector.getObject(0), is(TEST_UUID)); + } + + @Test + public void testBindParameterWithUuidString() { + String uuidString = "550e8400-e29b-41d4-a716-446655440000"; + TypedValue typedValue = TypedValue.ofLocal(ColumnMetaData.Rep.STRING, uuidString); + + boolean result = converter.bindParameter(vector, typedValue, 0); + + assertTrue(result); + assertThat(vector.getObject(0), is(TEST_UUID)); + } + + @Test + public void testBindParameterWithByteArray() { + byte[] uuidBytes = UuidUtility.getBytesFromUUID(TEST_UUID); + ByteString byteString = new ByteString(uuidBytes); + TypedValue typedValue = TypedValue.ofLocal(ColumnMetaData.Rep.BYTE_STRING, byteString); + + boolean result = converter.bindParameter(vector, typedValue, 0); + + assertTrue(result); + assertThat(vector.getObject(0), is(TEST_UUID)); + } + + @Test + public void testBindParameterWithNullValue() { + TypedValue typedValue = TypedValue.ofLocal(ColumnMetaData.Rep.OBJECT, null); + + boolean result = converter.bindParameter(vector, typedValue, 0); + + assertTrue(result); + assertTrue(vector.isNull(0)); + assertThat(vector.getObject(0), nullValue()); + } + + @Test + public void testBindParameterWithInvalidByteArrayLength() { + byte[] invalidBytes = new byte[8]; // Should be 16 bytes + ByteString byteString = new ByteString(invalidBytes); + TypedValue typedValue = TypedValue.ofLocal(ColumnMetaData.Rep.BYTE_STRING, byteString); + + assertThrows( + IllegalArgumentException.class, () -> converter.bindParameter(vector, typedValue, 0)); + } + + @Test + public void testBindParameterWithInvalidType() { + TypedValue typedValue = TypedValue.ofLocal(ColumnMetaData.Rep.INTEGER, 12345); + + assertThrows( + IllegalArgumentException.class, () -> converter.bindParameter(vector, typedValue, 0)); + } + + @Test + public void testBindParameterMultipleValues() { + UUID uuid1 = UUID.fromString("550e8400-e29b-41d4-a716-446655440000"); + UUID uuid2 = UUID.fromString("6ba7b810-9dad-11d1-80b4-00c04fd430c8"); + UUID uuid3 = UUID.fromString("f47ac10b-58cc-4372-a567-0e02b2c3d479"); + + converter.bindParameter(vector, TypedValue.ofLocal(ColumnMetaData.Rep.OBJECT, uuid1), 0); + converter.bindParameter(vector, TypedValue.ofLocal(ColumnMetaData.Rep.OBJECT, uuid2), 1); + converter.bindParameter(vector, TypedValue.ofLocal(ColumnMetaData.Rep.OBJECT, uuid3), 2); + + assertThat(vector.getObject(0), is(uuid1)); + assertThat(vector.getObject(1), is(uuid2)); + assertThat(vector.getObject(2), is(uuid3)); + } + + @Test + public void testCreateParameter() { + Field uuidField = new Field("uuid_col", new FieldType(true, UuidType.INSTANCE, null), null); + + AvaticaParameter parameter = converter.createParameter(uuidField); + + assertThat(parameter.name, is("uuid_col")); + assertThat(parameter.parameterType, is(Types.OTHER)); + assertThat(parameter.typeName, is("OTHER")); + assertThat(parameter.className, equalTo(UUID.class.getCanonicalName())); + } +} diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/CoreMockedSqlProducers.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/CoreMockedSqlProducers.java index 8197d7d95f..7c17755693 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/CoreMockedSqlProducers.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/CoreMockedSqlProducers.java @@ -28,8 +28,10 @@ import java.sql.SQLException; import java.sql.Timestamp; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.UUID; import java.util.function.Consumer; import java.util.stream.IntStream; import org.apache.arrow.flight.FlightProducer.ServerStreamListener; @@ -40,10 +42,13 @@ import org.apache.arrow.vector.DateDayVector; import org.apache.arrow.vector.Float4Vector; import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.TimeStampMilliVector; import org.apache.arrow.vector.UInt4Vector; +import org.apache.arrow.vector.UuidVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.types.DateUnit; import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.types.TimeUnit; @@ -52,6 +57,7 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.Text; +import org.apache.arrow.vector.util.UuidUtility; /** Standard {@link MockFlightSqlProducer} instances for tests. */ // TODO Remove this once all tests are refactor to use only the queries they need. @@ -62,6 +68,22 @@ public final class CoreMockedSqlProducers { public static final String LEGACY_CANCELLATION_SQL_CMD = "SELECT * FROM TAKES_FOREVER"; public static final String LEGACY_REGULAR_WITH_EMPTY_SQL_CMD = "SELECT * FROM TEST_EMPTIES"; + public static final String UUID_SQL_CMD = "SELECT * FROM UUID_TABLE"; + public static final String UUID_PREPARED_SELECT_SQL_CMD = + "SELECT * FROM UUID_TABLE WHERE uuid_col = ?"; + public static final String UUID_PREPARED_UPDATE_SQL_CMD = + "UPDATE UUID_TABLE SET uuid_col = ? WHERE id = ?"; + + public static final UUID UUID_1 = UUID.fromString("550e8400-e29b-41d4-a716-446655440000"); + public static final UUID UUID_2 = UUID.fromString("6ba7b810-9dad-11d1-80b4-00c04fd430c8"); + public static final UUID UUID_3 = UUID.fromString("f47ac10b-58cc-4372-a567-0e02b2c3d479"); + + public static final Schema UUID_SCHEMA = + new Schema( + ImmutableList.of( + new Field("id", new FieldType(true, new ArrowType.Int(32, true), null), null), + new Field("uuid_col", new FieldType(true, UuidType.INSTANCE, null), null))); + private CoreMockedSqlProducers() { // Prevent instantiation. } @@ -78,9 +100,109 @@ public static MockFlightSqlProducer getLegacyProducer() { addLegacyMetadataSqlCmdSupport(producer); addLegacyCancellationSqlCmdSupport(producer); addQueryWithEmbeddedEmptyRoot(producer); + addUuidSqlCmdSupport(producer); + addUuidPreparedSelectSqlCmdSupport(producer); + addUuidPreparedUpdateSqlCmdSupport(producer); return producer; } + /** + * Gets a {@link MockFlightSqlProducer} configured with UUID test data. + * + * @return a new producer with UUID support. + */ + public static MockFlightSqlProducer getUuidProducer() { + final MockFlightSqlProducer producer = new MockFlightSqlProducer(); + addUuidSqlCmdSupport(producer); + return producer; + } + + private static void addUuidPreparedUpdateSqlCmdSupport(final MockFlightSqlProducer producer) { + final String query = "UPDATE UUID_TABLE SET uuid_col = ? WHERE id = ?"; + final Schema parameterSchema = + new Schema( + Arrays.asList( + new Field("", new FieldType(true, UuidType.INSTANCE, null), null), + Field.nullable("", new ArrowType.Int(32, true)))); + + producer.addUpdateQuery(query, 1); + producer.addExpectedParameters( + UUID_PREPARED_UPDATE_SQL_CMD, + parameterSchema, + Collections.singletonList(Arrays.asList(CoreMockedSqlProducers.UUID_3, 1))); + } + + private static void addUuidPreparedSelectSqlCmdSupport(final MockFlightSqlProducer producer) { + final Schema parameterSchema = + new Schema( + Collections.singletonList( + new Field("", new FieldType(true, UuidType.INSTANCE, null), null))); + + final Consumer uuidResultProvider = + listener -> { + try (final BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + final VectorSchemaRoot root = VectorSchemaRoot.create(UUID_SCHEMA, allocator)) { + root.allocateNew(); + IntVector idVector = (IntVector) root.getVector("id"); + UuidVector uuidVector = (UuidVector) root.getVector("uuid_col"); + idVector.setSafe(0, 1); + uuidVector.setSafe(0, UuidUtility.getBytesFromUUID(CoreMockedSqlProducers.UUID_1)); + root.setRowCount(1); + listener.start(root); + listener.putNext(); + } catch (final Throwable throwable) { + listener.error(throwable); + } finally { + listener.completed(); + } + }; + + producer.addSelectQuery( + UUID_PREPARED_SELECT_SQL_CMD, UUID_SCHEMA, Collections.singletonList(uuidResultProvider)); + producer.addExpectedParameters( + UUID_PREPARED_SELECT_SQL_CMD, + parameterSchema, + Collections.singletonList(Collections.singletonList(CoreMockedSqlProducers.UUID_1))); + } + + private static void addUuidSqlCmdSupport(final MockFlightSqlProducer producer) { + final Consumer uuidResultProvider = + listener -> { + try (final BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + final VectorSchemaRoot root = VectorSchemaRoot.create(UUID_SCHEMA, allocator)) { + root.allocateNew(); + + IntVector idVector = (IntVector) root.getVector("id"); + UuidVector uuidVector = (UuidVector) root.getVector("uuid_col"); + + // Row 0: id=1, uuid=UUID_1 + idVector.setSafe(0, 1); + uuidVector.setSafe(0, UuidUtility.getBytesFromUUID(UUID_1)); + + // Row 1: id=2, uuid=UUID_2 + idVector.setSafe(1, 2); + uuidVector.setSafe(1, UuidUtility.getBytesFromUUID(UUID_2)); + + // Row 2: id=3, uuid=UUID_3 + idVector.setSafe(2, 3); + uuidVector.setSafe(2, UuidUtility.getBytesFromUUID(UUID_3)); + + // Row 3: id=4, uuid=NULL + idVector.setSafe(3, 4); + uuidVector.setNull(3); + + root.setRowCount(4); + listener.start(root); + listener.putNext(); + } finally { + listener.completed(); + } + }; + + producer.addSelectQuery( + UUID_SQL_CMD, UUID_SCHEMA, Collections.singletonList(uuidResultProvider)); + } + private static void addQueryWithEmbeddedEmptyRoot(final MockFlightSqlProducer producer) { final Schema querySchema = new Schema( diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/RootAllocatorTestExtension.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/RootAllocatorTestExtension.java index 347e92a16c..4b299d63e0 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/RootAllocatorTestExtension.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/RootAllocatorTestExtension.java @@ -19,6 +19,7 @@ import java.math.BigDecimal; import java.nio.charset.StandardCharsets; import java.util.Random; +import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; import org.apache.arrow.memory.BufferAllocator; @@ -53,6 +54,7 @@ import org.apache.arrow.vector.UInt2Vector; import org.apache.arrow.vector.UInt4Vector; import org.apache.arrow.vector.UInt8Vector; +import org.apache.arrow.vector.UuidVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.complex.FixedSizeListVector; import org.apache.arrow.vector.complex.LargeListVector; @@ -60,6 +62,7 @@ import org.apache.arrow.vector.complex.impl.UnionFixedSizeListWriter; import org.apache.arrow.vector.complex.impl.UnionLargeListWriter; import org.apache.arrow.vector.complex.impl.UnionListWriter; +import org.apache.arrow.vector.util.UuidUtility; import org.junit.jupiter.api.extension.AfterAllCallback; import org.junit.jupiter.api.extension.BeforeAllCallback; import org.junit.jupiter.api.extension.ExtensionContext; @@ -811,4 +814,23 @@ public FixedSizeListVector createFixedSizeListVector() { return valueVector; } + + /** + * Create a UuidVector to be used in the accessor tests. + * + * @return UuidVector + */ + public UuidVector createUuidVector() { + UuidVector valueVector = new UuidVector("", this.getRootAllocator()); + valueVector.allocateNew(3); + valueVector.setSafe( + 0, UuidUtility.getBytesFromUUID(UUID.fromString("550e8400-e29b-41d4-a716-446655440000"))); + valueVector.setSafe( + 1, UuidUtility.getBytesFromUUID(UUID.fromString("6ba7b810-9dad-11d1-80b4-00c04fd430c8"))); + valueVector.setSafe( + 2, UuidUtility.getBytesFromUUID(UUID.fromString("f47ac10b-58cc-4372-a567-0e02b2c3d479"))); + valueVector.setValueCount(3); + + return valueVector; + } } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/SqlTypesTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/SqlTypesTest.java index d69c549296..c4858d787d 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/SqlTypesTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/SqlTypesTest.java @@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import java.sql.Types; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.types.DateUnit; import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.types.IntervalUnit; @@ -85,6 +86,8 @@ public void testGetSqlTypeIdFromArrowType() { assertEquals(Types.JAVA_OBJECT, getSqlTypeIdFromArrowType(new ArrowType.Map(true))); assertEquals(Types.NULL, getSqlTypeIdFromArrowType(new ArrowType.Null())); + + assertEquals(Types.OTHER, getSqlTypeIdFromArrowType(UuidType.INSTANCE)); } @Test @@ -140,5 +143,7 @@ public void testGetSqlTypeNameFromArrowType() { assertEquals("JAVA_OBJECT", getSqlTypeNameFromArrowType(new ArrowType.Map(true))); assertEquals("NULL", getSqlTypeNameFromArrowType(new ArrowType.Null())); + + assertEquals("OTHER", getSqlTypeNameFromArrowType(UuidType.INSTANCE)); } } From 7e61d462c094ff9eb3a692176b040a08f81654fe Mon Sep 17 00:00:00 2001 From: Pedro Matias Date: Thu, 22 Jan 2026 14:19:32 +0000 Subject: [PATCH 267/408] GH-932: [JDBC] Fix memory leak on Connection#close due to unclosed Statement(s) (#933) ## What's Changed Closing a Connection when there was one or more ResultSet that matched the following 2 conditions 1. hadn't been fully consumed 2. was obtained via a Statement instance of this Connection instance would generate exceptions due to memory leaks. Now, closing a Connection will first close all the Statement instances obtained via that Connection, which has a side effect of closing all the ResultSet, and then proceed with the old closing logic. This side effect is guaranteed by the JDBC Spec 4.3, chapter 13.1.4 The old closing logic was also slightly refactored to: 1. remove duplicate calls to ArrowFlightSqlClientHandler.close() 5. make sure that any exception generated during Connection.close() would be wrapped in a SQLException. Closes #932. --- .../driver/jdbc/ArrowFlightConnection.java | 39 ++++++++++++++----- .../arrow/driver/jdbc/ConnectionTest.java | 38 ++++++++++++++++++ 2 files changed, 68 insertions(+), 9 deletions(-) diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java index f6f17770f1..f81233ec33 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java @@ -20,6 +20,7 @@ import io.netty.util.concurrent.DefaultThreadFactory; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -180,19 +181,39 @@ public Properties getClientInfo() { @Override public void close() throws SQLException { - clientHandler.close(); - if (executorService != null) { - executorService.shutdown(); + Exception topLevelException = null; + try { + if (executorService != null) { + executorService.shutdown(); + } + } catch (final Exception e) { + topLevelException = e; + } + ArrayList closeables = new ArrayList<>(statementMap.values()); + closeables.add(clientHandler); + closeables.addAll(allocator.getChildAllocators()); + closeables.add(allocator); + try { + AutoCloseables.close(closeables); + } catch (final Exception e) { + if (topLevelException == null) { + topLevelException = e; + } else { + topLevelException.addSuppressed(e); + } } - try { - AutoCloseables.close(clientHandler); - allocator.getChildAllocators().forEach(AutoCloseables::closeNoChecked); - AutoCloseables.close(allocator); - super.close(); } catch (final Exception e) { - throw AvaticaConnection.HELPER.createException(e.getMessage(), e); + if (topLevelException == null) { + topLevelException = e; + } else { + topLevelException.addSuppressed(e); + } + } + if (topLevelException != null) { + throw AvaticaConnection.HELPER.createException( + topLevelException.getMessage(), topLevelException); } } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java index 46762f3319..dbedbe9d36 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java @@ -17,6 +17,7 @@ package org.apache.arrow.driver.jdbc; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -27,6 +28,7 @@ import java.sql.Driver; import java.sql.DriverManager; import java.sql.SQLException; +import java.sql.Statement; import java.util.Map; import java.util.Properties; import org.apache.arrow.driver.jdbc.authentication.UserPasswordAuthentication; @@ -660,4 +662,40 @@ public String visit(String value) { assertEquals(catalog, actualCatalog); } } + + @Test + public void testStatementsClosedOnConnectionClose() throws Exception { + // create a connection + final Properties properties = new Properties(); + properties.put(ArrowFlightConnectionProperty.HOST.camelName(), "localhost"); + properties.put( + ArrowFlightConnectionProperty.PORT.camelName(), FLIGHT_SERVER_TEST_EXTENSION.getPort()); + properties.put(ArrowFlightConnectionProperty.USER.camelName(), userTest); + properties.put(ArrowFlightConnectionProperty.PASSWORD.camelName(), passTest); + properties.put("useEncryption", false); + + Connection connection = + DriverManager.getConnection( + "jdbc:arrow-flight-sql://" + + FLIGHT_SERVER_TEST_EXTENSION.getHost() + + ":" + + FLIGHT_SERVER_TEST_EXTENSION.getPort(), + properties); + + // create some statements + int numStatements = 3; + Statement[] statements = new Statement[numStatements]; + for (int i = 0; i < numStatements; i++) { + statements[i] = connection.createStatement(); + assertFalse(statements[i].isClosed()); + } + + // close the connection + connection.close(); + + // assert the statements are closed + for (int i = 0; i < numStatements; i++) { + assertTrue(statements[i].isClosed()); + } + } } From 44c49baf6c2fdfcf20c8611f45c627c9438b2adb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Thu, 22 Jan 2026 14:19:58 +0000 Subject: [PATCH 268/408] GH-952: Add OAuth support (#953) ## What's Changed - Add OAuth 2.0 support to the Flight SQL JDBC driver, including client credentials and token exchange flows - Integrate OAuth token acquisition into connection setup, wiring tokens through OAuthCredentialWriter and updating gRPC credential handling to fail fast on writer errors. - Document new OAuth connection properties and add example clients for both OAuth flows. - Connection Properties Added - oauth.flow - oauth.tokenUri - oauth.clientId - oauth.clientSecret - oauth.scope - oauth.resource - oauth.exchange.subjectToken - oauth.exchange.subjectTokenType - oauth.exchange.actorToken - oauth.exchange.actorTokenType - oauth.exchange.aud - oauth.exchange.requestedTokenType - Connection config now recognizes oauth.* and oauth.exchange.* properties and builds OAuth providers when oauth.flow is specified. - Adds com.nimbusds:oauth2-oidc-sdk dependency and mockwebserver for tests. Closes #952. --- docs/source/flight_sql_jdbc_driver.rst | 123 +++++ .../flight/grpc/CallCredentialAdapter.java | 12 +- flight/flight-sql-jdbc-core/pom.xml | 32 ++ .../driver/jdbc/ArrowFlightConnection.java | 1 + .../client/ArrowFlightSqlClientHandler.java | 23 +- .../oauth/AbstractOAuthTokenProvider.java | 108 ++++ .../oauth/ClientCredentialsTokenProvider.java | 58 +++ .../jdbc/client/oauth/OAuthConfiguration.java | 240 +++++++++ .../client/oauth/OAuthCredentialWriter.java | 42 ++ .../client/oauth/OAuthTokenException.java | 31 ++ .../jdbc/client/oauth/OAuthTokenProvider.java | 33 ++ .../client/oauth/OAuthTokenProviders.java | 419 ++++++++++++++++ .../oauth/TokenExchangeTokenProvider.java | 81 +++ .../driver/jdbc/client/oauth/TokenInfo.java | 45 ++ .../ArrowFlightConnectionConfigImpl.java | 52 ++ .../driver/jdbc/OAuthIntegrationTest.java | 474 ++++++++++++++++++ .../client/oauth/OAuthConfigurationTest.java | 296 +++++++++++ .../oauth/OAuthCredentialWriterTest.java | 95 ++++ .../src/shade/LICENSE.txt | 8 + .../driver/jdbc/ITDriverJarValidation.java | 4 + 20 files changed, 2173 insertions(+), 4 deletions(-) create mode 100644 flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/AbstractOAuthTokenProvider.java create mode 100644 flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/ClientCredentialsTokenProvider.java create mode 100644 flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthConfiguration.java create mode 100644 flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthCredentialWriter.java create mode 100644 flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthTokenException.java create mode 100644 flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthTokenProvider.java create mode 100644 flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthTokenProviders.java create mode 100644 flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/TokenExchangeTokenProvider.java create mode 100644 flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/TokenInfo.java create mode 100644 flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/OAuthIntegrationTest.java create mode 100644 flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthConfigurationTest.java create mode 100644 flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthCredentialWriterTest.java diff --git a/docs/source/flight_sql_jdbc_driver.rst b/docs/source/flight_sql_jdbc_driver.rst index 1806930943..4deb726b33 100644 --- a/docs/source/flight_sql_jdbc_driver.rst +++ b/docs/source/flight_sql_jdbc_driver.rst @@ -173,3 +173,126 @@ DriverManager#getConnection() `_, the username and password supplied on the URI supercede the username and password arguments to the function call. + +OAuth 2.0 Authentication +======================== + +The driver supports OAuth 2.0 authentication for obtaining access tokens +from an authorization server. Two OAuth flows are currently supported: + +* **Client Credentials** - For service-to-service authentication where no + user interaction is required. The application authenticates using its own + credentials (client ID and client secret). + +* **Token Exchange** (RFC 8693) - For exchanging one token for another, + commonly used for federated authentication, delegation, or impersonation + scenarios. + +OAuth Connection Properties +--------------------------- + +The following properties configure OAuth authentication. These properties +should be provided via the ``Properties`` object when connecting, as they +may contain special characters that are difficult to encode in a URI. + +**Common OAuth Properties** + +.. list-table:: + :header-rows: 1 + + * - Parameter + - Type + - Required + - Default + - Description + + * - oauth.flow + - String + - Yes (to enable OAuth) + - null + - The OAuth grant type. Supported values: ``client_credentials``, + ``token_exchange`` + + * - oauth.tokenUri + - String + - Yes + - null + - The OAuth 2.0 token endpoint URL (e.g., + ``https://auth.example.com/oauth/token``) + + * - oauth.clientId + - String + - Conditional + - null + - The OAuth 2.0 client ID. Required for ``client_credentials`` flow, + optional for ``token_exchange`` + + * - oauth.clientSecret + - String + - Conditional + - null + - The OAuth 2.0 client secret. Required for ``client_credentials`` flow, + optional for ``token_exchange`` + + * - oauth.scope + - String + - No + - null + - Space-separated list of OAuth scopes to request + + * - oauth.resource + - String + - No + - null + - The resource indicator for the token request (RFC 8707) + +**Token Exchange Properties** + +These properties are specific to the ``token_exchange`` flow: + +.. list-table:: + :header-rows: 1 + + * - Parameter + - Type + - Required + - Default + - Description + + * - oauth.exchange.subjectToken + - String + - Yes + - null + - The subject token to exchange (e.g., a JWT from an identity provider) + + * - oauth.exchange.subjectTokenType + - String + - Yes + - null + - The token type URI of the subject token. Common values: + ``urn:ietf:params:oauth:token-type:access_token``, + ``urn:ietf:params:oauth:token-type:jwt`` + + * - oauth.exchange.actorToken + - String + - No + - null + - The actor token for delegation/impersonation scenarios + + * - oauth.exchange.actorTokenType + - String + - No + - null + - The token type URI of the actor token + + * - oauth.exchange.aud + - String + - No + - null + - The target audience for the exchanged token + + * - oauth.exchange.requestedTokenType + - String + - No + - null + - The desired token type for the exchanged token diff --git a/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/CallCredentialAdapter.java b/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/CallCredentialAdapter.java index f33e9b2f94..fe81f3fb23 100644 --- a/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/CallCredentialAdapter.java +++ b/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/CallCredentialAdapter.java @@ -18,6 +18,7 @@ import io.grpc.CallCredentials; import io.grpc.Metadata; +import io.grpc.Status; import java.util.concurrent.Executor; import java.util.function.Consumer; import org.apache.arrow.flight.CallHeaders; @@ -36,9 +37,14 @@ public void applyRequestMetadata( RequestInfo requestInfo, Executor executor, MetadataApplier metadataApplier) { executor.execute( () -> { - final Metadata headers = new Metadata(); - credentialWriter.accept(new MetadataAdapter(headers)); - metadataApplier.apply(headers); + try { + final Metadata headers = new Metadata(); + credentialWriter.accept(new MetadataAdapter(headers)); + metadataApplier.apply(headers); + } catch (Throwable t) { + metadataApplier.fail( + Status.UNAUTHENTICATED.withCause(t).withDescription(t.getMessage())); + } }); } diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index bb191ca9ed..da00baf32a 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -120,6 +120,31 @@ under the License. test + + com.squareup.okhttp3 + mockwebserver3 + 5.3.2 + test + + + com.squareup.okhttp3 + mockwebserver3-junit5 + 5.3.2 + test + + + com.squareup.okhttp3 + okhttp-jvm + 5.3.2 + test + + + com.squareup.okio + okio-jvm + 3.16.4 + test + + io.netty netty-common @@ -153,6 +178,13 @@ under the License. caffeine 3.2.3 + + + com.nimbusds + oauth2-oidc-sdk + 11.20.1 + + diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java index f81233ec33..0e9c198f52 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java @@ -122,6 +122,7 @@ private static ArrowFlightSqlClientHandler createNewClientHandler( .withClientCache(config.useClientCache() ? new FlightClientCache() : null) .withConnectTimeout(config.getConnectTimeout()) .withDriverVersion(driverVersion) + .withOAuthConfiguration(config.getOauthConfiguration()) .build(); } catch (final SQLException e) { try { diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java index 666996cd95..f0ea284239 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java @@ -32,6 +32,9 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import org.apache.arrow.driver.jdbc.client.oauth.OAuthConfiguration; +import org.apache.arrow.driver.jdbc.client.oauth.OAuthCredentialWriter; +import org.apache.arrow.driver.jdbc.client.oauth.OAuthTokenProvider; import org.apache.arrow.driver.jdbc.client.utils.ClientAuthenticationUtils; import org.apache.arrow.driver.jdbc.client.utils.FlightClientCache; import org.apache.arrow.driver.jdbc.client.utils.FlightLocationQueue; @@ -675,6 +678,8 @@ public static final class Builder { @VisibleForTesting @Nullable Duration connectTimeout; + @VisibleForTesting @Nullable OAuthConfiguration oauthConfig; + // These two middleware are for internal use within build() and should not be // exposed by builder // APIs. @@ -714,6 +719,7 @@ public Builder() {} this.clientKeyPath = original.clientKeyPath; this.allocator = original.allocator; this.catalog = original.catalog; + this.oauthConfig = original.oauthConfig; if (original.retainCookies) { this.cookieFactory = original.cookieFactory; @@ -983,6 +989,17 @@ public Builder withDriverVersion(DriverVersion driverVersion) { return this; } + /** + * Sets the OAuth configuration for this handler. + * + * @param oauthConfig the OAuth configuration + * @return this builder instance + */ + public Builder withOAuthConfiguration(final OAuthConfiguration oauthConfig) { + this.oauthConfig = oauthConfig; + return this; + } + public String getCacheKey() { return getLocation().toString(); } @@ -1070,7 +1087,11 @@ public ArrowFlightSqlClientHandler build() throws SQLException { FlightGrpcUtils.createFlightClient( allocator, channelBuilder.build(), clientBuilder.middleware()); final ArrayList credentialOptions = new ArrayList<>(); - if (isUsingUserPasswordAuth) { + // Authentication priority: OAuth > token > username/password + if (oauthConfig != null) { + OAuthTokenProvider tokenProvider = oauthConfig.createTokenProvider(); + credentialOptions.add(new CredentialCallOption(new OAuthCredentialWriter(tokenProvider))); + } else if (isUsingUserPasswordAuth) { // If the authFactory has already been used for a handshake, use the existing // token. // This can occur if the authFactory is being re-used for a new connection diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/AbstractOAuthTokenProvider.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/AbstractOAuthTokenProvider.java new file mode 100644 index 0000000000..9c377a5850 --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/AbstractOAuthTokenProvider.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.client.oauth; + +import com.nimbusds.oauth2.sdk.ParseException; +import com.nimbusds.oauth2.sdk.Scope; +import com.nimbusds.oauth2.sdk.TokenErrorResponse; +import com.nimbusds.oauth2.sdk.TokenRequest; +import com.nimbusds.oauth2.sdk.TokenResponse; +import com.nimbusds.oauth2.sdk.auth.ClientAuthentication; +import com.nimbusds.oauth2.sdk.token.AccessToken; +import java.io.IOException; +import java.net.URI; +import java.sql.SQLException; +import java.time.Instant; +import org.apache.arrow.util.VisibleForTesting; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** + * Abstract base class for OAuth token providers that handles token caching, refresh logic, and + * common request/response handling. + */ +public abstract class AbstractOAuthTokenProvider implements OAuthTokenProvider { + protected static final int EXPIRATION_BUFFER_SECONDS = 30; + protected static final int DEFAULT_EXPIRATION_SECONDS = 3600; + + private final Object tokenLock = new Object(); + private volatile @Nullable TokenInfo cachedToken; + + @VisibleForTesting URI tokenUri; + + @VisibleForTesting @Nullable ClientAuthentication clientAuth; + + @VisibleForTesting @Nullable Scope scope; + + @Override + public String getValidToken() throws SQLException { + TokenInfo token = cachedToken; + if (token != null && !token.isExpired(EXPIRATION_BUFFER_SECONDS)) { + return token.getAccessToken(); + } + + synchronized (tokenLock) { + token = cachedToken; + if (token != null && !token.isExpired(EXPIRATION_BUFFER_SECONDS)) { + return token.getAccessToken(); + } + cachedToken = fetchNewToken(); + return cachedToken.getAccessToken(); + } + } + + /** + * Fetches a new token from the authorization server. This method handles the common + * request/response logic while delegating flow-specific request building to subclasses. + * + * @return the new token information + * @throws SQLException if token cannot be obtained + */ + protected TokenInfo fetchNewToken() throws SQLException { + try { + TokenRequest request = buildTokenRequest(); + TokenResponse response = TokenResponse.parse(request.toHTTPRequest().send()); + + if (!response.indicatesSuccess()) { + TokenErrorResponse errorResponse = response.toErrorResponse(); + String errorMsg = + String.format( + "OAuth request failed: %s - %s", + errorResponse.getErrorObject().getCode(), + errorResponse.getErrorObject().getDescription()); + throw new SQLException(errorMsg); + } + + AccessToken accessToken = response.toSuccessResponse().getTokens().getAccessToken(); + long expiresIn = + accessToken.getLifetime() > 0 ? accessToken.getLifetime() : DEFAULT_EXPIRATION_SECONDS; + Instant expiresAt = Instant.now().plusSeconds(expiresIn); + + return new TokenInfo(accessToken.getValue(), expiresAt); + } catch (ParseException e) { + throw new SQLException("Failed to parse OAuth token response", e); + } catch (IOException e) { + throw new SQLException("Failed to send OAuth token request", e); + } + } + + /** + * Builds the flow-specific token request. + * + * @return the token request to send to the authorization server + */ + protected abstract TokenRequest buildTokenRequest(); +} diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/ClientCredentialsTokenProvider.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/ClientCredentialsTokenProvider.java new file mode 100644 index 0000000000..7e6289819c --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/ClientCredentialsTokenProvider.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.client.oauth; + +import com.nimbusds.oauth2.sdk.ClientCredentialsGrant; +import com.nimbusds.oauth2.sdk.Scope; +import com.nimbusds.oauth2.sdk.TokenRequest; +import com.nimbusds.oauth2.sdk.auth.ClientSecretBasic; +import com.nimbusds.oauth2.sdk.auth.Secret; +import com.nimbusds.oauth2.sdk.id.ClientID; +import java.net.URI; +import java.util.Objects; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** + * OAuth 2.0 Client Credentials flow token provider (RFC 6749 Section 4.4). + * + *

This provider handles service-to-service authentication where no user interaction is required. + * Tokens are cached and automatically refreshed before expiration. + */ +public class ClientCredentialsTokenProvider extends AbstractOAuthTokenProvider { + + /** + * Creates a new ClientCredentialsTokenProvider. + * + * @param tokenUri the OAuth token endpoint URI + * @param clientId the OAuth client ID + * @param clientSecret the OAuth client secret + * @param scope optional OAuth scopes (space-separated) + */ + ClientCredentialsTokenProvider( + URI tokenUri, String clientId, String clientSecret, @Nullable String scope) { + this.tokenUri = Objects.requireNonNull(tokenUri, "tokenUri cannot be null"); + Objects.requireNonNull(clientId, "clientId cannot be null"); + Objects.requireNonNull(clientSecret, "clientSecret cannot be null"); + this.clientAuth = new ClientSecretBasic(new ClientID(clientId), new Secret(clientSecret)); + this.scope = (scope != null && !scope.isEmpty()) ? Scope.parse(scope) : null; + } + + @Override + protected TokenRequest buildTokenRequest() { + return new TokenRequest(tokenUri, clientAuth, new ClientCredentialsGrant(), scope); + } +} diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthConfiguration.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthConfiguration.java new file mode 100644 index 0000000000..cba9d4c2e6 --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthConfiguration.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.client.oauth; + +import com.nimbusds.oauth2.sdk.GrantType; +import java.net.URI; +import java.net.URISyntaxException; +import java.sql.SQLException; +import java.util.Locale; +import java.util.Objects; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** Configuration class for OAuth settings parsed from connection properties. */ +public class OAuthConfiguration { + + private final GrantType grantType; + private final URI tokenUri; + private final @Nullable String clientId; + private final @Nullable String clientSecret; + private final @Nullable String scope; + private final @Nullable String subjectToken; + private final @Nullable String subjectTokenType; + private final @Nullable String actorToken; + private final @Nullable String actorTokenType; + private final @Nullable String audience; + private final @Nullable String resource; + private final @Nullable String requestedTokenType; + + private OAuthConfiguration(Builder builder) throws SQLException { + this.grantType = builder.grantType; + this.tokenUri = builder.tokenUri; + this.clientId = builder.clientId; + this.clientSecret = builder.clientSecret; + this.scope = builder.scope; + this.subjectToken = builder.subjectToken; + this.subjectTokenType = builder.subjectTokenType; + this.actorToken = builder.actorToken; + this.actorTokenType = builder.actorTokenType; + this.audience = builder.audience; + this.resource = builder.resource; + this.requestedTokenType = builder.requestedTokenType; + + validate(); + } + + private void validate() throws SQLException { + Objects.requireNonNull(grantType, "OAuth grant type is required"); + Objects.requireNonNull(tokenUri, "Token URI is required"); + + if (GrantType.CLIENT_CREDENTIALS.equals(grantType)) { + if (clientId == null || clientId.isEmpty()) { + throw new SQLException("clientId is required for client_credentials flow"); + } + if (clientSecret == null || clientSecret.isEmpty()) { + throw new SQLException("clientSecret is required for client_credentials flow"); + } + } else if (GrantType.TOKEN_EXCHANGE.equals(grantType)) { + if (subjectToken == null || subjectToken.isEmpty()) { + throw new SQLException("subjectToken is required for token_exchange flow"); + } + if (subjectTokenType == null || subjectTokenType.isEmpty()) { + throw new SQLException("subjectTokenType is required for token_exchange flow"); + } + } else { + throw new SQLException("Unsupported OAuth grant type: " + grantType); + } + } + + /** + * Creates an OAuthTokenProvider based on the configured grant type. + * + * @return the token provider + * @throws SQLException if the grant type is not supported or configuration is invalid + */ + public OAuthTokenProvider createTokenProvider() throws SQLException { + if (GrantType.CLIENT_CREDENTIALS.equals(grantType)) { + return OAuthTokenProviders.clientCredentials() + .tokenUri(tokenUri) + .clientId(clientId) + .clientSecret(clientSecret) + .scope(scope) + .build(); + } else if (GrantType.TOKEN_EXCHANGE.equals(grantType)) { + OAuthTokenProviders.TokenExchangeBuilder builder = + OAuthTokenProviders.tokenExchange() + .tokenUri(tokenUri) + .subjectToken(subjectToken) + .subjectTokenType(subjectTokenType) + .actorToken(actorToken) + .actorTokenType(actorTokenType) + .audience(audience) + .requestedTokenType(requestedTokenType) + .scope(scope) + .resource(resource); + + if (clientId != null && clientSecret != null) { + builder.clientCredentials(clientId, clientSecret); + } + + return builder.build(); + } else { + throw new SQLException("Unsupported OAuth grant type: " + grantType); + } + } + + /** Builder for OAuthConfiguration. */ + public static class Builder { + private GrantType grantType; + private URI tokenUri; + private @Nullable String clientId; + private @Nullable String clientSecret; + private @Nullable String scope; + private @Nullable String subjectToken; + private @Nullable String subjectTokenType; + private @Nullable String actorToken; + private @Nullable String actorTokenType; + private @Nullable String audience; + private @Nullable String resource; + private @Nullable String requestedTokenType; + + /** + * Sets the OAuth grant type from a string value. + * + *

Accepts either user-friendly names ("client_credentials", "token_exchange") or the full + * URN format as defined in RFC 6749 and RFC 8693. + * + * @param flowStr the flow type string (e.g., "client_credentials", "token_exchange") + * @return this builder + * @throws SQLException if the flow string is invalid + */ + public Builder flow(String flowStr) throws SQLException { + if (flowStr == null || flowStr.isEmpty()) { + throw new SQLException("OAuth flow cannot be null or empty"); + } + try { + String normalized = flowStr.toLowerCase(Locale.ROOT); + // Map user-friendly names to URN format for token_exchange + if ("token_exchange".equals(normalized)) { + normalized = GrantType.TOKEN_EXCHANGE.getValue(); + } + GrantType parsed = GrantType.parse(normalized); + if (!parsed.equals(GrantType.CLIENT_CREDENTIALS) + && !parsed.equals(GrantType.TOKEN_EXCHANGE)) { + throw new SQLException("Unsupported OAuth flow: " + flowStr); + } + this.grantType = parsed; + } catch (com.nimbusds.oauth2.sdk.ParseException e) { + throw new SQLException("Invalid OAuth flow: " + flowStr, e); + } + return this; + } + + /** + * Sets the token URI. + * + * @param tokenUri the OAuth token endpoint URI + * @return this builder + * @throws SQLException if the URI is invalid + */ + public Builder tokenUri(String tokenUri) throws SQLException { + if (tokenUri == null || tokenUri.isEmpty()) { + throw new SQLException("Token URI cannot be null or empty"); + } + try { + this.tokenUri = new URI(tokenUri); + } catch (URISyntaxException e) { + throw new SQLException("Invalid token URI: " + tokenUri, e); + } + return this; + } + + public Builder clientId(@Nullable String clientId) { + this.clientId = clientId; + return this; + } + + public Builder clientSecret(@Nullable String clientSecret) { + this.clientSecret = clientSecret; + return this; + } + + public Builder scope(@Nullable String scope) { + this.scope = scope; + return this; + } + + public Builder subjectToken(@Nullable String subjectToken) { + this.subjectToken = subjectToken; + return this; + } + + public Builder subjectTokenType(@Nullable String subjectTokenType) { + this.subjectTokenType = subjectTokenType; + return this; + } + + public Builder actorToken(@Nullable String actorToken) { + this.actorToken = actorToken; + return this; + } + + public Builder actorTokenType(@Nullable String actorTokenType) { + this.actorTokenType = actorTokenType; + return this; + } + + public Builder audience(@Nullable String audience) { + this.audience = audience; + return this; + } + + public Builder resource(@Nullable String resource) { + this.resource = resource; + return this; + } + + public Builder requestedTokenType(@Nullable String requestedTokenType) { + this.requestedTokenType = requestedTokenType; + return this; + } + + public OAuthConfiguration build() throws SQLException { + return new OAuthConfiguration(this); + } + } +} diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthCredentialWriter.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthCredentialWriter.java new file mode 100644 index 0000000000..0d4ad4689f --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthCredentialWriter.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.client.oauth; + +import java.sql.SQLException; +import java.util.Objects; +import java.util.function.Consumer; +import org.apache.arrow.flight.CallHeaders; +import org.apache.arrow.flight.auth2.Auth2Constants; + +/** Writes OAuth bearer tokens to Flight call headers. */ +public class OAuthCredentialWriter implements Consumer { + private final OAuthTokenProvider tokenProvider; + + public OAuthCredentialWriter(OAuthTokenProvider tokenProvider) { + this.tokenProvider = Objects.requireNonNull(tokenProvider, "tokenProvider cannot be null"); + } + + @Override + public void accept(CallHeaders headers) { + try { + String token = tokenProvider.getValidToken(); + headers.insert(Auth2Constants.AUTHORIZATION_HEADER, Auth2Constants.BEARER_PREFIX + token); + } catch (SQLException e) { + throw new OAuthTokenException("Failed to obtain OAuth token", e); + } + } +} diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthTokenException.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthTokenException.java new file mode 100644 index 0000000000..aceadb327b --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthTokenException.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.client.oauth; + +/** + * Runtime exception thrown when OAuth token operations fail. Used to wrap checked exceptions in + * contexts that don't allow them. + */ +public class OAuthTokenException extends RuntimeException { + public OAuthTokenException(String message) { + super(message); + } + + public OAuthTokenException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthTokenProvider.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthTokenProvider.java new file mode 100644 index 0000000000..241611e432 --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthTokenProvider.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.client.oauth; + +import java.sql.SQLException; + +/** + * Interface for OAuth token providers that handle token acquisition and refresh. Implementations + * should cache tokens and automatically refresh them before expiration. + */ +public interface OAuthTokenProvider { + /** + * Gets a valid OAuth access token, refreshing if necessary. + * + * @return a valid access token string + * @throws SQLException if token cannot be obtained + */ + String getValidToken() throws SQLException; +} diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthTokenProviders.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthTokenProviders.java new file mode 100644 index 0000000000..bbf7072d39 --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthTokenProviders.java @@ -0,0 +1,419 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.client.oauth; + +import com.nimbusds.oauth2.sdk.ParseException; +import com.nimbusds.oauth2.sdk.Scope; +import com.nimbusds.oauth2.sdk.auth.ClientAuthentication; +import com.nimbusds.oauth2.sdk.auth.ClientSecretBasic; +import com.nimbusds.oauth2.sdk.auth.Secret; +import com.nimbusds.oauth2.sdk.id.Audience; +import com.nimbusds.oauth2.sdk.id.ClientID; +import com.nimbusds.oauth2.sdk.token.TokenTypeURI; +import com.nimbusds.oauth2.sdk.token.TypelessAccessToken; +import com.nimbusds.oauth2.sdk.tokenexchange.TokenExchangeGrant; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** + * Unified factory for creating OAuth token providers. + * + *

This class provides a single entry point for creating all OAuth token providers with a + * consistent builder API. It supports: + * + *

    + *
  • Client Credentials flow (RFC 6749 Section 4.4) + *
  • Token Exchange flow (RFC 8693) + *
+ * + *

Example usage: + * + * 

{@code
+ * // Client Credentials flow
+ * OAuthTokenProvider provider = OAuthTokenProviders.clientCredentials()
+ *     .tokenUri("https://auth.example.com/token")
+ *     .clientId("my-client")
+ *     .clientSecret("my-secret")
+ *     .scope("read write")
+ *     .build();
+ *
+ * // Token Exchange flow
+ * OAuthTokenProvider provider = OAuthTokenProviders.tokenExchange()
+ *     .tokenUri("https://auth.example.com/token")
+ *     .subjectToken("user-token")
+ *     .subjectTokenType("urn:ietf:params:oauth:token-type:access_token")
+ *     .build();
+ * }
+ */ +public final class OAuthTokenProviders { + + private OAuthTokenProviders() {} + + /** + * Creates a new builder for Client Credentials flow. + * + * @return a new ClientCredentialsBuilder instance + */ + public static ClientCredentialsBuilder clientCredentials() { + return new ClientCredentialsBuilder(); + } + + /** + * Creates a new builder for Token Exchange flow. + * + * @return a new TokenExchangeBuilder instance + */ + public static TokenExchangeBuilder tokenExchange() { + return new TokenExchangeBuilder(); + } + + /** Builder for creating {@link ClientCredentialsTokenProvider} instances. */ + public static class ClientCredentialsBuilder { + private @Nullable URI tokenUri; + private @Nullable String clientId; + private @Nullable String clientSecret; + private @Nullable String scope; + + ClientCredentialsBuilder() {} + + /** + * Sets the OAuth token endpoint URI (required). + * + * @param tokenUri the token endpoint URI + * @return this builder + */ + public ClientCredentialsBuilder tokenUri(URI tokenUri) { + this.tokenUri = Objects.requireNonNull(tokenUri, "tokenUri cannot be null"); + return this; + } + + /** + * Sets the OAuth token endpoint URI from a string (required). + * + * @param tokenUri the token endpoint URI string + * @return this builder + * @throws IllegalArgumentException if the URI is invalid + */ + public ClientCredentialsBuilder tokenUri(String tokenUri) { + Objects.requireNonNull(tokenUri, "tokenUri cannot be null"); + try { + this.tokenUri = new URI(tokenUri); + } catch (URISyntaxException e) { + throw new IllegalArgumentException("Invalid token URI: " + tokenUri, e); + } + return this; + } + + /** + * Sets the OAuth client ID (required). + * + * @param clientId the client ID + * @return this builder + */ + public ClientCredentialsBuilder clientId(String clientId) { + this.clientId = Objects.requireNonNull(clientId, "clientId cannot be null"); + return this; + } + + /** + * Sets the OAuth client secret (required). + * + * @param clientSecret the client secret + * @return this builder + */ + public ClientCredentialsBuilder clientSecret(String clientSecret) { + this.clientSecret = Objects.requireNonNull(clientSecret, "clientSecret cannot be null"); + return this; + } + + /** + * Sets the OAuth scopes (optional). + * + * @param scope the space-separated scope string + * @return this builder + */ + public ClientCredentialsBuilder scope(@Nullable String scope) { + this.scope = scope; + return this; + } + + /** + * Builds a new ClientCredentialsTokenProvider instance. + * + * @return the configured ClientCredentialsTokenProvider + * @throws IllegalStateException if required parameters are missing + */ + public ClientCredentialsTokenProvider build() { + if (tokenUri == null) { + throw new IllegalStateException("tokenUri is required"); + } + if (clientId == null) { + throw new IllegalStateException("clientId is required"); + } + if (clientSecret == null) { + throw new IllegalStateException("clientSecret is required"); + } + return new ClientCredentialsTokenProvider(tokenUri, clientId, clientSecret, scope); + } + } + + /** Builder for creating {@link TokenExchangeTokenProvider} instances. */ + public static class TokenExchangeBuilder { + private @Nullable URI tokenUri; + private @Nullable String subjectToken; + private @Nullable String subjectTokenType; + private @Nullable String actorToken; + private @Nullable String actorTokenType; + private @Nullable String audience; + private @Nullable String requestedTokenType; + private @Nullable Scope scope; + private @Nullable List resources; + private @Nullable ClientAuthentication clientAuth; + + TokenExchangeBuilder() {} + + /** + * Sets the OAuth token endpoint URI (required). + * + * @param tokenUri the token endpoint URI + * @return this builder + */ + public TokenExchangeBuilder tokenUri(URI tokenUri) { + this.tokenUri = Objects.requireNonNull(tokenUri, "tokenUri cannot be null"); + return this; + } + + /** + * Sets the OAuth token endpoint URI from a string (required). + * + * @param tokenUri the token endpoint URI string + * @return this builder + * @throws IllegalArgumentException if the URI is invalid + */ + public TokenExchangeBuilder tokenUri(String tokenUri) { + Objects.requireNonNull(tokenUri, "tokenUri cannot be null"); + try { + this.tokenUri = new URI(tokenUri); + } catch (URISyntaxException e) { + throw new IllegalArgumentException("Invalid token URI: " + tokenUri, e); + } + return this; + } + + /** + * Sets the subject token to exchange (required). + * + * @param subjectToken the subject token value + * @return this builder + */ + public TokenExchangeBuilder subjectToken(String subjectToken) { + this.subjectToken = Objects.requireNonNull(subjectToken, "subjectToken cannot be null"); + return this; + } + + /** + * Sets the type of the subject token (required). + * + * @param subjectTokenType the subject token type URI + * @return this builder + */ + public TokenExchangeBuilder subjectTokenType(String subjectTokenType) { + this.subjectTokenType = + Objects.requireNonNull(subjectTokenType, "subjectTokenType cannot be null"); + return this; + } + + /** + * Sets the optional actor token for delegation scenarios. + * + * @param actorToken the actor token value + * @return this builder + */ + public TokenExchangeBuilder actorToken(@Nullable String actorToken) { + this.actorToken = actorToken; + return this; + } + + /** + * Sets the type of the actor token. + * + * @param actorTokenType the actor token type URI + * @return this builder + */ + public TokenExchangeBuilder actorTokenType(@Nullable String actorTokenType) { + this.actorTokenType = actorTokenType; + return this; + } + + /** + * Sets the target audience for the exchanged token. + * + * @param audience the target audience + * @return this builder + */ + public TokenExchangeBuilder audience(@Nullable String audience) { + this.audience = audience; + return this; + } + + /** + * Sets the requested token type for the exchanged token. + * + * @param requestedTokenType the requested token type URI + * @return this builder + */ + public TokenExchangeBuilder requestedTokenType(@Nullable String requestedTokenType) { + this.requestedTokenType = requestedTokenType; + return this; + } + + /** + * Sets the OAuth scopes for the token request. + * + * @param scope the OAuth scope object + * @return this builder + */ + public TokenExchangeBuilder scope(@Nullable Scope scope) { + this.scope = scope; + return this; + } + + /** + * Sets the OAuth scopes from a space-separated string. + * + * @param scope the space-separated scope string + * @return this builder + */ + public TokenExchangeBuilder scope(@Nullable String scope) { + this.scope = (scope != null && !scope.isEmpty()) ? Scope.parse(scope) : null; + return this; + } + + /** + * Sets the target resource URIs (RFC 8707). + * + * @param resources the list of resource URIs + * @return this builder + */ + public TokenExchangeBuilder resources(@Nullable List resources) { + this.resources = resources; + return this; + } + + /** + * Sets a single target resource URI (RFC 8707). + * + * @param resource the resource URI + * @return this builder + */ + public TokenExchangeBuilder resource(@Nullable URI resource) { + this.resources = resource != null ? Collections.singletonList(resource) : null; + return this; + } + + /** + * Sets a single target resource URI from a string (RFC 8707). + * + * @param resource the resource URI string + * @return this builder + */ + public TokenExchangeBuilder resource(@Nullable String resource) { + if (resource != null && !resource.isEmpty()) { + this.resources = Collections.singletonList(URI.create(resource)); + } else { + this.resources = null; + } + return this; + } + + /** + * Sets the client authentication. + * + * @param clientAuth the client authentication object + * @return this builder + */ + public TokenExchangeBuilder clientAuthentication(@Nullable ClientAuthentication clientAuth) { + this.clientAuth = clientAuth; + return this; + } + + /** + * Sets client authentication using client ID and secret. + * + * @param clientId the client ID + * @param clientSecret the client secret + * @return this builder + */ + public TokenExchangeBuilder clientCredentials(String clientId, String clientSecret) { + Objects.requireNonNull(clientId, "clientId cannot be null"); + Objects.requireNonNull(clientSecret, "clientSecret cannot be null"); + this.clientAuth = new ClientSecretBasic(new ClientID(clientId), new Secret(clientSecret)); + return this; + } + + /** + * Builds a new TokenExchangeTokenProvider instance. + * + * @return the configured TokenExchangeTokenProvider + * @throws IllegalStateException if required parameters are missing + */ + public TokenExchangeTokenProvider build() { + if (tokenUri == null) { + throw new IllegalStateException("tokenUri is required"); + } + if (subjectToken == null) { + throw new IllegalStateException("subjectToken is required"); + } + if (subjectTokenType == null) { + throw new IllegalStateException("subjectTokenType is required"); + } + + TokenExchangeGrant grant = createGrant(); + return new TokenExchangeTokenProvider(tokenUri, grant, clientAuth, scope, resources); + } + + private TokenExchangeGrant createGrant() { + try { + TypelessAccessToken subjectAccessToken = new TypelessAccessToken(subjectToken); + TokenTypeURI subjectTypeUri = TokenTypeURI.parse(subjectTokenType); + + TypelessAccessToken actorAccessToken = + actorToken != null ? new TypelessAccessToken(actorToken) : null; + TokenTypeURI actorTypeUri = + actorTokenType != null ? TokenTypeURI.parse(actorTokenType) : null; + TokenTypeURI requestedTypeUri = + requestedTokenType != null ? TokenTypeURI.parse(requestedTokenType) : null; + List audienceList = + audience != null ? Collections.singletonList(new Audience(audience)) : null; + + return new TokenExchangeGrant( + subjectAccessToken, + subjectTypeUri, + actorAccessToken, + actorTypeUri, + requestedTypeUri, + audienceList); + } catch (ParseException e) { + throw new IllegalStateException("Failed to create TokenExchangeGrant", e); + } + } + } +} diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/TokenExchangeTokenProvider.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/TokenExchangeTokenProvider.java new file mode 100644 index 0000000000..af433a2712 --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/TokenExchangeTokenProvider.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.client.oauth; + +import com.nimbusds.oauth2.sdk.Scope; +import com.nimbusds.oauth2.sdk.TokenRequest; +import com.nimbusds.oauth2.sdk.auth.ClientAuthentication; +import com.nimbusds.oauth2.sdk.tokenexchange.TokenExchangeGrant; +import java.net.URI; +import java.util.List; +import java.util.Objects; +import org.apache.arrow.util.VisibleForTesting; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** + * OAuth 2.0 Token Exchange flow token provider (RFC 8693). + * + *

This provider exchanges one token for another, commonly used for federated authentication, + * delegation, or impersonation scenarios. Tokens are cached and automatically refreshed. + */ +public class TokenExchangeTokenProvider extends AbstractOAuthTokenProvider { + + @VisibleForTesting TokenExchangeGrant grant; + + @VisibleForTesting @Nullable List resources; + + /** + * Creates a new TokenExchangeTokenProvider with full configuration. + * + * @param tokenUri the OAuth token endpoint URI + * @param grant the token exchange grant containing subject/actor token information + * @param clientAuth optional client authentication + * @param scope optional OAuth scopes + * @param resource optional target resource URI (RFC 8707) + */ + TokenExchangeTokenProvider( + URI tokenUri, + TokenExchangeGrant grant, + @Nullable ClientAuthentication clientAuth, + @Nullable Scope scope, + @Nullable List resource) { + this.tokenUri = Objects.requireNonNull(tokenUri, "tokenUri cannot be null"); + this.grant = Objects.requireNonNull(grant, "grant cannot be null"); + this.scope = scope; + this.resources = resource; + this.clientAuth = clientAuth; + } + + @Override + protected TokenRequest buildTokenRequest() { + TokenRequest.Builder builder; + if (clientAuth != null) { + builder = new TokenRequest.Builder(tokenUri, clientAuth, grant); + } else { + builder = new TokenRequest.Builder(tokenUri, grant); + } + + if (scope != null) { + builder.scope(scope); + } + if (resources != null) { + builder.resources(resources.toArray(new URI[0])); + } + + return builder.build(); + } +} diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/TokenInfo.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/TokenInfo.java new file mode 100644 index 0000000000..f47cc8b053 --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/TokenInfo.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.client.oauth; + +import java.time.Instant; +import java.util.Objects; + +/** Holds OAuth token information including the access token and expiration time. */ +public class TokenInfo { + private final String accessToken; + private final Instant expiresAt; + + public TokenInfo(String accessToken, Instant expiresAt) { + this.accessToken = Objects.requireNonNull(accessToken, "accessToken cannot be null"); + this.expiresAt = Objects.requireNonNull(expiresAt, "expiresAt cannot be null"); + } + + public String getAccessToken() { + return accessToken; + } + + /** + * Checks if the token is expired or will expire within the buffer period. + * + * @param bufferSeconds seconds before actual expiration to consider token expired + * @return true if token should be refreshed + */ + public boolean isExpired(int bufferSeconds) { + return Instant.now().plusSeconds(bufferSeconds).isAfter(expiresAt); + } +} diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java index 76ba964a53..d0ba74dbcc 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.driver.jdbc.utils; +import java.sql.SQLException; import java.time.Duration; import java.util.Arrays; import java.util.HashMap; @@ -23,6 +24,7 @@ import java.util.Objects; import java.util.Properties; import org.apache.arrow.driver.jdbc.ArrowFlightConnection; +import org.apache.arrow.driver.jdbc.client.oauth.OAuthConfiguration; import org.apache.arrow.flight.CallHeaders; import org.apache.arrow.flight.CallOption; import org.apache.arrow.flight.FlightCallHeaders; @@ -31,6 +33,7 @@ import org.apache.calcite.avatica.ConnectionConfig; import org.apache.calcite.avatica.ConnectionConfigImpl; import org.apache.calcite.avatica.ConnectionProperty; +import org.checkerframework.checker.nullness.qual.Nullable; /** A {@link ConnectionConfig} for the {@link ArrowFlightConnection}. */ public final class ArrowFlightConnectionConfigImpl extends ConnectionConfigImpl { @@ -211,6 +214,38 @@ public Map getHeaderAttributes() { return headers; } + /** + * Returns OAuth configuration if oauth.flow is specified, null otherwise. + * + * @return the OAuth configuration or null + * @throws SQLException if the OAuth configuration is invalid + */ + public @Nullable OAuthConfiguration getOauthConfiguration() throws SQLException { + String flow = ArrowFlightConnectionProperty.OAUTH_FLOW.getString(properties); + if (flow == null) { + return null; + } + + return new OAuthConfiguration.Builder() + .flow(flow) + .clientId(ArrowFlightConnectionProperty.OAUTH_CLIENT_ID.getString(properties)) + .clientSecret(ArrowFlightConnectionProperty.OAUTH_CLIENT_SECRET.getString(properties)) + .tokenUri(ArrowFlightConnectionProperty.OAUTH_TOKEN_URI.getString(properties)) + .scope(ArrowFlightConnectionProperty.OAUTH_SCOPE.getString(properties)) + .resource(ArrowFlightConnectionProperty.OAUTH_RESOURCE.getString(properties)) + .subjectToken( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_SUBJECT_TOKEN.getString(properties)) + .subjectTokenType( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_SUBJECT_TOKEN_TYPE.getString(properties)) + .actorToken(ArrowFlightConnectionProperty.OAUTH_EXCHANGE_ACTOR_TOKEN.getString(properties)) + .actorTokenType( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_ACTOR_TOKEN_TYPE.getString(properties)) + .audience(ArrowFlightConnectionProperty.OAUTH_EXCHANGE_AUDIENCE.getString(properties)) + .requestedTokenType( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_REQUESTED_TOKEN_TYPE.getString(properties)) + .build(); + } + /** Custom {@link ConnectionProperty} for the {@link ArrowFlightConnectionConfigImpl}. */ public enum ArrowFlightConnectionProperty implements ConnectionProperty { HOST("host", null, Type.STRING, true), @@ -232,6 +267,23 @@ public enum ArrowFlightConnectionProperty implements ConnectionProperty { CATALOG("catalog", null, Type.STRING, false), CONNECT_TIMEOUT_MILLIS("connectTimeoutMs", 10000, Type.NUMBER, false), USE_CLIENT_CACHE("useClientCache", true, Type.BOOLEAN, false), + + // OAuth configuration properties + OAUTH_FLOW("oauth.flow", null, Type.STRING, false), + OAUTH_CLIENT_ID("oauth.clientId", null, Type.STRING, false), + OAUTH_CLIENT_SECRET("oauth.clientSecret", null, Type.STRING, false), + OAUTH_TOKEN_URI("oauth.tokenUri", null, Type.STRING, false), + OAUTH_SCOPE("oauth.scope", null, Type.STRING, false), + OAUTH_RESOURCE("oauth.resource", null, Type.STRING, false), + + // Token exchange specific properties + OAUTH_EXCHANGE_SUBJECT_TOKEN("oauth.exchange.subjectToken", null, Type.STRING, false), + OAUTH_EXCHANGE_SUBJECT_TOKEN_TYPE("oauth.exchange.subjectTokenType", null, Type.STRING, false), + OAUTH_EXCHANGE_ACTOR_TOKEN("oauth.exchange.actorToken", null, Type.STRING, false), + OAUTH_EXCHANGE_ACTOR_TOKEN_TYPE("oauth.exchange.actorTokenType", null, Type.STRING, false), + OAUTH_EXCHANGE_AUDIENCE("oauth.exchange.aud", null, Type.STRING, false), + OAUTH_EXCHANGE_REQUESTED_TOKEN_TYPE( + "oauth.exchange.requestedTokenType", null, Type.STRING, false), ; private final String camelName; diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/OAuthIntegrationTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/OAuthIntegrationTest.java new file mode 100644 index 0000000000..5e782db031 --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/OAuthIntegrationTest.java @@ -0,0 +1,474 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.URI; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import mockwebserver3.MockResponse; +import mockwebserver3.MockWebServer; +import mockwebserver3.RecordedRequest; +import mockwebserver3.junit5.StartStop; +import org.apache.arrow.driver.jdbc.authentication.TokenAuthentication; +import org.apache.arrow.driver.jdbc.utils.ArrowFlightConnectionConfigImpl.ArrowFlightConnectionProperty; +import org.apache.arrow.driver.jdbc.utils.MockFlightSqlProducer; +import org.apache.arrow.flight.sql.FlightSqlProducer.Schemas; +import org.apache.arrow.flight.sql.impl.FlightSql.CommandGetCatalogs; +import org.apache.arrow.flight.sql.impl.FlightSql.CommandGetDbSchemas; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.util.AutoCloseables; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * Integration tests for OAuth authentication flows in the JDBC driver. + * + *

These tests verify that OAuth tokens obtained from an OAuth server are correctly used in + * Flight SQL requests. + */ +public class OAuthIntegrationTest { + + private static final String VALID_ACCESS_TOKEN = "valid-oauth-access-token-12345"; + private static final String CLIENT_ID = "test-client-id"; + private static final String CLIENT_SECRET = "test-client-secret"; + private static final String SUBJECT_TOKEN = "original-subject-token"; + private static final String SUBJECT_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:jwt"; + private static final String TEST_SCOPE = "dremio.all"; + + private static final MockFlightSqlProducer FLIGHT_SQL_PRODUCER = new MockFlightSqlProducer(); + + @RegisterExtension public static FlightServerTestExtension FLIGHT_SERVER_TEST_EXTENSION; + + static { + FLIGHT_SERVER_TEST_EXTENSION = + new FlightServerTestExtension.Builder() + .authentication(new TokenAuthentication.Builder().token(VALID_ACCESS_TOKEN).build()) + .producer(FLIGHT_SQL_PRODUCER) + .build(); + } + + @StartStop private final MockWebServer oauthServer = new MockWebServer(); + private URI tokenEndpoint; + + @BeforeAll + public static void setUpClass() { + // Register a simple catalog query handler + FLIGHT_SQL_PRODUCER.addCatalogQuery( + CommandGetCatalogs.getDefaultInstance(), + listener -> { + try (BufferAllocator allocator = new RootAllocator(); + VectorSchemaRoot root = + VectorSchemaRoot.create(Schemas.GET_CATALOGS_SCHEMA, allocator)) { + root.setRowCount(0); + listener.start(root); + listener.putNext(); + } catch (Throwable t) { + listener.error(t); + } finally { + listener.completed(); + } + }); + + // Register a simple schema query handler for getSchemas() + FLIGHT_SQL_PRODUCER.addCatalogQuery( + CommandGetDbSchemas.getDefaultInstance(), + listener -> { + try (BufferAllocator allocator = new RootAllocator(); + VectorSchemaRoot root = + VectorSchemaRoot.create(Schemas.GET_SCHEMAS_SCHEMA, allocator)) { + root.setRowCount(0); + listener.start(root); + listener.putNext(); + } catch (Throwable t) { + listener.error(t); + } finally { + listener.completed(); + } + }); + } + + @AfterAll + public static void tearDownClass() { + AutoCloseables.closeNoChecked(FLIGHT_SQL_PRODUCER); + } + + @BeforeEach + public void setUp() { + tokenEndpoint = oauthServer.url("/oauth/token").uri(); + } + + @AfterEach + public void tearDown() { + oauthServer.close(); + } + + // Helper methods for mock OAuth responses + + private void enqueueSuccessfulTokenResponse() { + enqueueSuccessfulTokenResponse(VALID_ACCESS_TOKEN, 3600); + } + + private void enqueueSuccessfulTokenResponse(String token, int expiresIn) { + String body = + String.format( + "{\"access_token\":\"%s\",\"token_type\":\"Bearer\",\"expires_in\":%d}", + token, expiresIn); + oauthServer.enqueue( + new MockResponse.Builder() + .code(200) + .setHeader("Content-Type", "application/json") + .body(body) + .build()); + } + + private void enqueueErrorResponse(String error, String description) { + String body = + String.format("{\"error\":\"%s\",\"error_description\":\"%s\"}", error, description); + oauthServer.enqueue( + new MockResponse.Builder() + .code(400) + .setHeader("Content-Type", "application/json") + .body(body) + .build()); + } + + private Properties createBaseProperties() { + Properties props = new Properties(); + props.put(ArrowFlightConnectionProperty.HOST.camelName(), "localhost"); + props.put( + ArrowFlightConnectionProperty.PORT.camelName(), FLIGHT_SERVER_TEST_EXTENSION.getPort()); + props.put(ArrowFlightConnectionProperty.USE_ENCRYPTION.camelName(), false); + return props; + } + + private String getJdbcUrl() { + return String.format( + "jdbc:arrow-flight-sql://localhost:%d", FLIGHT_SERVER_TEST_EXTENSION.getPort()); + } + + // ==================== Client Credentials Flow Tests ==================== + + @Test + public void testClientCredentialsFlowSuccess() throws Exception { + enqueueSuccessfulTokenResponse(); + + Properties props = createBaseProperties(); + props.put(ArrowFlightConnectionProperty.OAUTH_FLOW.camelName(), "client_credentials"); + props.put(ArrowFlightConnectionProperty.OAUTH_TOKEN_URI.camelName(), tokenEndpoint.toString()); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_ID.camelName(), CLIENT_ID); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_SECRET.camelName(), CLIENT_SECRET); + props.put(ArrowFlightConnectionProperty.OAUTH_SCOPE.camelName(), TEST_SCOPE); + + try (Connection conn = DriverManager.getConnection(getJdbcUrl(), props)) { + assertFalse(conn.isClosed()); + // Trigger a Flight call to force OAuth token retrieval + conn.getMetaData().getCatalogs().close(); + } + + // Verify OAuth request was made + RecordedRequest oauthRequest = oauthServer.takeRequest(5, TimeUnit.SECONDS); + assertNotNull(oauthRequest, "OAuth request should have been made"); + assertEquals("POST", oauthRequest.getMethod()); + String body = oauthRequest.getBody().utf8(); + assertTrue(body.contains("grant_type=client_credentials")); + assertTrue(body.contains("scope=" + TEST_SCOPE)); + } + + @Test + public void testClientCredentialsFlowWithUrlParameters() throws Exception { + enqueueSuccessfulTokenResponse(); + + String url = + String.format( + "jdbc:arrow-flight-sql://localhost:%d?useEncryption=false" + + "&oauth.flow=client_credentials" + + "&oauth.tokenUri=%s" + + "&oauth.clientId=%s" + + "&oauth.clientSecret=%s", + FLIGHT_SERVER_TEST_EXTENSION.getPort(), + tokenEndpoint.toString(), + CLIENT_ID, + CLIENT_SECRET); + + try (Connection conn = DriverManager.getConnection(url)) { + conn.getMetaData().getCatalogs().close(); + } + + assertEquals(1, oauthServer.getRequestCount()); + } + + @Test + public void testClientCredentialsFlowInvalidCredentials() throws Exception { + enqueueErrorResponse("invalid_client", "Client authentication failed"); + + Properties props = createBaseProperties(); + props.put(ArrowFlightConnectionProperty.OAUTH_FLOW.camelName(), "client_credentials"); + props.put(ArrowFlightConnectionProperty.OAUTH_TOKEN_URI.camelName(), tokenEndpoint.toString()); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_ID.camelName(), "wrong-client"); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_SECRET.camelName(), "wrong-secret"); + + Exception ex = + assertThrows( + Exception.class, + () -> { + try (Connection conn = DriverManager.getConnection(getJdbcUrl(), props)) { + conn.getMetaData().getCatalogs().close(); + } + }); + // Verify the error message contains the OAuth error somewhere in the exception chain + assertTrue( + containsInExceptionChain(ex, "invalid_client"), + "Exception chain should contain 'invalid_client'"); + } + + private boolean containsInExceptionChain(Throwable t, String message) { + while (t != null) { + if (t.getMessage() != null && t.getMessage().contains(message)) { + return true; + } + t = t.getCause(); + } + return false; + } + + // ==================== Token Exchange Flow Tests ==================== + + @Test + public void testTokenExchangeFlowMinimalParameters() throws Exception { + enqueueSuccessfulTokenResponse(); + + Properties props = createBaseProperties(); + props.put(ArrowFlightConnectionProperty.OAUTH_FLOW.camelName(), "token_exchange"); + props.put(ArrowFlightConnectionProperty.OAUTH_TOKEN_URI.camelName(), tokenEndpoint.toString()); + props.put( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_SUBJECT_TOKEN.camelName(), SUBJECT_TOKEN); + props.put( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_SUBJECT_TOKEN_TYPE.camelName(), + SUBJECT_TOKEN_TYPE); + + try (Connection conn = DriverManager.getConnection(getJdbcUrl(), props)) { + conn.getMetaData().getCatalogs().close(); + } + + RecordedRequest oauthRequest = oauthServer.takeRequest(5, TimeUnit.SECONDS); + assertNotNull(oauthRequest, "OAuth request should have been made"); + String body = oauthRequest.getBody().utf8(); + assertTrue( + body.contains("grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Atoken-exchange"), + "Should contain token exchange grant type"); + assertTrue(body.contains("subject_token=" + SUBJECT_TOKEN)); + } + + @Test + public void testTokenExchangeFlowWithAllParameters() throws Exception { + enqueueSuccessfulTokenResponse(); + + String actorToken = "actor-token-value"; + String actorTokenType = "urn:ietf:params:oauth:token-type:access_token"; + String audience = "https://api.example.com"; + String resource = "https://api.example.com/resource"; + String requestedTokenType = "urn:ietf:params:oauth:token-type:access_token"; + + Properties props = createBaseProperties(); + props.put(ArrowFlightConnectionProperty.OAUTH_FLOW.camelName(), "token_exchange"); + props.put(ArrowFlightConnectionProperty.OAUTH_TOKEN_URI.camelName(), tokenEndpoint.toString()); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_ID.camelName(), CLIENT_ID); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_SECRET.camelName(), CLIENT_SECRET); + props.put(ArrowFlightConnectionProperty.OAUTH_SCOPE.camelName(), TEST_SCOPE); + props.put( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_SUBJECT_TOKEN.camelName(), SUBJECT_TOKEN); + props.put( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_SUBJECT_TOKEN_TYPE.camelName(), + SUBJECT_TOKEN_TYPE); + props.put(ArrowFlightConnectionProperty.OAUTH_EXCHANGE_ACTOR_TOKEN.camelName(), actorToken); + props.put( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_ACTOR_TOKEN_TYPE.camelName(), actorTokenType); + props.put(ArrowFlightConnectionProperty.OAUTH_EXCHANGE_AUDIENCE.camelName(), audience); + props.put(ArrowFlightConnectionProperty.OAUTH_RESOURCE.camelName(), resource); + props.put( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_REQUESTED_TOKEN_TYPE.camelName(), + requestedTokenType); + + try (Connection conn = DriverManager.getConnection(getJdbcUrl(), props)) { + conn.getMetaData().getCatalogs().close(); + } + + RecordedRequest oauthRequest = oauthServer.takeRequest(5, TimeUnit.SECONDS); + assertNotNull(oauthRequest, "OAuth request should have been made"); + String body = oauthRequest.getBody().utf8(); + assertTrue(body.contains("subject_token=" + SUBJECT_TOKEN)); + assertTrue(body.contains("actor_token=" + actorToken)); + } + + @Test + public void testTokenExchangeFlowWithClientAuthentication() throws Exception { + enqueueSuccessfulTokenResponse(); + + Properties props = createBaseProperties(); + props.put(ArrowFlightConnectionProperty.OAUTH_FLOW.camelName(), "token_exchange"); + props.put(ArrowFlightConnectionProperty.OAUTH_TOKEN_URI.camelName(), tokenEndpoint.toString()); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_ID.camelName(), CLIENT_ID); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_SECRET.camelName(), CLIENT_SECRET); + props.put( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_SUBJECT_TOKEN.camelName(), SUBJECT_TOKEN); + props.put( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_SUBJECT_TOKEN_TYPE.camelName(), + SUBJECT_TOKEN_TYPE); + + try (Connection conn = DriverManager.getConnection(getJdbcUrl(), props)) { + conn.getMetaData().getCatalogs().close(); + } + + RecordedRequest oauthRequest = oauthServer.takeRequest(5, TimeUnit.SECONDS); + assertNotNull(oauthRequest, "OAuth request should have been made"); + String authHeader = oauthRequest.getHeaders().get("Authorization"); + assertNotNull(authHeader, "Should have Basic auth header for client authentication"); + assertTrue(authHeader.startsWith("Basic ")); + } + + // ==================== Token Caching Tests ==================== + + @Test + public void testTokenCachingAcrossMultipleOperations() throws Exception { + enqueueSuccessfulTokenResponse(); + + Properties props = createBaseProperties(); + props.put(ArrowFlightConnectionProperty.OAUTH_FLOW.camelName(), "client_credentials"); + props.put(ArrowFlightConnectionProperty.OAUTH_TOKEN_URI.camelName(), tokenEndpoint.toString()); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_ID.camelName(), CLIENT_ID); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_SECRET.camelName(), CLIENT_SECRET); + + try (Connection conn = DriverManager.getConnection(getJdbcUrl(), props)) { + // Execute multiple operations + conn.isValid(5); + conn.getMetaData().getCatalogs().close(); + conn.getMetaData().getSchemas().close(); + } + + // Should only have made one OAuth request due to caching + assertEquals(1, oauthServer.getRequestCount()); + } + + @Test + public void testTokenRefreshAfterExpiration() throws Exception { + enqueueSuccessfulTokenResponse(VALID_ACCESS_TOKEN, 1); + enqueueSuccessfulTokenResponse(VALID_ACCESS_TOKEN, 3600); + + Properties props = createBaseProperties(); + props.put(ArrowFlightConnectionProperty.OAUTH_FLOW.camelName(), "token_exchange"); + props.put(ArrowFlightConnectionProperty.OAUTH_TOKEN_URI.camelName(), tokenEndpoint.toString()); + props.put( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_SUBJECT_TOKEN.camelName(), SUBJECT_TOKEN); + props.put( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_SUBJECT_TOKEN_TYPE.camelName(), + SUBJECT_TOKEN_TYPE); + + try (Connection conn = DriverManager.getConnection(getJdbcUrl(), props)) { + // First operation triggers initial token fetch + conn.getMetaData().getCatalogs().close(); + + // Token with 1s expiry is immediately considered expired (due to 30s buffer) + // so the next operation should trigger a refresh + conn.getMetaData().getCatalogs().close(); + } + + // Should have made exactly 2 OAuth requests: initial + refresh + assertEquals(2, oauthServer.getRequestCount()); + } + + // ==================== Error Handling Tests ==================== + + @Test + public void testMissingRequiredParametersClientCredentials() { + Properties props = createBaseProperties(); + props.put(ArrowFlightConnectionProperty.OAUTH_FLOW.camelName(), "client_credentials"); + props.put(ArrowFlightConnectionProperty.OAUTH_TOKEN_URI.camelName(), tokenEndpoint.toString()); + // Missing client_id and client_secret + + assertThrows(SQLException.class, () -> DriverManager.getConnection(getJdbcUrl(), props)); + } + + @Test + public void testMissingRequiredParametersTokenExchange() { + Properties props = createBaseProperties(); + props.put(ArrowFlightConnectionProperty.OAUTH_FLOW.camelName(), "token_exchange"); + props.put(ArrowFlightConnectionProperty.OAUTH_TOKEN_URI.camelName(), tokenEndpoint.toString()); + // Missing subject_token and subject_token_type + + assertThrows(SQLException.class, () -> DriverManager.getConnection(getJdbcUrl(), props)); + } + + @Test + public void testInvalidOAuthFlow() { + Properties props = createBaseProperties(); + props.put(ArrowFlightConnectionProperty.OAUTH_FLOW.camelName(), "invalid_flow"); + props.put(ArrowFlightConnectionProperty.OAUTH_TOKEN_URI.camelName(), tokenEndpoint.toString()); + + assertThrows(SQLException.class, () -> DriverManager.getConnection(getJdbcUrl(), props)); + } + + @Test + public void testMalformedTokenEndpoint() { + Properties props = createBaseProperties(); + props.put(ArrowFlightConnectionProperty.OAUTH_FLOW.camelName(), "client_credentials"); + props.put(ArrowFlightConnectionProperty.OAUTH_TOKEN_URI.camelName(), "not-a-valid-uri://"); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_ID.camelName(), CLIENT_ID); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_SECRET.camelName(), CLIENT_SECRET); + + assertThrows(SQLException.class, () -> DriverManager.getConnection(getJdbcUrl(), props)); + } + + // ==================== Authorization Header Verification ==================== + + @Test + public void testOAuthTokenSentAsBearer() throws Exception { + enqueueSuccessfulTokenResponse(); + + Properties props = createBaseProperties(); + props.put(ArrowFlightConnectionProperty.OAUTH_FLOW.camelName(), "client_credentials"); + props.put(ArrowFlightConnectionProperty.OAUTH_TOKEN_URI.camelName(), tokenEndpoint.toString()); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_ID.camelName(), CLIENT_ID); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_SECRET.camelName(), CLIENT_SECRET); + + try (Connection conn = DriverManager.getConnection(getJdbcUrl(), props)) { + conn.getMetaData().getCatalogs().close(); + } + + // Verify the Flight server received the bearer token + String authHeader = + FLIGHT_SERVER_TEST_EXTENSION + .getInterceptorFactory() + .getHeader(org.apache.arrow.flight.FlightMethod.GET_FLIGHT_INFO, "authorization"); + assertNotNull(authHeader, "Authorization header should be present in Flight requests"); + assertEquals("Bearer " + VALID_ACCESS_TOKEN, authHeader); + } +} diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthConfigurationTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthConfigurationTest.java new file mode 100644 index 0000000000..c258a7c652 --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthConfigurationTest.java @@ -0,0 +1,296 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.client.oauth; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.nimbusds.oauth2.sdk.Scope; +import java.net.URI; +import java.sql.SQLException; +import java.util.Collections; +import java.util.stream.Stream; +import org.junit.jupiter.api.Named; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** Tests for {@link OAuthConfiguration}. */ +public class OAuthConfigurationTest { + + private static final String TOKEN_URI = "https://auth.example.com/token"; + private static final String CLIENT_ID = "test-client-id"; + private static final String CLIENT_SECRET = "test-client-secret"; + private static final String SCOPE = "read write"; + private static final String SUBJECT_TOKEN = "subject-token-value"; + public static final String RESOURCE = "https://api.example.com/resource"; + + @FunctionalInterface + interface BuilderConfigurer { + void configure(OAuthConfiguration.Builder builder) throws SQLException; + } + + static Stream createFlowCases() { + return Stream.of( + Arguments.of( + Named.of( + "string flow", (BuilderConfigurer) builder -> builder.flow("client_credentials"))), + Arguments.of( + Named.of( + "uppercase string flow", + (BuilderConfigurer) builder -> builder.flow("CLIENT_CREDENTIALS")))); + } + + @ParameterizedTest + @MethodSource("createFlowCases") + public void testCreateFlowConfiguration(BuilderConfigurer flowConfigurer) throws SQLException { + OAuthConfiguration.Builder builder = new OAuthConfiguration.Builder(); + flowConfigurer.configure(builder); + OAuthConfiguration config = + builder.tokenUri(TOKEN_URI).clientId(CLIENT_ID).clientSecret(CLIENT_SECRET).build(); + + // Verify configuration creates correct provider type + OAuthTokenProvider provider = config.createTokenProvider(); + assertInstanceOf(ClientCredentialsTokenProvider.class, provider); + } + + @Test + public void testCreateClientCredentialsTokenProvider() throws SQLException { + OAuthConfiguration config = + new OAuthConfiguration.Builder() + .flow("client_credentials") + .tokenUri(TOKEN_URI) + .clientId(CLIENT_ID) + .clientSecret(CLIENT_SECRET) + .scope(SCOPE) + .build(); + + OAuthTokenProvider provider = config.createTokenProvider(); + + assertNotNull(provider); + assertInstanceOf(ClientCredentialsTokenProvider.class, provider); + + ClientCredentialsTokenProvider ccProvider = (ClientCredentialsTokenProvider) provider; + assertEquals(URI.create(TOKEN_URI), ccProvider.tokenUri); + assertEquals(CLIENT_ID, ccProvider.clientAuth.getClientID().getValue()); + assertEquals(Scope.parse(SCOPE), ccProvider.scope); + } + + @Test + public void testCreateTokenExchangeTokenProviderWithAllOptions() throws SQLException { + String subjectTokenType = "urn:ietf:params:oauth:token-type:access_token"; + String actorToken = "actor-token-value"; + String actorTokenType = "urn:ietf:params:oauth:token-type:jwt"; + String audience = "https://api.example.com"; + String requestedTokenType = "urn:ietf:params:oauth:token-type:access_token"; + + OAuthConfiguration config = + new OAuthConfiguration.Builder() + .flow("token_exchange") + .tokenUri(TOKEN_URI) + .scope(SCOPE) + .clientId(CLIENT_ID) + .clientSecret(CLIENT_SECRET) + .resource(RESOURCE) + .subjectToken(SUBJECT_TOKEN) + .subjectTokenType(subjectTokenType) + .actorToken(actorToken) + .actorTokenType(actorTokenType) + .audience(audience) + .requestedTokenType(requestedTokenType) + .build(); + + OAuthTokenProvider provider = config.createTokenProvider(); + + assertNotNull(provider); + assertInstanceOf(TokenExchangeTokenProvider.class, provider); + + TokenExchangeTokenProvider teProvider = (TokenExchangeTokenProvider) provider; + assertEquals(URI.create(TOKEN_URI), teProvider.tokenUri); + assertNotNull(teProvider.grant); + assertEquals(SUBJECT_TOKEN, teProvider.grant.getSubjectToken().getValue()); + assertEquals(subjectTokenType, teProvider.grant.getSubjectTokenType().getURI().toString()); + assertEquals(actorToken, teProvider.grant.getActorToken().getValue()); + assertEquals(actorTokenType, teProvider.grant.getActorTokenType().getURI().toString()); + assertNotNull(teProvider.grant.getAudience()); + assertEquals(1, teProvider.grant.getAudience().size()); + assertEquals(audience, teProvider.grant.getAudience().get(0).getValue()); + assertEquals(requestedTokenType, teProvider.grant.getRequestedTokenType().getURI().toString()); + assertEquals(Scope.parse(SCOPE), teProvider.scope); + assertEquals(Collections.singletonList(URI.create(RESOURCE)), teProvider.resources); + + assertEquals(CLIENT_ID, teProvider.clientAuth.getClientID().getValue()); + } + + static Stream generalValidationErrorCases() { + return Stream.of( + Arguments.of( + Named.of( + "null flow", + (BuilderConfigurer) builder -> builder.flow((String) null).tokenUri(TOKEN_URI)), + "OAuth flow cannot be null or empty"), + Arguments.of( + Named.of( + "empty flow", (BuilderConfigurer) builder -> builder.flow("").tokenUri(TOKEN_URI)), + "OAuth flow cannot be null or empty"), + Arguments.of( + Named.of( + "invalid flow", + (BuilderConfigurer) builder -> builder.flow("invalid_flow").tokenUri(TOKEN_URI)), + "Unsupported OAuth flow: invalid_flow"), + Arguments.of( + Named.of( + "null tokenUri", + (BuilderConfigurer) + builder -> + builder + .flow("client_credentials") + .tokenUri((String) null) + .clientId(CLIENT_ID) + .clientSecret(CLIENT_SECRET)), + "Token URI cannot be null or empty"), + Arguments.of( + Named.of( + "empty tokenUri", + (BuilderConfigurer) + builder -> + builder + .flow("client_credentials") + .tokenUri("") + .clientId(CLIENT_ID) + .clientSecret(CLIENT_SECRET)), + "Token URI cannot be null or empty"), + Arguments.of( + Named.of( + "invalid tokenUri", + (BuilderConfigurer) + builder -> + builder + .flow("client_credentials") + .tokenUri("not a valid uri ://") + .clientId(CLIENT_ID) + .clientSecret(CLIENT_SECRET)), + null), + Arguments.of( + Named.of( + "invalid tokenUri", + (BuilderConfigurer) + builder -> + builder.flow("client_credentials").tokenUri(TOKEN_URI).clientId(CLIENT_ID)), + // null means verify exception has message and cause + "clientSecret is required for client_credentials flow")); + } + + @ParameterizedTest + @MethodSource("generalValidationErrorCases") + public void testGeneralValidationErrors(BuilderConfigurer configurer, String expectedMessage) { + SQLException exception = + assertThrows( + SQLException.class, + () -> { + OAuthConfiguration.Builder builder = new OAuthConfiguration.Builder(); + configurer.configure(builder); + builder.build(); + }); + + if (expectedMessage != null) { + assertEquals(expectedMessage, exception.getMessage()); + } else { + assertNotNull(exception.getMessage()); + assertNotNull(exception.getCause()); + } + } + + static Stream flowSpecificValidationErrorCases() { + return Stream.of( + // client_credentials flow validation + Arguments.of( + Named.of( + "client_credentials: missing clientId", + (BuilderConfigurer) + builder -> + builder + .flow("client_credentials") + .tokenUri(TOKEN_URI) + .clientSecret(CLIENT_SECRET)), + "clientId is required for client_credentials flow"), + Arguments.of( + Named.of( + "client_credentials: missing clientSecret", + (BuilderConfigurer) + builder -> + builder.flow("client_credentials").tokenUri(TOKEN_URI).clientId(CLIENT_ID)), + "clientSecret is required for client_credentials flow"), + // token_exchange flow validation + Arguments.of( + Named.of( + "token_exchange: missing subjectToken", + (BuilderConfigurer) builder -> builder.flow("token_exchange").tokenUri(TOKEN_URI)), + "subjectToken is required for token_exchange flow"), + Arguments.of( + Named.of( + "token_exchange: empty subjectToken", + (BuilderConfigurer) + builder -> + builder + .flow("token_exchange") + .tokenUri(TOKEN_URI) + .subjectToken("") + .subjectTokenType("urn:ietf:params:oauth:token-type:access_token")), + "subjectToken is required for token_exchange flow"), + Arguments.of( + Named.of( + "token_exchange: missing subjectTokenType", + (BuilderConfigurer) + builder -> + builder + .flow("token_exchange") + .tokenUri(TOKEN_URI) + .subjectToken(SUBJECT_TOKEN)), + "subjectTokenType is required for token_exchange flow"), + Arguments.of( + Named.of( + "token_exchange: empty subjectTokenType", + (BuilderConfigurer) + builder -> + builder + .flow("token_exchange") + .tokenUri(TOKEN_URI) + .subjectToken(SUBJECT_TOKEN) + .subjectTokenType("")), + "subjectTokenType is required for token_exchange flow")); + } + + @ParameterizedTest + @MethodSource("flowSpecificValidationErrorCases") + public void testFlowSpecificValidationErrors( + BuilderConfigurer configurer, String expectedMessage) { + SQLException exception = + assertThrows( + SQLException.class, + () -> { + OAuthConfiguration.Builder builder = new OAuthConfiguration.Builder(); + configurer.configure(builder); + builder.build(); + }); + + assertEquals(expectedMessage, exception.getMessage()); + } +} diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthCredentialWriterTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthCredentialWriterTest.java new file mode 100644 index 0000000000..1a33f7f0ae --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthCredentialWriterTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.client.oauth; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.SQLException; +import org.apache.arrow.flight.CallHeaders; +import org.apache.arrow.flight.FlightCallHeaders; +import org.apache.arrow.flight.auth2.Auth2Constants; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +/** Tests for {@link OAuthCredentialWriter}. */ +@ExtendWith(MockitoExtension.class) +public class OAuthCredentialWriterTest { + + @Mock private OAuthTokenProvider mockTokenProvider; + + @Test + public void testConstructorRejectsNullTokenProvider() { + assertThrows(NullPointerException.class, () -> new OAuthCredentialWriter(null)); + } + + @Test + public void testAcceptWritesBearerTokenToHeaders() throws SQLException { + String testToken = "test-access-token-12345"; + when(mockTokenProvider.getValidToken()).thenReturn(testToken); + + OAuthCredentialWriter writer = new OAuthCredentialWriter(mockTokenProvider); + CallHeaders headers = new FlightCallHeaders(); + + writer.accept(headers); + + verify(mockTokenProvider).getValidToken(); + assertEquals( + Auth2Constants.BEARER_PREFIX + testToken, headers.get(Auth2Constants.AUTHORIZATION_HEADER)); + } + + @Test + public void testAcceptThrowsOAuthTokenExceptionOnSQLException() throws SQLException { + SQLException sqlException = new SQLException("Token fetch failed"); + when(mockTokenProvider.getValidToken()).thenThrow(sqlException); + + OAuthCredentialWriter writer = new OAuthCredentialWriter(mockTokenProvider); + CallHeaders headers = new FlightCallHeaders(); + + OAuthTokenException exception = + assertThrows(OAuthTokenException.class, () -> writer.accept(headers)); + + assertEquals("Failed to obtain OAuth token", exception.getMessage()); + assertEquals(sqlException, exception.getCause()); + } + + @Test + public void testAcceptCallsTokenProviderEachTime() throws SQLException { + when(mockTokenProvider.getValidToken()) + .thenReturn("token1") + .thenReturn("token2") + .thenReturn("token3"); + + OAuthCredentialWriter writer = new OAuthCredentialWriter(mockTokenProvider); + + CallHeaders headers1 = new FlightCallHeaders(); + writer.accept(headers1); + assertEquals("Bearer token1", headers1.get(Auth2Constants.AUTHORIZATION_HEADER)); + + CallHeaders headers2 = new FlightCallHeaders(); + writer.accept(headers2); + assertEquals("Bearer token2", headers2.get(Auth2Constants.AUTHORIZATION_HEADER)); + + CallHeaders headers3 = new FlightCallHeaders(); + writer.accept(headers3); + assertEquals("Bearer token3", headers3.get(Auth2Constants.AUTHORIZATION_HEADER)); + } +} diff --git a/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt b/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt index 8bc43cbe0f..8476bd9995 100644 --- a/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt +++ b/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt @@ -345,6 +345,14 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- +This binary artifact contains Nimbus OAuth 2.0 SDK with OpenID Connect extensions 11.20.1. + +Copyright: Copyright 2012-2024 Connect2id Ltd. +Home page: https://connect2id.com/products/nimbus-oauth-openid-connect-sdk +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + This binary artifact contains Bouncycastle 1.80. Copyright: Copyright (c) 2000-2024 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org). diff --git a/flight/flight-sql-jdbc-driver/src/test/java/org/apache/arrow/driver/jdbc/ITDriverJarValidation.java b/flight/flight-sql-jdbc-driver/src/test/java/org/apache/arrow/driver/jdbc/ITDriverJarValidation.java index c1bd111fb9..145744ad38 100644 --- a/flight/flight-sql-jdbc-driver/src/test/java/org/apache/arrow/driver/jdbc/ITDriverJarValidation.java +++ b/flight/flight-sql-jdbc-driver/src/test/java/org/apache/arrow/driver/jdbc/ITDriverJarValidation.java @@ -70,6 +70,10 @@ public class ITDriverJarValidation { "LICENSE.txt", "NOTICE.txt", "arrow-git.properties", + "iso3166_1alpha2-codes.properties", + "iso3166_1alpha3-codes.properties", + "iso3166_1alpha-2-3-map.properties", + "iso3166_3-codes.properties", "properties/flight.properties", "META-INF/io.netty.versions.properties", "META-INF/MANIFEST.MF", From ccaac9ad688265d272d4c3d9d824426ef7681669 Mon Sep 17 00:00:00 2001 From: Kaustav Sarkar <70840177+Kaustav-Sarkar@users.noreply.github.com> Date: Thu, 22 Jan 2026 23:26:49 +0530 Subject: [PATCH 269/408] GH-125: Allow null timestamp holder sans timezone (#941) ## Description Fixes an `IllegalArgumentException` in `TimeStamp*TZVector.set/setSafe` when unsetting values using a holder with a `null` timezone. The validation logic now correctly ignores the timezone check when `holder.isSet <= 0`, allowing default-constructed holders to be used for unsetting values as expected. Comprehensive tests added for all timestamp precisions (Micro, Milli, Nano, Sec) to verify the fix and ensure the existing workaround (setting explicit timezone) remains supported. Closes #125 . --- .../arrow/vector/TimeStampMicroTZVector.java | 11 +- .../arrow/vector/TimeStampMilliTZVector.java | 11 +- .../arrow/vector/TimeStampNanoTZVector.java | 11 +- .../arrow/vector/TimeStampSecTZVector.java | 11 +- .../apache/arrow/vector/TestValueVector.java | 193 ++++++++++++++++++ 5 files changed, 217 insertions(+), 20 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java b/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java index abaefcfc12..50f2f066cc 100644 --- a/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java @@ -155,12 +155,13 @@ public void set(int index, NullableTimeStampMicroTZHolder holder) throws IllegalArgumentException { if (holder.isSet < 0) { throw new IllegalArgumentException(); - } else if (!this.timeZone.equals(holder.timezone)) { - throw new IllegalArgumentException( - String.format( - "holder.timezone: %s not equal to vector timezone: %s", - holder.timezone, this.timeZone)); } else if (holder.isSet > 0) { + if (!this.timeZone.equals(holder.timezone)) { + throw new IllegalArgumentException( + String.format( + "holder.timezone: %s not equal to vector timezone: %s", + holder.timezone, this.timeZone)); + } BitVectorHelper.setBit(validityBuffer, index); setValue(index, holder.value); } else { diff --git a/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java b/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java index b5e5fb1be1..9e4998396c 100644 --- a/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java @@ -155,12 +155,13 @@ public void set(int index, NullableTimeStampMilliTZHolder holder) throws IllegalArgumentException { if (holder.isSet < 0) { throw new IllegalArgumentException(); - } else if (!this.timeZone.equals(holder.timezone)) { - throw new IllegalArgumentException( - String.format( - "holder.timezone: %s not equal to vector timezone: %s", - holder.timezone, this.timeZone)); } else if (holder.isSet > 0) { + if (!this.timeZone.equals(holder.timezone)) { + throw new IllegalArgumentException( + String.format( + "holder.timezone: %s not equal to vector timezone: %s", + holder.timezone, this.timeZone)); + } BitVectorHelper.setBit(validityBuffer, index); setValue(index, holder.value); } else { diff --git a/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java b/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java index 2386b3a859..b44b3da8d3 100644 --- a/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java @@ -154,12 +154,13 @@ public Long getObject(int index) { public void set(int index, NullableTimeStampNanoTZHolder holder) throws IllegalArgumentException { if (holder.isSet < 0) { throw new IllegalArgumentException(); - } else if (!this.timeZone.equals(holder.timezone)) { - throw new IllegalArgumentException( - String.format( - "holder.timezone: %s not equal to vector timezone: %s", - holder.timezone, this.timeZone)); } else if (holder.isSet > 0) { + if (!this.timeZone.equals(holder.timezone)) { + throw new IllegalArgumentException( + String.format( + "holder.timezone: %s not equal to vector timezone: %s", + holder.timezone, this.timeZone)); + } BitVectorHelper.setBit(validityBuffer, index); setValue(index, holder.value); } else { diff --git a/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java b/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java index f1774f2703..a64a87f699 100644 --- a/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java @@ -150,12 +150,13 @@ public Long getObject(int index) { public void set(int index, NullableTimeStampSecTZHolder holder) throws IllegalArgumentException { if (holder.isSet < 0) { throw new IllegalArgumentException(); - } else if (!this.timeZone.equals(holder.timezone)) { - throw new IllegalArgumentException( - String.format( - "holder.timezone: %s not equal to vector timezone: %s", - holder.timezone, this.timeZone)); } else if (holder.isSet > 0) { + if (!this.timeZone.equals(holder.timezone)) { + throw new IllegalArgumentException( + String.format( + "holder.timezone: %s not equal to vector timezone: %s", + holder.timezone, this.timeZone)); + } BitVectorHelper.setBit(validityBuffer, index); setValue(index, holder.value); } else { diff --git a/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index ac82246671..df42d04e60 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -57,6 +57,10 @@ import org.apache.arrow.vector.complex.impl.UnionListViewWriter; import org.apache.arrow.vector.complex.impl.UnionListWriter; import org.apache.arrow.vector.holders.NullableIntHolder; +import org.apache.arrow.vector.holders.NullableTimeStampMicroTZHolder; +import org.apache.arrow.vector.holders.NullableTimeStampMilliTZHolder; +import org.apache.arrow.vector.holders.NullableTimeStampNanoTZHolder; +import org.apache.arrow.vector.holders.NullableTimeStampSecTZHolder; import org.apache.arrow.vector.holders.NullableUInt4Holder; import org.apache.arrow.vector.holders.NullableVarBinaryHolder; import org.apache.arrow.vector.holders.NullableVarCharHolder; @@ -2567,6 +2571,195 @@ public void testSetNullableVarCharHolderSafe() { } } + @Test + public void testTimeStampTZVectorSetSafeUnset() { + // reproduction of https://github.com/apache/arrow/issues/45084 + try (TimeStampMicroTZVector vector = new TimeStampMicroTZVector("vector", allocator, "UTC")) { + vector.allocateNew(); + // Set a valid value + NullableTimeStampMicroTZHolder validHolder = new NullableTimeStampMicroTZHolder(); + validHolder.isSet = 1; + validHolder.value = 1000L; + validHolder.timezone = "UTC"; + vector.setSafe(0, validHolder); + + assertEquals(1000L, vector.get(0)); + + // Unset the value using a holder with default (null) timezone + // The bug used to throw IllegalArgumentException because holder.timezone (null) != + // vector.timezone ("UTC") + // The correct behaviour is to not throw an exception and to unset the value. + NullableTimeStampMicroTZHolder unsetHolder = new NullableTimeStampMicroTZHolder(); + unsetHolder.isSet = 0; + vector.setSafe(0, unsetHolder); + + assertNull(vector.getObject(0)); + } + } + + @Test + public void testTimeStampMilliTZVectorSetSafeUnset() { + // reproduction of https://github.com/apache/arrow/issues/45084 + try (TimeStampMilliTZVector vector = new TimeStampMilliTZVector("vector", allocator, "UTC")) { + vector.allocateNew(); + + NullableTimeStampMilliTZHolder validHolder = new NullableTimeStampMilliTZHolder(); + validHolder.isSet = 1; + validHolder.value = 1000L; + validHolder.timezone = "UTC"; + vector.setSafe(0, validHolder); + + assertEquals(1000L, vector.get(0)); + + NullableTimeStampMilliTZHolder unsetHolder = new NullableTimeStampMilliTZHolder(); + unsetHolder.isSet = 0; + vector.setSafe(0, unsetHolder); + + assertNull(vector.getObject(0)); + } + } + + @Test + public void testTimeStampNanoTZVectorSetSafeUnset() { + // reproduction of https://github.com/apache/arrow/issues/45084 + try (TimeStampNanoTZVector vector = new TimeStampNanoTZVector("vector", allocator, "UTC")) { + vector.allocateNew(); + + NullableTimeStampNanoTZHolder validHolder = new NullableTimeStampNanoTZHolder(); + validHolder.isSet = 1; + validHolder.value = 1000L; + validHolder.timezone = "UTC"; + vector.setSafe(0, validHolder); + + assertEquals(1000L, vector.get(0)); + + NullableTimeStampNanoTZHolder unsetHolder = new NullableTimeStampNanoTZHolder(); + unsetHolder.isSet = 0; + vector.setSafe(0, unsetHolder); + + assertNull(vector.getObject(0)); + } + } + + @Test + public void testTimeStampSecTZVectorSetSafeUnset() { + // reproduction of https://github.com/apache/arrow/issues/45084 + try (TimeStampSecTZVector vector = new TimeStampSecTZVector("vector", allocator, "UTC")) { + vector.allocateNew(); + + NullableTimeStampSecTZHolder validHolder = new NullableTimeStampSecTZHolder(); + validHolder.isSet = 1; + validHolder.value = 1000L; + validHolder.timezone = "UTC"; + vector.setSafe(0, validHolder); + + assertEquals(1000L, vector.get(0)); + + NullableTimeStampSecTZHolder unsetHolder = new NullableTimeStampSecTZHolder(); + unsetHolder.isSet = 0; + vector.setSafe(0, unsetHolder); + + assertNull(vector.getObject(0)); + } + } + + @Test + public void testTimeStampMicroTZVectorSetSafeUnsetExplicitTimezone() { + // Test to ensure fix added for https://github.com/apache/arrow/issues/45084 does not break + // workaround. + try (TimeStampMicroTZVector vector = new TimeStampMicroTZVector("vector", allocator, "UTC")) { + vector.allocateNew(); + + NullableTimeStampMicroTZHolder validHolder = new NullableTimeStampMicroTZHolder(); + validHolder.isSet = 1; + validHolder.value = 1000L; + validHolder.timezone = "UTC"; + vector.setSafe(0, validHolder); + + assertEquals(1000L, vector.get(0)); + + NullableTimeStampMicroTZHolder unsetHolder = new NullableTimeStampMicroTZHolder(); + unsetHolder.isSet = 0; + unsetHolder.timezone = "UTC"; + + vector.setSafe(0, unsetHolder); + + assertNull(vector.getObject(0)); + } + } + + @Test + public void testTimeStampMilliTZVectorSetSafeUnsetExplicitTimezone() { + // Test to ensure fix added for https://github.com/apache/arrow/issues/45084 does not break + // workaround. + try (TimeStampMilliTZVector vector = new TimeStampMilliTZVector("vector", allocator, "UTC")) { + vector.allocateNew(); + + NullableTimeStampMilliTZHolder validHolder = new NullableTimeStampMilliTZHolder(); + validHolder.isSet = 1; + validHolder.value = 1000L; + validHolder.timezone = "UTC"; + vector.setSafe(0, validHolder); + + assertEquals(1000L, vector.get(0)); + + NullableTimeStampMilliTZHolder unsetHolder = new NullableTimeStampMilliTZHolder(); + unsetHolder.isSet = 0; + unsetHolder.timezone = "UTC"; + vector.setSafe(0, unsetHolder); + + assertNull(vector.getObject(0)); + } + } + + @Test + public void testTimeStampNanoTZVectorSetSafeUnsetExplicitTimezone() { + // Test to ensure fix added for https://github.com/apache/arrow/issues/45084 does not break + // workaround. + try (TimeStampNanoTZVector vector = new TimeStampNanoTZVector("vector", allocator, "UTC")) { + vector.allocateNew(); + + NullableTimeStampNanoTZHolder validHolder = new NullableTimeStampNanoTZHolder(); + validHolder.isSet = 1; + validHolder.value = 1000L; + validHolder.timezone = "UTC"; + vector.setSafe(0, validHolder); + + assertEquals(1000L, vector.get(0)); + + NullableTimeStampNanoTZHolder unsetHolder = new NullableTimeStampNanoTZHolder(); + unsetHolder.isSet = 0; + unsetHolder.timezone = "UTC"; + vector.setSafe(0, unsetHolder); + + assertNull(vector.getObject(0)); + } + } + + @Test + public void testTimeStampSecTZVectorSetSafeUnsetExplicitTimezone() { + // Test to ensure fix added for https://github.com/apache/arrow/issues/45084 does not break + // workaround. + try (TimeStampSecTZVector vector = new TimeStampSecTZVector("vector", allocator, "UTC")) { + vector.allocateNew(); + + NullableTimeStampSecTZHolder validHolder = new NullableTimeStampSecTZHolder(); + validHolder.isSet = 1; + validHolder.value = 1000L; + validHolder.timezone = "UTC"; + vector.setSafe(0, validHolder); + + assertEquals(1000L, vector.get(0)); + + NullableTimeStampSecTZHolder unsetHolder = new NullableTimeStampSecTZHolder(); + unsetHolder.isSet = 0; + unsetHolder.timezone = "UTC"; + vector.setSafe(0, unsetHolder); + + assertNull(vector.getObject(0)); + } + } + @Test public void testSetNullableVarBinaryHolder() { try (VarBinaryVector vector = new VarBinaryVector("", allocator)) { From 0f8a0808fd9cf0bd22d3c6b40a2016ee724ce185 Mon Sep 17 00:00:00 2001 From: Yicong Huang <17627829+Yicong-Huang@users.noreply.github.com> Date: Fri, 23 Jan 2026 01:18:13 -0800 Subject: [PATCH 270/408] GH-343: Fix ListVector offset buffer not properly serialized for nested empty arrays (#967) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What's Changed Fix `ListVector`/`LargeListVector` IPC serialization when `valueCount` is 0. ### Problem When `valueCount == 0`, `setReaderAndWriterIndex()` was setting `offsetBuffer.writerIndex(0)`, which means `readableBytes() == 0`. IPC serializer uses `readableBytes()` to determine buffer size, so 0 bytes were written to the IPC stream. This crashes IPC readers in other libraries because Arrow spec requires offset buffer to have at least one entry `[0]`. @viirya: > The offset buffers are allocated properly. But during IPC serialization, they are ignored. > ``` > public long readableBytes() { > return writerIndex - readerIndex; > } > ``` > So when ListVector.setReaderAndWriterIndex() sets writerIndex(0) and readerIndex(0), readableBytes() returns 0 - 0 = 0. > > Then when MessageSerializer.writeBatchBuffers() calls WriteChannel.write(buffer), it writes 0 bytes. > > So the flow is: > > valueCount=0 → ListVector.setReaderAndWriterIndex() sets offsetBuffer.writerIndex(0) > VectorUnloader.getFieldBuffers() returns the buffer with writerIndex=0 > MessageSerializer.writeBatchBuffers() writes the buffer > WriteChannel.write(buffer) checks buffer.readableBytes() which is 0 > 0 bytes are written to the IPC stream > PyArrow read the batch with the missing buffer → crash when other libraries to read ### Fix Simplify `setReaderAndWriterIndex()` to always use `(valueCount + 1) * OFFSET_WIDTH` for offset buffer's `writerIndex`. When `valueCount == 0`, this correctly sets `writerIndex` to `OFFSET_WIDTH`, ensuring `offset[0]` is included in serialization. ### Testing Added tests for nested empty lists verifying offset buffer has correct `readableBytes()`. Closes #343. --------- Co-authored-by: Yicong Huang --- .../arrow/vector/complex/LargeListVector.java | 7 +++++-- .../arrow/vector/complex/ListVector.java | 7 +++++-- .../arrow/vector/TestLargeListVector.java | 20 +++++++++++++++++++ .../apache/arrow/vector/TestListVector.java | 20 +++++++++++++++++++ 4 files changed, 50 insertions(+), 4 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java index 997b5a8b78..92dd3eaef7 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java @@ -309,11 +309,14 @@ private void setReaderAndWriterIndex() { offsetBuffer.readerIndex(0); if (valueCount == 0) { validityBuffer.writerIndex(0); - offsetBuffer.writerIndex(0); } else { validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); - offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH); } + // IPC serializer will determine readable bytes based on `readerIndex` and `writerIndex`. + // Both are set to 0 means 0 bytes are written to the IPC stream which will crash IPC readers + // in other libraries. According to Arrow spec, we should still output the offset buffer which + // is [0]. + offsetBuffer.writerIndex((long) (valueCount + 1) * OFFSET_WIDTH); } /** diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 93a313ef4f..6c3993df63 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -267,11 +267,14 @@ private void setReaderAndWriterIndex() { offsetBuffer.readerIndex(0); if (valueCount == 0) { validityBuffer.writerIndex(0); - offsetBuffer.writerIndex(0); } else { validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); - offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH); } + // IPC serializer will determine readable bytes based on `readerIndex` and `writerIndex`. + // Both are set to 0 means 0 bytes are written to the IPC stream which will crash IPC readers + // in other libraries. According to Arrow spec, we should still output the offset buffer which + // is [0]. + offsetBuffer.writerIndex((long) (valueCount + 1) * OFFSET_WIDTH); } /** diff --git a/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java index ccc0d3e176..bf9bba9c78 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java @@ -1100,6 +1100,26 @@ public void testCopyValueSafeForExtensionType() throws Exception { } } + @Test + public void testEmptyLargeListOffsetBuffer() { + // Test that LargeListVector has correct readableBytes after allocation. + // According to Arrow spec, offset buffer must have N+1 entries. + // Even when N=0, it should contain [0]. + try (LargeListVector list = LargeListVector.empty("list", allocator)) { + list.addOrGetVector(FieldType.nullable(MinorType.INT.getType())); + list.allocateNew(); + list.setValueCount(0); + + List buffers = list.getFieldBuffers(); + assertTrue( + buffers.get(1).readableBytes() >= LargeListVector.OFFSET_WIDTH, + "Offset buffer should have at least " + + LargeListVector.OFFSET_WIDTH + + " bytes for offset[0]"); + assertEquals(0L, list.getOffsetBuffer().getLong(0)); + } + } + private void writeIntValues(UnionLargeListWriter writer, int[] values) { writer.startList(); for (int v : values) { diff --git a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index 1fe4c59f63..0c90b32abc 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -1379,6 +1379,26 @@ public void testCopyValueSafeForExtensionType() throws Exception { } } + @Test + public void testEmptyListOffsetBuffer() { + // Test that ListVector has correct readableBytes after allocation. + // According to Arrow spec, offset buffer must have N+1 entries. + // Even when N=0, it should contain [0]. + try (ListVector list = ListVector.empty("list", allocator)) { + list.addOrGetVector(FieldType.nullable(MinorType.INT.getType())); + list.allocateNew(); + list.setValueCount(0); + + List buffers = list.getFieldBuffers(); + assertTrue( + buffers.get(1).readableBytes() >= BaseRepeatedValueVector.OFFSET_WIDTH, + "Offset buffer should have at least " + + BaseRepeatedValueVector.OFFSET_WIDTH + + " bytes for offset[0]"); + assertEquals(0, list.getOffsetBuffer().getInt(0)); + } + } + private void writeIntValues(UnionListWriter writer, int[] values) { writer.startList(); for (int v : values) { From 3325625662fa0412fccacc9d416289ac93b5b318 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jan 2026 08:49:41 +0100 Subject: [PATCH 271/408] MINOR: Bump org.apache.commons:commons-dbcp2 from 2.13.0 to 2.14.0 (#983) Bumps org.apache.commons:commons-dbcp2 from 2.13.0 to 2.14.0. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.commons:commons-dbcp2&package-manager=maven&previous-version=2.13.0&new-version=2.14.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---

Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql/pom.xml b/flight/flight-sql/pom.xml index 4175ff70d3..fe03406738 100644 --- a/flight/flight-sql/pom.xml +++ b/flight/flight-sql/pom.xml @@ -95,7 +95,7 @@ under the License. org.apache.commons commons-dbcp2 - 2.13.0 + 2.14.0 test From 096f582ae7e7f39af8f80f23470c4802a27aeac8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jan 2026 09:06:41 +0100 Subject: [PATCH 272/408] MINOR: Bump org.apache.commons:commons-compress from 1.27.1 to 1.28.0 (#985) Bumps [org.apache.commons:commons-compress](https://github.com/apache/commons-compress) from 1.27.1 to 1.28.0.
Changelog

Sourced from org.apache.commons:commons-compress's changelog.

Apache Commons Compress 1.28.0 Release Notes

The Apache Commons Compress team is pleased to announce the release of Apache Commons Compress 1.28.0.

Apache Commons Compress defines an API for working with compression and archive formats. These include bzip2, gzip, pack200, LZMA, XZ, Snappy, traditional Unix Compress, DEFLATE, DEFLATE64, LZ4, Brotli, Zstandard and ar, cpio, jar, tar, zip, dump, 7z, arj.

This is a feature and maintenance release. Java 8 or later is required.

This release updates Apache Commons Lang to 3.18.0 to pick up the fix for CVE-2025-48924 (https://nvd.nist.gov/vuln/detail/CVE-2025-48924), but is not affected by it.

Changes in this version

Changes in this version include the following.

New Features

  •  Add GzipParameters.getModificationInstant(). Thanks to Gary
Gregory.

  •  Add GzipParameters.setModificationInstant(Instant). Thanks
to Gary Gregory.

  •  Add GzipParameters.OS, setOS(OS), getOS(). Thanks to Gary
Gregory.

  •  Add GzipParameters.toString(). Thanks to Gary Gregory.
  • COMPRESS-638: Add GzipParameters.setFileNameCharset(Charset) and getFileNameCharset() to override the default ISO-8859-1 Charset #602. Thanks to vincexjl, Gary Gregory, Piotr P. Karwasz.
  •  Add support for gzip extra subfields, see
GzipParameters.setExtra(HeaderExtraField)
[#604](https://github.com/apache/commons-compress/issues/604). Thanks to
ddeschenes-1, Gary Gregory.

  •  Add CompressFilterOutputStream and refactor to use. Thanks
to Gary Gregory.

  •        Add ZipFile.stream(). Thanks to Gary Gregory.

  •  GzipCompressorInputStream reads the modification time
(MTIME) and stores its value incorrectly multiplied by 1,000. Thanks to
Danny Deschenes, Gary Gregory.

  •  GzipCompressorInputStream writes the modification time
(MTIME) the value incorrectly divided by 1,000. Thanks to Danny
Deschenes, Gary Gregory.

  •  Add optional FHCRC to GZIP header
[#627](https://github.com/apache/commons-compress/issues/627). Thanks to
Danny Deschenes, Gary Gregory.

  •  Add GzipCompressorInputStream.Builder allowing to customize
the file name and comment Charsets. Thanks to Gary Gregory.

  •  Add
GzipCompressorInputStream.Builder.setOnMemberStart(IOConsumer) to
monitor member parsing. Thanks to Gary Gregory.

  •  Add
GzipCompressorInputStream.Builder.setOnMemberEnd(IOConsumer) to monitor
member parsing. Thanks to Gary Gregory.

  •  Add PMD check to default Maven goal. Thanks to Gary Gregory.

  •  Add SevenZFile.Builder.setMaxMemoryLimitKiB(int). Thanks to
Gary Gregory.

  •  Add MemoryLimitException.MemoryLimitException(long, int,
Throwable) and deprecate MemoryLimitException.MemoryLimitException(long,
int, Exception). Thanks to Gary Gregory.
  • COMPRESS-692: Add support for zstd compression in zip archives. Thanks to Mehmet Karaman, Andrey Loskutov, Gary Gregory.
  •  Add support for XZ compression in ZIP archives. Thanks to
Gary Gregory.
  • COMPRESS-695: Add ZipArchiveInputStream.createZstdInputStream(InputStream) to provide a different InputStream implementation for Zstandard (Zstd) #649. Thanks to Gary Gregory.
  •  Add
org.apache.commons.compress.harmony.pack200.Pack200Exception.Pack200Exception(String,
Throwable). Thanks to Gary Gregory.
  • COMPRESS-697: Move BitStream.nextBit() method to BitInputStream #663. Thanks to Fredrik Kjellberg, Gary Gregory.
  •  Add
org.apache.commons.compress.compressors.lzma.LZMACompressorInputStream.builder/Builder().
Thanks to Gary Gregory.

  •  Add
org.apache.commons.compress.compressors.lzma.LZMACompressorOutputStream.builder/Builder().
Thanks to Gary Gregory.

  •  Add
org.apache.commons.compress.compressors.xz.XZCompressorInputStream.builder/Builder().
Thanks to Gary Gregory.

  •  Add
org.apache.commons.compress.compressors.xz.XZCompressorOutputStream.builder/Builder().
Thanks to Gary Gregory.

  •  Add
org.apache.commons.compress.compressors.xz.ZstdCompressorOutputStream.builder/Builder()
[#666](https://github.com/apache/commons-compress/issues/666). Thanks to
Gary Gregory, David Walluck, Piotr P. Karwasz.

  •  Add org.apache.commons.compress.compressors.xz.ZstdConstants
[#666](https://github.com/apache/commons-compress/issues/666). Thanks to
Gary Gregory, David Walluck, Piotr P. Karwasz.

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.commons:commons-compress&package-manager=maven&previous-version=1.27.1&new-version=1.28.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- compression/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compression/pom.xml b/compression/pom.xml index ba13156243..29f8b41788 100644 --- a/compression/pom.xml +++ b/compression/pom.xml @@ -50,7 +50,7 @@ under the License. org.apache.commons commons-compress - 1.27.1 + 1.28.0 com.github.luben From 3db456262fe00fc980f4846f31ed0e3cea1e6379 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Jan 2026 17:21:30 +0100 Subject: [PATCH 273/408] MINOR: Bump org.assertj:assertj-core from 3.27.3 to 3.27.7 (#988) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.assertj:assertj-core](https://github.com/assertj/assertj) from 3.27.3 to 3.27.7.
Release notes

Sourced from org.assertj:assertj-core's releases.

v3.27.7

:lock: Security

Core

:no_entry_sign: Deprecated

Core

  • Deprecate XmlStringPrettyFormatter with no replacement

:bug: Bug Fixes

Guava

  • Navigation to assertj-core or guava types from assertj-guava Javadoc site has unnecessary header #3478

:hammer: Dependency Upgrades

Core

  • Upgrade to Byte Buddy 1.18.3
  • Upgrade to JUnit BOM 5.14.1

Guava

  • Upgrade to Guava 33.5.0-jre

v3.27.6

:bug: Bug Fixes

Core

  • Add missing export for org.assertj.core.annotation #3951

:heart: Contributors

Thanks to all the contributors who worked on this release:

@​duponter

v3.27.5

:zap: Improvements

Core

  • ByteBuddy in AssertJ 3.27.4 not compatible with Java 25 #3946

... (truncated)

Commits
  • e840716 [maven-release-plugin] prepare release assertj-build-3.27.7
  • 85ca7eb Deprecate XmlStringPrettyFormatter
  • 77081dc Merge commit from fork
  • b68fc24 Bump github/codeql-action from 4.31.9 to 4.31.10 in the github-actions group ...
  • 0cf5bb6 Bump kotlin.version from 2.1.0 to 2.2.21
  • d393ef1 Abort tests when symbolic links cannot be created (#3788)
  • 2212433 Add IntelliJ custom inspection for test class names
  • 5717d02 Update JetBrains icon
  • a8ec20b Add icon for JetBrains products
  • c05fb3d Bump Maven to 3.9.12 and Wrapper to 3.3.4
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.assertj:assertj-core&package-manager=maven&previous-version=3.27.3&new-version=3.27.7)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/apache/arrow-java/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b3141d5cd8..51920947c8 100644 --- a/pom.xml +++ b/pom.xml @@ -175,7 +175,7 @@ under the License. org.assertj assertj-core - 3.27.3 + 3.27.7 test From 9b1f946db4927fa49c6ba4d713d7e83bd712a026 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Jan 2026 17:57:43 +0100 Subject: [PATCH 274/408] MINOR: Bump org.apache.commons:commons-pool2 from 2.12.1 to 2.13.1 (#987) Bumps org.apache.commons:commons-pool2 from 2.12.1 to 2.13.1. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.commons:commons-pool2&package-manager=maven&previous-version=2.12.1&new-version=2.13.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql/pom.xml b/flight/flight-sql/pom.xml index fe03406738..56c47f64dd 100644 --- a/flight/flight-sql/pom.xml +++ b/flight/flight-sql/pom.xml @@ -107,7 +107,7 @@ under the License. org.apache.commons commons-pool2 - 2.12.1 + 2.13.1 test From 923a5df0d668aa90f0d322f618314118ea0399e6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Jan 2026 18:26:34 +0100 Subject: [PATCH 275/408] MINOR: Bump logback.version from 1.5.25 to 1.5.26 (#981) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps `logback.version` from 1.5.25 to 1.5.26. Updates `ch.qos.logback:logback-classic` from 1.5.25 to 1.5.26
Release notes

Sourced from ch.qos.logback:logback-classic's releases.

Logback 1.5.26

2026-01-25 Release of logback version 1.5.26

• InsertFromJNDIModelHandler was accessing javax.naming package forcing the inclusion of the optional java.naming module. This problem was raised in issues/1003 by Marius Hanl who also provided the relevant PR.

• In applications using shadow/fat/shade jars, module or package information could be lost. Thus, in the absence of version information, logback-classic would warn about version mismatches. Logback components now ship with properties files containing version information that survive shadow/fat/shade jars. This issue was reporteed in issues/1002 by Christoph Gritschenberger.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit 33deb54506bbfaf1ff151f26f3a5f86936011619 associated with the tag v_1.5.26. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits
  • 33deb54 prepare release 1.5.26
  • d38a3e2 refactoring based on usage in logback-access
  • 4368333 move VersionUtil.getCoreVersionBySelfDeclaredProperties to CoreVersionUtil
  • 8bd5660 modify VersionCheckTest to use logback-core 1.5.25
  • 7a8f0b6 version information is self declared by modules.
  • 00d272f Do not use javax.naming namespace in the catch block, so that Logback can be ...
  • 420d67c mention country only, add missing 2016-03-29
  • 033aba4 fix javadoc errors
  • 6d52744 start work on 1.5.26-SNAPSHOT
  • See full diff in compare view

Updates `ch.qos.logback:logback-core` from 1.5.25 to 1.5.26
Release notes

Sourced from ch.qos.logback:logback-core's releases.

Logback 1.5.26

2026-01-25 Release of logback version 1.5.26

• InsertFromJNDIModelHandler was accessing javax.naming package forcing the inclusion of the optional java.naming module. This problem was raised in issues/1003 by Marius Hanl who also provided the relevant PR.

• In applications using shadow/fat/shade jars, module or package information could be lost. Thus, in the absence of version information, logback-classic would warn about version mismatches. Logback components now ship with properties files containing version information that survive shadow/fat/shade jars. This issue was reporteed in issues/1002 by Christoph Gritschenberger.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit 33deb54506bbfaf1ff151f26f3a5f86936011619 associated with the tag v_1.5.26. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits
  • 33deb54 prepare release 1.5.26
  • d38a3e2 refactoring based on usage in logback-access
  • 4368333 move VersionUtil.getCoreVersionBySelfDeclaredProperties to CoreVersionUtil
  • 8bd5660 modify VersionCheckTest to use logback-core 1.5.25
  • 7a8f0b6 version information is self declared by modules.
  • 00d272f Do not use javax.naming namespace in the catch block, so that Logback can be ...
  • 420d67c mention country only, add missing 2016-03-29
  • 033aba4 fix javadoc errors
  • 6d52744 start work on 1.5.26-SNAPSHOT
  • See full diff in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 51920947c8..4bab9b06c5 100644 --- a/pom.xml +++ b/pom.xml @@ -111,7 +111,7 @@ under the License. true 2.42.0 3.53.0 - 1.5.25 + 1.5.26 none -Xdoclint:none From 0eb50b5e840d7538d10478934929cc6b7ea40429 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Jan 2026 18:44:08 +0100 Subject: [PATCH 276/408] MINOR: Bump com.google.protobuf:protobuf-bom from 4.33.1 to 4.33.4 (#984) Bumps [com.google.protobuf:protobuf-bom](https://github.com/protocolbuffers/protobuf) from 4.33.1 to 4.33.4.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.protobuf:protobuf-bom&package-manager=maven&previous-version=4.33.1&new-version=4.33.4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4bab9b06c5..d64df1ade3 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,7 @@ under the License. 33.4.8-jre 4.2.9.Final 1.78.0 - 4.33.1 + 4.33.4 2.21.0 3.4.2 25.2.10 From ad59035ec880920f285158a140467d8b8d41789c Mon Sep 17 00:00:00 2001 From: Pedro Matias Date: Tue, 27 Jan 2026 22:25:29 +0000 Subject: [PATCH 277/408] GH-990: [JDBC] Fix memory leak on Connection#close due to unclosed ResultSet(s) (#991) ## What's Changed Closing a Connection when there was one or more unclosed ResultSet that had been obtained via methods of the interface DatabaseMetaData would generate exceptions due to memory leaks. Now, closing a Connection will first close all the ResultSet instances obtained from DatabaseMetadata instances associated with that Connection. Closes #990. --- .../driver/jdbc/ArrowFlightConnection.java | 32 +++++++++ .../ArrowFlightJdbcFlightStreamResultSet.java | 7 ++ ...owFlightJdbcVectorSchemaRootResultSet.java | 11 ++- .../arrow/driver/jdbc/ConnectionTest.java | 70 +++++++++++++++++++ 4 files changed, 113 insertions(+), 7 deletions(-) diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java index 0e9c198f52..623c2b81be 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java @@ -21,6 +21,8 @@ import io.netty.util.concurrent.DefaultThreadFactory; import java.sql.SQLException; import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -42,6 +44,8 @@ public final class ArrowFlightConnection extends AvaticaConnection { private final ArrowFlightSqlClientHandler clientHandler; private final ArrowFlightConnectionConfigImpl config; private ExecutorService executorService; + private int metadataResultSetCount; + private Map metadataResultSetMap = new HashMap<>(); /** * Creates a new {@link ArrowFlightConnection}. @@ -66,6 +70,7 @@ private ArrowFlightConnection( this.config = Preconditions.checkNotNull(config, "Config cannot be null."); this.allocator = Preconditions.checkNotNull(allocator, "Allocator cannot be null."); this.clientHandler = Preconditions.checkNotNull(clientHandler, "Handler cannot be null."); + this.metadataResultSetCount = 0; } /** @@ -173,6 +178,31 @@ synchronized ExecutorService getExecutorService() { : executorService; } + /** + * Registers a new metadata ResultSet and assigns it a unique ID. Metadata ResultSets are those + * created without an associated Statement. + * + * @param resultSet the ResultSet to register + * @return the assigned ID + */ + int getNewMetadataResultSetId(ArrowFlightJdbcFlightStreamResultSet resultSet) { + metadataResultSetMap.put(metadataResultSetCount, resultSet); + return metadataResultSetCount++; + } + + /** + * Unregisters a metadata ResultSet when it is closed. This method is called by metadata + * ResultSets during their close operation to remove themselves from the tracking map. + * + * @param id the ID of the ResultSet to unregister, or null if not a metadata ResultSet + */ + void onResultSetClose(Integer id) { + if (id == null) { + return; + } + metadataResultSetMap.remove(id); + } + @Override public Properties getClientInfo() { final Properties copy = new Properties(); @@ -190,7 +220,9 @@ public void close() throws SQLException { } catch (final Exception e) { topLevelException = e; } + // copies of the collections are used to avoid concurrent modification problems ArrayList closeables = new ArrayList<>(statementMap.values()); + closeables.addAll(new ArrayList<>(metadataResultSetMap.values())); closeables.add(clientHandler); closeables.addAll(allocator.getChildAllocators()); closeables.add(allocator); diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcFlightStreamResultSet.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcFlightStreamResultSet.java index aabaf01e63..2885f7895b 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcFlightStreamResultSet.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcFlightStreamResultSet.java @@ -54,6 +54,7 @@ public final class ArrowFlightJdbcFlightStreamResultSet private VectorSchemaRoot currentVectorSchemaRoot; private Schema schema; + private Integer id = null; // used for metadata result sets only /** Public constructor used by ArrowFlightJdbcFactory. */ ArrowFlightJdbcFlightStreamResultSet( @@ -82,6 +83,7 @@ private ArrowFlightJdbcFlightStreamResultSet( super(null, state, signature, resultSetMetaData, timeZone, firstFrame); this.connection = connection; this.flightInfo = flightInfo; + this.id = connection.getNewMetadataResultSetId(this); } /** @@ -234,7 +236,12 @@ protected void cancel() { @Override public synchronized void close() { + try { + if (isClosed()) { + return; + } + this.connection.onResultSetClose(id); if (flightEndpointDataQueue != null) { // flightStreamQueue should close currentFlightStream internally flightEndpointDataQueue.close(); diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java index 622e5fe7f6..49334951de 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java @@ -22,7 +22,6 @@ import java.sql.Types; import java.util.HashSet; import java.util.List; -import java.util.Objects; import java.util.Set; import java.util.TimeZone; import org.apache.arrow.driver.jdbc.utils.ConvertUtils; @@ -159,12 +158,10 @@ public void close() { } catch (final Exception e) { exceptions.add(e); } - if (!Objects.isNull(statement)) { - try { - super.close(); - } catch (final Exception e) { - exceptions.add(e); - } + try { + super.close(); + } catch (final Exception e) { + exceptions.add(e); } exceptions.parallelStream().forEach(e -> LOGGER.error(e.getMessage(), e)); exceptions.stream() diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java index dbedbe9d36..55722f60fb 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java @@ -16,6 +16,8 @@ */ package org.apache.arrow.driver.jdbc; +import static java.lang.String.format; +import static java.util.stream.IntStream.range; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -23,24 +25,33 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +import com.google.protobuf.Message; import java.net.URISyntaxException; import java.sql.Connection; import java.sql.Driver; import java.sql.DriverManager; +import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.Map; import java.util.Properties; +import java.util.function.Consumer; import org.apache.arrow.driver.jdbc.authentication.UserPasswordAuthentication; import org.apache.arrow.driver.jdbc.client.ArrowFlightSqlClientHandler; import org.apache.arrow.driver.jdbc.utils.ArrowFlightConnectionConfigImpl.ArrowFlightConnectionProperty; import org.apache.arrow.driver.jdbc.utils.MockFlightSqlProducer; import org.apache.arrow.flight.FlightMethod; +import org.apache.arrow.flight.FlightProducer.ServerStreamListener; import org.apache.arrow.flight.NoOpSessionOptionValueVisitor; import org.apache.arrow.flight.SessionOptionValue; +import org.apache.arrow.flight.sql.FlightSqlProducer.Schemas; +import org.apache.arrow.flight.sql.impl.FlightSql.CommandGetTableTypes; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.util.AutoCloseables; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.util.Text; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -698,4 +709,63 @@ public void testStatementsClosedOnConnectionClose() throws Exception { assertTrue(statements[i].isClosed()); } } + + @Test + public void testResultSetsFromDatabaseMetadataClosedOnConnectionClose() throws Exception { + // set up the FlightProducer to respond to metadata queries + // getTableTypes() is being used, but any other method would work + int rowCount = 3; + final Message commandGetTableTypes = CommandGetTableTypes.getDefaultInstance(); + final Consumer commandGetTableTypesResultProducer = + listener -> { + try (final BufferAllocator allocator = new RootAllocator(); + final VectorSchemaRoot root = + VectorSchemaRoot.create(Schemas.GET_TABLE_TYPES_SCHEMA, allocator)) { + final VarCharVector tableType = (VarCharVector) root.getVector("table_type"); + range(0, rowCount) + .forEach(i -> tableType.setSafe(i, new Text(format("table_type #%d", i)))); + root.setRowCount(rowCount); + listener.start(root); + listener.putNext(); + } catch (final Throwable throwable) { + listener.error(throwable); + } finally { + listener.completed(); + } + }; + PRODUCER.addCatalogQuery(commandGetTableTypes, commandGetTableTypesResultProducer); + + // create a connection + final Properties properties = new Properties(); + properties.put(ArrowFlightConnectionProperty.HOST.camelName(), "localhost"); + properties.put( + ArrowFlightConnectionProperty.PORT.camelName(), FLIGHT_SERVER_TEST_EXTENSION.getPort()); + properties.put(ArrowFlightConnectionProperty.USER.camelName(), userTest); + properties.put(ArrowFlightConnectionProperty.PASSWORD.camelName(), passTest); + properties.put("useEncryption", false); + + Connection connection = + DriverManager.getConnection( + "jdbc:arrow-flight-sql://" + + FLIGHT_SERVER_TEST_EXTENSION.getHost() + + ":" + + FLIGHT_SERVER_TEST_EXTENSION.getPort(), + properties); + + // create ResultSets from DatabaseMetadata + int numResultSets = 3; + ResultSet[] resultSets = new ResultSet[numResultSets]; + for (int i = 0; i < numResultSets; i++) { + resultSets[i] = connection.getMetaData().getTableTypes(); + assertFalse(resultSets[i].isClosed()); + } + + // close the connection + connection.close(); + + // assert the ResultSets are closed + for (int i = 0; i < numResultSets; i++) { + assertTrue(resultSets[i].isClosed()); + } + } } From ce1f3d75b25038af068b62cb9ebd35817b7a04a3 Mon Sep 17 00:00:00 2001 From: Tamas Mate <50709850+tmater@users.noreply.github.com> Date: Wed, 28 Jan 2026 22:58:30 +0100 Subject: [PATCH 278/408] GH-993: Fix missing pipe in milestone assignment script (#992) The `head -n1` command was not piped to grep output, causing all matching milestones to be captured instead of just the first one. Example failure: ``` Assigning milestone: 19.0.0 20.0.0 '19.0.0 20.0.0' not found ``` Closes #993 --- .github/workflows/dev_pr_milestone.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dev_pr_milestone.sh b/.github/workflows/dev_pr_milestone.sh index b6876b4b08..4a77eb1f73 100755 --- a/.github/workflows/dev_pr_milestone.sh +++ b/.github/workflows/dev_pr_milestone.sh @@ -37,8 +37,8 @@ main() { local -r milestone=$( gh api "/repos/${repo}/milestones" | jq --raw-output '.[] | .title' | - grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' - head -n1 + grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | + head -n1 ) echo "Assigning milestone: ${milestone}" From b3113ab797020a8cfd6ea83ab3c549eb808b2d09 Mon Sep 17 00:00:00 2001 From: Aleksei Starikov Date: Sun, 8 Feb 2026 14:07:13 +0100 Subject: [PATCH 279/408] GH-1011: [Docs] Fix broken Java API reference links in documentation (#1012) ## What's Changed Fix Java API references in docs. For example: https://arrow.apache.org/docs/java/reference/org/apache/arrow/flight/FlightClient.html -> https://arrow.apache.org/java/current/reference/org.apache.arrow.flight.core/org/apache/arrow/flight/FlightClient.html Closes #1011. --- docs/source/flight.rst | 10 +++++----- docs/source/flight_sql.rst | 2 +- docs/source/jdbc.rst | 6 +++--- docs/source/memory.rst | 18 +++++++++--------- docs/source/table.rst | 24 ++++++++++++------------ docs/source/vector_schema_root.rst | 16 ++++++++-------- 6 files changed, 38 insertions(+), 38 deletions(-) diff --git a/docs/source/flight.rst b/docs/source/flight.rst index fabced8094..fd0fdf07bc 100644 --- a/docs/source/flight.rst +++ b/docs/source/flight.rst @@ -232,8 +232,8 @@ Servers can add other gRPC services. For example, to add the `Health Check servi See the :external+arrow:ref:`best practices for C++ `. -.. _`FlightClient`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/flight/FlightClient.html -.. _`FlightProducer`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/flight/FlightProducer.html -.. _`FlightServer`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/flight/FlightServer.html -.. _`NoOpFlightProducer`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/flight/NoOpFlightProducer.html -.. _`Location`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/flight/Location.html +.. _`FlightClient`: https://arrow.apache.org/java/current/reference/org.apache.arrow.flight.core/org/apache/arrow/flight/FlightClient.html +.. _`FlightProducer`: https://arrow.apache.org/java/current/reference/org.apache.arrow.flight.core/org/apache/arrow/flight/FlightProducer.html +.. _`FlightServer`: https://arrow.apache.org/java/current/reference/org.apache.arrow.flight.core/org/apache/arrow/flight/FlightServer.html +.. _`NoOpFlightProducer`: https://arrow.apache.org/java/current/reference/org.apache.arrow.flight.core/org/apache/arrow/flight/NoOpFlightProducer.html +.. _`Location`: https://arrow.apache.org/java/current/reference/org.apache.arrow.flight.core/org/apache/arrow/flight/Location.html diff --git a/docs/source/flight_sql.rst b/docs/source/flight_sql.rst index 169a0e24bf..09ce1dda0d 100644 --- a/docs/source/flight_sql.rst +++ b/docs/source/flight_sql.rst @@ -29,4 +29,4 @@ over the network. For usage information, see the `API documentation`_. -.. _API documentation: https://arrow.apache.org/docs/java/reference/org/apache/arrow/flight/sql/package-summary.html +.. _API documentation: https://arrow.apache.org/java/current/reference/org.apache.arrow.flight.sql/org/apache/arrow/flight/sql/package-summary.html diff --git a/docs/source/jdbc.rst b/docs/source/jdbc.rst index a4c95dbf00..2f57c34bf8 100644 --- a/docs/source/jdbc.rst +++ b/docs/source/jdbc.rst @@ -95,7 +95,7 @@ Type Mapping The JDBC to Arrow type mapping can be obtained at runtime from `JdbcToArrowUtils.getArrowTypeFromJdbcType`_. -.. _JdbcToArrowUtils.getArrowTypeFromJdbcType: https://arrow.apache.org/docs/java/reference/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.html#getArrowTypeFromJdbcType-org.apache.arrow.adapter.jdbc.JdbcFieldInfo-java.util.Calendar- +.. _JdbcToArrowUtils.getArrowTypeFromJdbcType: https://arrow.apache.org/java/current/reference/org.apache.arrow.adapter.jdbc/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.html#getArrowTypeFromJdbcType-org.apache.arrow.adapter.jdbc.JdbcFieldInfo-java.util.Calendar- +--------------------+--------------------+-------+ | JDBC Type | Arrow Type | Notes | @@ -171,8 +171,8 @@ The JDBC to Arrow type mapping can be obtained at runtime from timezone of the calendar, else it will be a timestamp without timezone. -.. _setArraySubTypeByColumnIndexMap: https://arrow.apache.org/docs/java/reference/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigBuilder.html#setArraySubTypeByColumnIndexMap-java.util.Map- -.. _setArraySubTypeByColumnNameMap: https://arrow.apache.org/docs/java/reference/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigBuilder.html#setArraySubTypeByColumnNameMap-java.util.Map- +.. _setArraySubTypeByColumnIndexMap: https://arrow.apache.org/java/current/reference/org.apache.arrow.adapter.jdbc/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigBuilder.html#setArraySubTypeByColumnIndexMap-java.util.Map- +.. _setArraySubTypeByColumnNameMap: https://arrow.apache.org/java/current/reference/org.apache.arrow.adapter.jdbc/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigBuilder.html#setArraySubTypeByColumnNameMap-java.util.Map- .. _ARROW-17006: https://issues.apache.org/jira/browse/ARROW-17006 VectorSchemaRoot to PreparedStatement Parameter Conversion diff --git a/docs/source/memory.rst b/docs/source/memory.rst index 58ef382dc9..4a71ed846a 100644 --- a/docs/source/memory.rst +++ b/docs/source/memory.rst @@ -333,18 +333,18 @@ How this works: } } -.. _`ArrowBuf`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/ArrowBuf.html -.. _`ArrowBuf.print()`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/ArrowBuf.html#print-java.lang.StringBuilder-int-org.apache.arrow.memory.BaseAllocator.Verbosity- -.. _`BufferAllocator`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/BufferAllocator.html -.. _`BufferLedger`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/BufferLedger.html -.. _`RootAllocator`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/RootAllocator.html -.. _`newChildAllocator`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/RootAllocator.html#newChildAllocator-java.lang.String-org.apache.arrow.memory.AllocationListener-long-long- +.. _`ArrowBuf`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/ArrowBuf.html +.. _`ArrowBuf.print()`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/ArrowBuf.html#print-java.lang.StringBuilder-int-org.apache.arrow.memory.BaseAllocator.Verbosity- +.. _`BufferAllocator`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/BufferAllocator.html +.. _`BufferLedger`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/BufferLedger.html +.. _`RootAllocator`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/RootAllocator.html +.. _`newChildAllocator`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/RootAllocator.html#newChildAllocator-java.lang.String-org.apache.arrow.memory.AllocationListener-long-long- .. _`Netty`: https://netty.io/wiki/ .. _`sun.misc.unsafe`: https://web.archive.org/web/20210929024401/http://www.docjar.com/html/api/sun/misc/Unsafe.java.html .. _`Direct Memory`: https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/nio/ByteBuffer.html -.. _`ReferenceManager`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/ReferenceManager.html -.. _`ReferenceManager.release`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/ReferenceManager.html#release-- -.. _`ReferenceManager.retain`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/ReferenceManager.html#retain-- +.. _`ReferenceManager`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/ReferenceManager.html +.. _`ReferenceManager.release`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/ReferenceManager.html#release-- +.. _`ReferenceManager.retain`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/ReferenceManager.html#retain-- Arrow Memory In-Depth ===================== diff --git a/docs/source/table.rst b/docs/source/table.rst index 5aa95e153c..880ef84d29 100644 --- a/docs/source/table.rst +++ b/docs/source/table.rst @@ -364,15 +364,15 @@ If the table contains dictionary-encoded vectors and was constructed with a ``Di Data.exportTable(bufferAllocator, table, outArrowArray); -.. _`ArrowBuf`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/ArrowBuf.html -.. _`Data`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/c/Data.html -.. _`DictionaryProvider`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/dictionary/DictionaryProvider.html -.. _`Field`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/types/pojo/Field.html -.. _`FieldReader`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/complex/reader/FieldReader.html -.. _`FieldVector`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/FieldVector.html -.. _`Row`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/table/Row.html -.. _`Schema`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/types/pojo/Schema.html -.. _`Table`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/table/Table.html -.. _`ValueHolder`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/holders/ValueHolder.html -.. _`ValueVector`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/ValueVector.html -.. _`VectorSchemaRoot`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/VectorSchemaRoot.html +.. _`ArrowBuf`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/ArrowBuf.html +.. _`Data`: https://arrow.apache.org/java/current/reference/org.apache.arrow.c/org/apache/arrow/c/Data.html +.. _`DictionaryProvider`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/dictionary/DictionaryProvider.html +.. _`Field`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/types/pojo/Field.html +.. _`FieldReader`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/complex/reader/FieldReader.html +.. _`FieldVector`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/FieldVector.html +.. _`Row`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/table/Row.html +.. _`Schema`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/types/pojo/Schema.html +.. _`Table`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/table/Table.html +.. _`ValueHolder`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/holders/ValueHolder.html +.. _`ValueVector`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/ValueVector.html +.. _`VectorSchemaRoot`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/VectorSchemaRoot.html diff --git a/docs/source/vector_schema_root.rst b/docs/source/vector_schema_root.rst index 3119122d9a..f4a497c4e5 100644 --- a/docs/source/vector_schema_root.rst +++ b/docs/source/vector_schema_root.rst @@ -153,11 +153,11 @@ A `Table`_ is an immutable tabular data structure, very similar to VectorSchemaR See the :doc:`table` documentation for more information. -.. _`ArrowRecordBatch`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/ipc/message/ArrowRecordBatch.html -.. _`Field`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/types/pojo/Field.html -.. _`Flight`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/flight/package-summary.html -.. _`Schema`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/types/pojo/Schema.html -.. _`Table`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/table/Table.html -.. _`VectorLoader`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/VectorLoader.html -.. _`VectorSchemaRoot`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/VectorSchemaRoot.html -.. _`VectorUnloader`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/VectorUnloader.html +.. _`ArrowRecordBatch`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/ipc/message/ArrowRecordBatch.html +.. _`Field`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/types/pojo/Field.html +.. _`Flight`: https://arrow.apache.org/java/current/reference/org.apache.arrow.flight.core/org/apache/arrow/flight/package-summary.html +.. _`Schema`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/types/pojo/Schema.html +.. _`Table`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/table/Table.html +.. _`VectorLoader`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/VectorLoader.html +.. _`VectorSchemaRoot`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/VectorSchemaRoot.html +.. _`VectorUnloader`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/VectorUnloader.html From 776466e904f3ae44f52c3baa019e795e8a68c527 Mon Sep 17 00:00:00 2001 From: Aleksei Starikov Date: Sun, 8 Feb 2026 14:08:20 +0100 Subject: [PATCH 280/408] GH-141: Correct capacity behavior in BufferAllocator.buffer docstrings (#1010) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What's Changed Update the `BufferAllocator.buffer(long)` and `BufferAllocator.buffer(long, BufferManager)` docstrings so they match the actual behavior: the returned buffer’s capacity is the allocated (possibly rounded) size, not the requested size. The previous text said the capacity would be set to the configured size, which was incorrect. The new text also mentions that callers can use `ArrowBuf#capacity(long)` to set the capacity to the requested size when needed. Documentation-only change; no code or behavioral changes. Closes #141. --- .../java/org/apache/arrow/memory/ArrowBuf.java | 2 +- .../org/apache/arrow/memory/BufferAllocator.java | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java b/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java index 775a8925ad..b8012fe643 100644 --- a/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java +++ b/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java @@ -136,7 +136,7 @@ public long capacity() { /** * Adjusts the capacity of this buffer. Size increases are NOT supported. * - * @param newCapacity Must be in in the range [0, length). + * @param newCapacity Must be in the range [0, length). */ public synchronized ArrowBuf capacity(long newCapacity) { diff --git a/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferAllocator.java b/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferAllocator.java index 4f9d3c61c6..dbd6da3291 100644 --- a/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferAllocator.java +++ b/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferAllocator.java @@ -25,9 +25,10 @@ public interface BufferAllocator extends AutoCloseable { /** - * Allocate a new or reused buffer of the provided size. Note that the buffer may technically be - * larger than the requested size for rounding purposes. However, the buffer's capacity will be - * set to the configured size. + * Allocate a new or reused buffer of the provided size. The buffer may be larger than the + * requested size for rounding purposes (e.g. to a power of two), and the buffer's capacity will + * reflect the actual allocated size. Use {@link ArrowBuf#capacity(long)} to set the capacity to + * the requested size if needed. * * @param size The size in bytes. * @return a new ArrowBuf, or null if the request can't be satisfied @@ -36,9 +37,10 @@ public interface BufferAllocator extends AutoCloseable { ArrowBuf buffer(long size); /** - * Allocate a new or reused buffer of the provided size. Note that the buffer may technically be - * larger than the requested size for rounding purposes. However, the buffer's capacity will be - * set to the configured size. + * Allocate a new or reused buffer of the provided size. The buffer may be larger than the + * requested size for rounding purposes (e.g. to a power of two), and the buffer's capacity will + * reflect the actual allocated size. Use {@link ArrowBuf#capacity(long)} to set the capacity to + * the requested size if needed. * * @param size The size in bytes. * @param manager A buffer manager to manage reallocation. From 60a1a424200675710cc002c87b5a9a3afc6bada5 Mon Sep 17 00:00:00 2001 From: Aleksei Starikov Date: Wed, 11 Feb 2026 12:00:23 +0100 Subject: [PATCH 281/408] GH-1014: [Docs] Fix broken and obsolete links in the README.md (#1015) ## What's Changed Fix broken links in `README.md` file. Remove the unused reference [2]: https://github.com/apache/arrow/blob/main/cpp/README.md. Inline footnote links. Closes #1014. --- README.md | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index c46b61c49e..32a7e82811 100644 --- a/README.md +++ b/README.md @@ -23,11 +23,11 @@ The following guides explain the fundamental data structures used in the Java implementation of Apache Arrow. -- [ValueVector](https://arrow.apache.org/docs/java/vector.html) is an abstraction that is used to store a sequence of values having the same type in an individual column. -- [VectorSchemaRoot](https://arrow.apache.org/docs/java/vector_schema_root.html) is a container that can hold multiple vectors based on a schema. -- The [Reading/Writing IPC formats](https://arrow.apache.org/docs/java/ipc.html) guide explains how to stream record batches as well as serializing record batches to files. +- [ValueVector](https://arrow.apache.org/java/current/vector.html) is an abstraction that is used to store a sequence of values having the same type in an individual column. +- [VectorSchemaRoot](https://arrow.apache.org/java/current/vector_schema_root.html#vectorschemaroot) is a container that can hold multiple vectors based on a schema. +- The [Reading/Writing IPC formats](https://arrow.apache.org/java/current/ipc.html) guide explains how to stream record batches as well as serializing record batches to files. -Generated javadoc documentation is available [here](https://arrow.apache.org/docs/java/). +Generated javadoc documentation is available [here](https://arrow.apache.org/java/current/). ## Building from source @@ -93,7 +93,7 @@ conflicting or duplicate fields set this JVM flag or use the correct static cons ## Java Code Style Guide -Arrow Java follows the Google style guide [here][3] with the following +Arrow Java follows the [Google Java Style Guide](http://google.github.io/styleguide/javaguide.html) with the following differences: * Imports are grouped, from top to bottom, in this order: static imports, @@ -119,12 +119,12 @@ following command run in the project root directory: mvn -Dlogback.configurationFile=file: ``` -See [Logback Configuration][1] for more details. +See [Logback Configuration](https://logback.qos.ch/manual/configuration.html) for more details. ## Integration Tests Integration tests which require more time or more memory can be run by activating -the `integration-tests` profile. This activates the [maven failsafe][4] plugin +the `integration-tests` profile. This activates the [Maven Failsafe](https://maven.apache.org/surefire/maven-failsafe-plugin/) plugin and any class prefixed with `IT` will be run during the testing phase. The integration tests currently require a larger amount of memory (>4GB) and time to complete. To activate the profile: @@ -133,7 +133,3 @@ the profile: mvn -Pintegration-tests ``` -[1]: https://logback.qos.ch/manual/configuration.html -[2]: https://github.com/apache/arrow/blob/main/cpp/README.md -[3]: http://google.github.io/styleguide/javaguide.html -[4]: https://maven.apache.org/surefire/maven-failsafe-plugin/ From 2b74309ba4d5844593d6a3191d7b958f18468ae9 Mon Sep 17 00:00:00 2001 From: Aleksei Starikov Date: Mon, 16 Feb 2026 11:42:29 +0100 Subject: [PATCH 282/408] MINOR: [Docs] Remove extra line in README.md (fix pre-commit) (#1018) ## What's Changed Remove extra line in README.md for fixing the `end-of-file-fixer` hook in the `pre-commit` build. E.g. [here](https://github.com/apache/arrow-java/actions/runs/21902454512/job/63233962122) in the main branch build. ``` fix end of files.........................................................Failed - hook id: end-of-file-fixer - exit code: 1 - files were modified by this hook Fixing README.md ``` --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 32a7e82811..b0715aadf1 100644 --- a/README.md +++ b/README.md @@ -132,4 +132,3 @@ the profile: ```bash mvn -Pintegration-tests ``` - From bc7132b92c9c8ab693e5264cddf5145a3eed902e Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 16 Feb 2026 22:02:33 +0900 Subject: [PATCH 283/408] GH-1021: Use released apache/arrow instead of main (#1022) ## What's Changed In general, we should use released apache/arrow for apache/arrow-java release. Closes #1021. --- .github/workflows/rc.yml | 44 +++++++++++++++------------------------- 1 file changed, 16 insertions(+), 28 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index efa69533d3..37b2209966 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -107,18 +107,12 @@ jobs: - name: Extract source archive run: | tar -xf apache-arrow-java-*.tar.gz --strip-components=1 - # We always use the main branch for apache/arrow for now. - # Because we want to use - # https://github.com/apache/arrow/pull/45114 in - # apache/arrow-java. We can revert this workaround once Apache - # Arrow 20.0.0 that includes the change released. - # - # - name: Download the latest Apache Arrow C++ - # if: github.event_name != 'schedule' - # run: | - # ci/scripts/download_cpp.sh + - name: Download the latest Apache Arrow C++ + if: github.event_name != 'schedule' + run: | + ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ - # if: github.event_name == 'schedule' + if: github.event_name == 'schedule' uses: actions/checkout@v6 with: repository: apache/arrow @@ -180,12 +174,12 @@ jobs: - name: Extract source archive run: | tar -xf apache-arrow-java-*.tar.gz --strip-components=1 - # - name: Download the latest Apache Arrow C++ - # if: github.event_name != 'schedule' - # run: | - # ci/scripts/download_cpp.sh + - name: Download the latest Apache Arrow C++ + if: github.event_name != 'schedule' + run: | + ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ - # if: github.event_name == 'schedule' + if: github.event_name == 'schedule' uses: actions/checkout@v6 with: repository: apache/arrow @@ -309,19 +303,13 @@ jobs: shell: bash run: | tar -xf apache-arrow-java-*.tar.gz --strip-components=1 - # We always use the main branch for apache/arrow for now. - # Because we want to use - # https://github.com/apache/arrow/pull/47749 in - # apache/arrow-java. We can revert this workaround once Apache - # Arrow 22.0.0 that includes the change released. - # - # - name: Download the latest Apache Arrow C++ - # if: github.event_name != 'schedule' - # shell: bash - # run: | - # ci/scripts/download_cpp.sh + - name: Download the latest Apache Arrow C++ + if: github.event_name != 'schedule' + shell: bash + run: | + ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ - # if: github.event_name == 'schedule' + if: github.event_name == 'schedule' uses: actions/checkout@v6 with: repository: apache/arrow From 6dbc5d3690168e381f6b70c94638a9907c5489b3 Mon Sep 17 00:00:00 2001 From: Tamas Mate <50709850+tmater@users.noreply.github.com> Date: Tue, 17 Feb 2026 06:16:03 +0100 Subject: [PATCH 284/408] GH-946: Add Variant extension type support (#947) ### Summary This PR adds support for the Variant extension type in Arrow Java, enabling storage and manipulation of semi-structured variant data with metadata and value buffers. ### Changes A new `arrow-variant` module introduces the `Variant` class for parsing and working with variant data. This module is separated from the core vector module to isolate the `parquet-variant` dependency, so users of the Arrow vector library don't have to depend on Parquet. This also maintains a clean API boundary between Arrow's core functionality and variant-specific parsing logic. The core vector module gains `VariantType` as an extension type along with `VariantVector` for storing variant data as metadata/value buffer pairs. The implementation includes reader and writer support through `VariantReaderImpl`, `VariantWriterImpl`, and `NullableVariantHolderReaderImpl`, with corresponding holder classes for use in generated code paths. ### Testing - Unit tests for `VariantType`, `VariantVector`, and `Variant` parsing - Integration tests with `ListVector` and `MapVector` - Extension type round-trip tests Closes #946 --- arrow-variant/pom.xml | 51 ++ arrow-variant/src/main/java/module-info.java | 28 + .../org/apache/arrow/variant/Variant.java | 217 +++++ .../arrow/variant/extension/VariantType.java | 93 ++ .../variant/extension/VariantVector.java | 348 ++++++++ .../holders/NullableVariantHolder.java | 56 ++ .../arrow/variant/holders/VariantHolder.java | 56 ++ .../impl/NullableVariantHolderReaderImpl.java | 69 ++ .../arrow/variant/impl/VariantReaderImpl.java | 73 ++ .../arrow/variant/impl/VariantWriterImpl.java | 121 +++ .../org/apache/arrow/variant/TestVariant.java | 439 +++++++++ .../extension/TestVariantExtensionType.java | 249 ++++++ .../extension/TestVariantInListVector.java | 202 +++++ .../extension/TestVariantInMapVector.java | 125 +++ .../variant/extension/TestVariantType.java | 308 +++++++ .../variant/extension/TestVariantVector.java | 844 ++++++++++++++++++ bom/pom.xml | 5 + pom.xml | 2 + .../templates/AbstractFieldReader.java | 4 +- 19 files changed, 3288 insertions(+), 2 deletions(-) create mode 100644 arrow-variant/pom.xml create mode 100644 arrow-variant/src/main/java/module-info.java create mode 100644 arrow-variant/src/main/java/org/apache/arrow/variant/Variant.java create mode 100644 arrow-variant/src/main/java/org/apache/arrow/variant/extension/VariantType.java create mode 100644 arrow-variant/src/main/java/org/apache/arrow/variant/extension/VariantVector.java create mode 100644 arrow-variant/src/main/java/org/apache/arrow/variant/holders/NullableVariantHolder.java create mode 100644 arrow-variant/src/main/java/org/apache/arrow/variant/holders/VariantHolder.java create mode 100644 arrow-variant/src/main/java/org/apache/arrow/variant/impl/NullableVariantHolderReaderImpl.java create mode 100644 arrow-variant/src/main/java/org/apache/arrow/variant/impl/VariantReaderImpl.java create mode 100644 arrow-variant/src/main/java/org/apache/arrow/variant/impl/VariantWriterImpl.java create mode 100644 arrow-variant/src/test/java/org/apache/arrow/variant/TestVariant.java create mode 100644 arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantExtensionType.java create mode 100644 arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantInListVector.java create mode 100644 arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantInMapVector.java create mode 100644 arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantType.java create mode 100644 arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantVector.java diff --git a/arrow-variant/pom.xml b/arrow-variant/pom.xml new file mode 100644 index 0000000000..3a842178a4 --- /dev/null +++ b/arrow-variant/pom.xml @@ -0,0 +1,51 @@ + + + + 4.0.0 + + org.apache.arrow + arrow-java-root + 19.0.0-SNAPSHOT + + arrow-variant + Arrow Variant + Arrow Variant type support. + + + + org.apache.arrow + arrow-memory-core + + + org.apache.arrow + arrow-vector + + + org.apache.parquet + parquet-variant + ${dep.parquet.version} + + + org.apache.arrow + arrow-memory-unsafe + test + + + diff --git a/arrow-variant/src/main/java/module-info.java b/arrow-variant/src/main/java/module-info.java new file mode 100644 index 0000000000..da94173969 --- /dev/null +++ b/arrow-variant/src/main/java/module-info.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@SuppressWarnings("requires-automatic") +module org.apache.arrow.variant { + exports org.apache.arrow.variant; + exports org.apache.arrow.variant.extension; + exports org.apache.arrow.variant.impl; + exports org.apache.arrow.variant.holders; + + requires org.apache.arrow.memory.core; + requires org.apache.arrow.vector; + requires parquet.variant; +} diff --git a/arrow-variant/src/main/java/org/apache/arrow/variant/Variant.java b/arrow-variant/src/main/java/org/apache/arrow/variant/Variant.java new file mode 100644 index 0000000000..fa05cdd93f --- /dev/null +++ b/arrow-variant/src/main/java/org/apache/arrow/variant/Variant.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant; + +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.util.Objects; +import java.util.UUID; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.variant.holders.NullableVariantHolder; + +/** + * Wrapper around parquet-variant's Variant implementation. + * + *

This wrapper exists to isolate the parquet-variant dependency from Arrow's public API, + * allowing the vector module to expose variant functionality without requiring users to depend on + * parquet-variant directly. It also ensures that nested variant values (from arrays and objects) + * are consistently wrapped. + */ +public class Variant { + + private final org.apache.parquet.variant.Variant delegate; + + /** Creates a Variant from raw metadata and value byte arrays. */ + public Variant(byte[] metadata, byte[] value) { + this.delegate = new org.apache.parquet.variant.Variant(value, metadata); + } + + /** Creates a Variant by copying data from ArrowBuf instances. */ + public Variant( + ArrowBuf metadataBuffer, + int metadataStart, + int metadataEnd, + ArrowBuf valueBuffer, + int valueStart, + int valueEnd) { + byte[] metadata = new byte[metadataEnd - metadataStart]; + byte[] value = new byte[valueEnd - valueStart]; + metadataBuffer.getBytes(metadataStart, metadata); + valueBuffer.getBytes(valueStart, value); + this.delegate = new org.apache.parquet.variant.Variant(value, metadata); + } + + private Variant(org.apache.parquet.variant.Variant delegate) { + this.delegate = delegate; + } + + /** Constructs a Variant from a NullableVariantHolder. */ + public Variant(NullableVariantHolder holder) { + this( + holder.metadataBuffer, + holder.metadataStart, + holder.metadataEnd, + holder.valueBuffer, + holder.valueStart, + holder.valueEnd); + } + + public ByteBuffer getValueBuffer() { + return delegate.getValueBuffer(); + } + + public ByteBuffer getMetadataBuffer() { + return delegate.getMetadataBuffer(); + } + + public boolean getBoolean() { + return delegate.getBoolean(); + } + + public byte getByte() { + return delegate.getByte(); + } + + public short getShort() { + return delegate.getShort(); + } + + public int getInt() { + return delegate.getInt(); + } + + public long getLong() { + return delegate.getLong(); + } + + public double getDouble() { + return delegate.getDouble(); + } + + public BigDecimal getDecimal() { + return delegate.getDecimal(); + } + + public float getFloat() { + return delegate.getFloat(); + } + + public ByteBuffer getBinary() { + return delegate.getBinary(); + } + + public UUID getUUID() { + return delegate.getUUID(); + } + + public String getString() { + return delegate.getString(); + } + + public Type getType() { + return Type.fromParquet(delegate.getType()); + } + + public int numObjectElements() { + return delegate.numObjectElements(); + } + + public Variant getFieldByKey(String key) { + org.apache.parquet.variant.Variant result = delegate.getFieldByKey(key); + return result != null ? wrap(result) : null; + } + + public ObjectField getFieldAtIndex(int idx) { + org.apache.parquet.variant.Variant.ObjectField field = delegate.getFieldAtIndex(idx); + return new ObjectField(field.key, wrap(field.value)); + } + + public int numArrayElements() { + return delegate.numArrayElements(); + } + + public Variant getElementAtIndex(int index) { + org.apache.parquet.variant.Variant result = delegate.getElementAtIndex(index); + return result != null ? wrap(result) : null; + } + + private static Variant wrap(org.apache.parquet.variant.Variant parquetVariant) { + return new Variant(parquetVariant); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Variant variant = (Variant) o; + return delegate.getMetadataBuffer().equals(variant.delegate.getMetadataBuffer()) + && delegate.getValueBuffer().equals(variant.delegate.getValueBuffer()); + } + + @Override + public int hashCode() { + return Objects.hash(delegate.getMetadataBuffer(), delegate.getValueBuffer()); + } + + @Override + public String toString() { + return "Variant{type=" + getType() + '}'; + } + + public enum Type { + OBJECT, + ARRAY, + NULL, + BOOLEAN, + BYTE, + SHORT, + INT, + LONG, + STRING, + DOUBLE, + DECIMAL4, + DECIMAL8, + DECIMAL16, + DATE, + TIMESTAMP_TZ, + TIMESTAMP_NTZ, + FLOAT, + BINARY, + TIME, + TIMESTAMP_NANOS_TZ, + TIMESTAMP_NANOS_NTZ, + UUID; + + static Type fromParquet(org.apache.parquet.variant.Variant.Type parquetType) { + return Type.valueOf(parquetType.name()); + } + } + + public static final class ObjectField { + public final String key; + public final Variant value; + + public ObjectField(String key, Variant value) { + this.key = key; + this.value = value; + } + } +} diff --git a/arrow-variant/src/main/java/org/apache/arrow/variant/extension/VariantType.java b/arrow-variant/src/main/java/org/apache/arrow/variant/extension/VariantType.java new file mode 100644 index 0000000000..3deb70cdc0 --- /dev/null +++ b/arrow-variant/src/main/java/org/apache/arrow/variant/extension/VariantType.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant.extension; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.variant.impl.VariantWriterImpl; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.complex.writer.FieldWriter; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ArrowType.ExtensionType; +import org.apache.arrow.vector.types.pojo.ExtensionTypeRegistry; +import org.apache.arrow.vector.types.pojo.FieldType; + +/** + * Arrow extension type for Parquet + * Variant binary encoding. The type itself does not support shredded variant data. + */ +public final class VariantType extends ExtensionType { + + public static final VariantType INSTANCE = new VariantType(); + + public static final String EXTENSION_NAME = "parquet.variant"; + + static { + ExtensionTypeRegistry.register(INSTANCE); + } + + private VariantType() {} + + @Override + public ArrowType storageType() { + return ArrowType.Struct.INSTANCE; + } + + @Override + public String extensionName() { + return EXTENSION_NAME; + } + + @Override + public boolean extensionEquals(ExtensionType other) { + return other instanceof VariantType; + } + + @Override + public String serialize() { + return ""; + } + + @Override + public ArrowType deserialize(ArrowType storageType, String serializedData) { + if (!storageType.equals(this.storageType())) { + throw new UnsupportedOperationException( + "Cannot construct VariantType from underlying type " + storageType); + } + return INSTANCE; + } + + @Override + public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator) { + return new VariantVector(name, allocator); + } + + @Override + public boolean isComplex() { + // The type itself is not complex meaning we need separate functions to convert/extract + // different types. + // Meanwhile, the containing vector is complex in terms of containing multiple values (metadata + // and value) + return false; + } + + @Override + public FieldWriter getNewFieldWriter(ValueVector vector) { + return new VariantWriterImpl((VariantVector) vector); + } +} diff --git a/arrow-variant/src/main/java/org/apache/arrow/variant/extension/VariantVector.java b/arrow-variant/src/main/java/org/apache/arrow/variant/extension/VariantVector.java new file mode 100644 index 0000000000..1bbf1a6bdb --- /dev/null +++ b/arrow-variant/src/main/java/org/apache/arrow/variant/extension/VariantVector.java @@ -0,0 +1,348 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant.extension; + +import java.nio.ByteBuffer; +import java.util.List; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.util.hash.ArrowBufHasher; +import org.apache.arrow.variant.Variant; +import org.apache.arrow.variant.holders.NullableVariantHolder; +import org.apache.arrow.variant.holders.VariantHolder; +import org.apache.arrow.vector.BitVectorHelper; +import org.apache.arrow.vector.ExtensionTypeVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.complex.AbstractStructVector; +import org.apache.arrow.vector.complex.StructVector; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ArrowType.Binary; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.TransferPair; + +/** + * Arrow vector for storing {@link VariantType} values. + * + *

Stores semi-structured data (like JSON) as metadata + value binary pairs, allowing + * type-flexible columnar storage within Arrow's type system. + */ +public class VariantVector extends ExtensionTypeVector { + + public static final String METADATA_VECTOR_NAME = "metadata"; + public static final String VALUE_VECTOR_NAME = "value"; + + private final Field rootField; + + /** + * Constructs a new VariantVector with the given name and allocator. + * + * @param name the name of the vector + * @param allocator the buffer allocator for memory management + */ + public VariantVector(String name, BufferAllocator allocator) { + super( + name, + allocator, + new StructVector( + name, + allocator, + FieldType.nullable(ArrowType.Struct.INSTANCE), + null, + AbstractStructVector.ConflictPolicy.CONFLICT_ERROR, + false)); + rootField = createVariantField(name); + ((FieldVector) this.getUnderlyingVector()) + .initializeChildrenFromFields(rootField.getChildren()); + } + + /** + * Creates a new VariantVector with the given name. The Variant Field schema has to be the same + * everywhere, otherwise ArrowBuffer loading might fail during serialization/deserialization and + * schema mismatches can occur. This includes CompleteType's VARIANT and VARIANT_REQUIRED types. + */ + public static Field createVariantField(String name) { + return new Field( + name, new FieldType(true, VariantType.INSTANCE, null), createVariantChildFields()); + } + + /** + * Creates the child fields for the VariantVector. Metadata vector will be index 0 and value + * vector will be index 1. + */ + public static List createVariantChildFields() { + return List.of( + new Field(METADATA_VECTOR_NAME, new FieldType(false, Binary.INSTANCE, null), null), + new Field(VALUE_VECTOR_NAME, new FieldType(false, Binary.INSTANCE, null), null)); + } + + @Override + public void initializeChildrenFromFields(List children) { + // No-op, as children are initialized in the constructor + } + + @Override + public Field getField() { + return rootField; + } + + public VarBinaryVector getMetadataVector() { + return getUnderlyingVector().getChild(METADATA_VECTOR_NAME, VarBinaryVector.class); + } + + public VarBinaryVector getValueVector() { + return getUnderlyingVector().getChild(VALUE_VECTOR_NAME, VarBinaryVector.class); + } + + @Override + public TransferPair makeTransferPair(ValueVector target) { + return new VariantTransferPair(this, (VariantVector) target); + } + + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator) { + return new VariantTransferPair(this, new VariantVector(field.getName(), allocator)); + } + + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator, CallBack callBack) { + return getTransferPair(field, allocator); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new VariantTransferPair(this, new VariantVector(ref, allocator)); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return getTransferPair(ref, allocator); + } + + @Override + public TransferPair getTransferPair(BufferAllocator allocator) { + return getTransferPair(this.getField().getName(), allocator); + } + + @Override + public void copyFrom(int fromIndex, int thisIndex, ValueVector from) { + getUnderlyingVector() + .copyFrom(fromIndex, thisIndex, ((VariantVector) from).getUnderlyingVector()); + } + + @Override + public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { + getUnderlyingVector() + .copyFromSafe(fromIndex, thisIndex, ((VariantVector) from).getUnderlyingVector()); + } + + @Override + public Object getObject(int index) { + if (isNull(index)) { + return null; + } + VarBinaryVector metadataVector = getMetadataVector(); + VarBinaryVector valueVector = getValueVector(); + + int metadataStart = metadataVector.getStartOffset(index); + int metadataEnd = metadataVector.getEndOffset(index); + int valueStart = valueVector.getStartOffset(index); + int valueEnd = valueVector.getEndOffset(index); + + return new Variant( + metadataVector.getDataBuffer(), + metadataStart, + metadataEnd, + valueVector.getDataBuffer(), + valueStart, + valueEnd); + } + + /** + * Retrieves the variant value at the specified index into the provided holder. + * + * @param index the index of the value to retrieve + * @param holder the holder to populate with the variant data + */ + public void get(int index, NullableVariantHolder holder) { + if (isNull(index)) { + holder.isSet = 0; + } else { + holder.isSet = 1; + VarBinaryVector metadataVector = getMetadataVector(); + VarBinaryVector valueVector = getValueVector(); + assert !metadataVector.isNull(index) && !valueVector.isNull(index); + + holder.metadataStart = metadataVector.getStartOffset(index); + holder.metadataEnd = metadataVector.getEndOffset(index); + holder.metadataBuffer = metadataVector.getDataBuffer(); + holder.valueStart = valueVector.getStartOffset(index); + holder.valueEnd = valueVector.getEndOffset(index); + holder.valueBuffer = valueVector.getDataBuffer(); + } + } + + /** + * Retrieves the variant value at the specified index into the provided non-nullable holder. + * + * @param index the index of the value to retrieve + * @param holder the holder to populate with the variant data + */ + public void get(int index, VariantHolder holder) { + VarBinaryVector metadataVector = getMetadataVector(); + VarBinaryVector valueVector = getValueVector(); + assert !metadataVector.isNull(index) && !valueVector.isNull(index); + + holder.metadataStart = metadataVector.getStartOffset(index); + holder.metadataEnd = metadataVector.getEndOffset(index); + holder.metadataBuffer = metadataVector.getDataBuffer(); + holder.valueStart = valueVector.getStartOffset(index); + holder.valueEnd = valueVector.getEndOffset(index); + holder.valueBuffer = valueVector.getDataBuffer(); + } + + /** + * Sets the variant value at the specified index from the provided holder. + * + * @param index the index at which to set the value + * @param holder the holder containing the variant data to set + */ + public void set(int index, VariantHolder holder) { + BitVectorHelper.setBit(getUnderlyingVector().getValidityBuffer(), index); + getMetadataVector() + .set(index, 1, holder.metadataStart, holder.metadataEnd, holder.metadataBuffer); + getValueVector().set(index, 1, holder.valueStart, holder.valueEnd, holder.valueBuffer); + } + + /** + * Sets the variant value at the specified index from the provided nullable holder. + * + * @param index the index at which to set the value + * @param holder the nullable holder containing the variant data to set + */ + public void set(int index, NullableVariantHolder holder) { + BitVectorHelper.setValidityBit(getUnderlyingVector().getValidityBuffer(), index, holder.isSet); + if (holder.isSet == 0) { + return; + } + getMetadataVector() + .set(index, 1, holder.metadataStart, holder.metadataEnd, holder.metadataBuffer); + getValueVector().set(index, 1, holder.valueStart, holder.valueEnd, holder.valueBuffer); + } + + /** + * Sets the variant value at the specified index from the provided holder, with bounds checking. + * + * @param index the index at which to set the value + * @param holder the holder containing the variant data to set + */ + public void setSafe(int index, VariantHolder holder) { + getUnderlyingVector().setIndexDefined(index); + getMetadataVector() + .setSafe(index, 1, holder.metadataStart, holder.metadataEnd, holder.metadataBuffer); + getValueVector().setSafe(index, 1, holder.valueStart, holder.valueEnd, holder.valueBuffer); + } + + /** + * Sets the variant value at the specified index from the provided nullable holder, with bounds + * checking. + * + * @param index the index at which to set the value + * @param holder the nullable holder containing the variant data to set + */ + public void setSafe(int index, NullableVariantHolder holder) { + if (holder.isSet == 0) { + getUnderlyingVector().setNull(index); + return; + } + getUnderlyingVector().setIndexDefined(index); + getMetadataVector() + .setSafe(index, 1, holder.metadataStart, holder.metadataEnd, holder.metadataBuffer); + getValueVector().setSafe(index, 1, holder.valueStart, holder.valueEnd, holder.valueBuffer); + } + + /** Sets the value at the given index from the provided Variant. */ + public void setSafe(int index, Variant variant) { + ByteBuffer metadataBuffer = variant.getMetadataBuffer(); + ByteBuffer valueBuffer = variant.getValueBuffer(); + int metadataLength = metadataBuffer.remaining(); + int valueLength = valueBuffer.remaining(); + try (ArrowBuf metaBuf = getAllocator().buffer(metadataLength); + ArrowBuf valBuf = getAllocator().buffer(valueLength)) { + metaBuf.setBytes(0, metadataBuffer.duplicate()); + valBuf.setBytes(0, valueBuffer.duplicate()); + getUnderlyingVector().setIndexDefined(index); + getMetadataVector().setSafe(index, 1, 0, metadataLength, metaBuf); + getValueVector().setSafe(index, 1, 0, valueLength, valBuf); + } + } + + @Override + protected FieldReader getReaderImpl() { + return new org.apache.arrow.variant.impl.VariantReaderImpl(this); + } + + @Override + public int hashCode(int index) { + return hashCode(index, null); + } + + @Override + public int hashCode(int index, ArrowBufHasher hasher) { + return getUnderlyingVector().hashCode(index, hasher); + } + + /** + * VariantTransferPair is a transfer pair for VariantVector. It transfers the metadata and value + * together using the underlyingVector's transfer pair. + */ + protected static class VariantTransferPair implements TransferPair { + private final TransferPair pair; + private final VariantVector from; + private final VariantVector to; + + public VariantTransferPair(VariantVector from, VariantVector to) { + this.from = from; + this.to = to; + this.pair = from.getUnderlyingVector().makeTransferPair((to).getUnderlyingVector()); + } + + @Override + public void transfer() { + pair.transfer(); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + pair.splitAndTransfer(startIndex, length); + } + + @Override + public ValueVector getTo() { + return to; + } + + @Override + public void copyValueSafe(int from, int to) { + pair.copyValueSafe(from, to); + } + } +} diff --git a/arrow-variant/src/main/java/org/apache/arrow/variant/holders/NullableVariantHolder.java b/arrow-variant/src/main/java/org/apache/arrow/variant/holders/NullableVariantHolder.java new file mode 100644 index 0000000000..b78d4a2013 --- /dev/null +++ b/arrow-variant/src/main/java/org/apache/arrow/variant/holders/NullableVariantHolder.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant.holders; + +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.variant.extension.VariantType; +import org.apache.arrow.vector.holders.ExtensionHolder; +import org.apache.arrow.vector.types.pojo.ArrowType; + +@SuppressWarnings("checkstyle:VisibilityModifier") +public final class NullableVariantHolder extends ExtensionHolder { + + public int isSet; + public int metadataStart; + public int metadataEnd; + public ArrowBuf metadataBuffer; + public int valueStart; + public int valueEnd; + public ArrowBuf valueBuffer; + + public NullableVariantHolder() {} + + @Override + public boolean equals(Object obj) { + throw new UnsupportedOperationException(); + } + + @Override + public int hashCode() { + throw new UnsupportedOperationException(); + } + + @Override + public String toString() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowType type() { + return VariantType.INSTANCE; + } +} diff --git a/arrow-variant/src/main/java/org/apache/arrow/variant/holders/VariantHolder.java b/arrow-variant/src/main/java/org/apache/arrow/variant/holders/VariantHolder.java new file mode 100644 index 0000000000..e3947ac439 --- /dev/null +++ b/arrow-variant/src/main/java/org/apache/arrow/variant/holders/VariantHolder.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant.holders; + +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.variant.extension.VariantType; +import org.apache.arrow.vector.holders.ExtensionHolder; +import org.apache.arrow.vector.types.pojo.ArrowType; + +@SuppressWarnings("checkstyle:VisibilityModifier") +public final class VariantHolder extends ExtensionHolder { + + public final int isSet = 1; + public int metadataStart; + public int metadataEnd; + public ArrowBuf metadataBuffer; + public int valueStart; + public int valueEnd; + public ArrowBuf valueBuffer; + + public VariantHolder() {} + + @Override + public boolean equals(Object obj) { + throw new UnsupportedOperationException(); + } + + @Override + public int hashCode() { + throw new UnsupportedOperationException(); + } + + @Override + public String toString() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowType type() { + return VariantType.INSTANCE; + } +} diff --git a/arrow-variant/src/main/java/org/apache/arrow/variant/impl/NullableVariantHolderReaderImpl.java b/arrow-variant/src/main/java/org/apache/arrow/variant/impl/NullableVariantHolderReaderImpl.java new file mode 100644 index 0000000000..1645529c0c --- /dev/null +++ b/arrow-variant/src/main/java/org/apache/arrow/variant/impl/NullableVariantHolderReaderImpl.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant.impl; + +import org.apache.arrow.variant.holders.NullableVariantHolder; +import org.apache.arrow.vector.complex.impl.AbstractFieldReader; +import org.apache.arrow.vector.types.Types; + +public class NullableVariantHolderReaderImpl extends AbstractFieldReader { + private final NullableVariantHolder holder; + + public NullableVariantHolderReaderImpl(NullableVariantHolder holder) { + this.holder = holder; + } + + @Override + public int size() { + throw new UnsupportedOperationException("You can't call size on a Holder value reader."); + } + + @Override + public boolean next() { + throw new UnsupportedOperationException("You can't call next on a single value reader."); + } + + @Override + public void setPosition(int index) { + throw new UnsupportedOperationException("You can't call setPosition on a single value reader."); + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.EXTENSIONTYPE; + } + + @Override + public boolean isSet() { + return holder.isSet == 1; + } + + /** + * Reads the variant holder data into the provided holder. + * + * @param h the holder to read into + */ + public void read(NullableVariantHolder h) { + h.metadataStart = this.holder.metadataStart; + h.metadataEnd = this.holder.metadataEnd; + h.metadataBuffer = this.holder.metadataBuffer; + h.valueStart = this.holder.valueStart; + h.valueEnd = this.holder.valueEnd; + h.valueBuffer = this.holder.valueBuffer; + h.isSet = this.isSet() ? 1 : 0; + } +} diff --git a/arrow-variant/src/main/java/org/apache/arrow/variant/impl/VariantReaderImpl.java b/arrow-variant/src/main/java/org/apache/arrow/variant/impl/VariantReaderImpl.java new file mode 100644 index 0000000000..670104b7d1 --- /dev/null +++ b/arrow-variant/src/main/java/org/apache/arrow/variant/impl/VariantReaderImpl.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant.impl; + +import org.apache.arrow.variant.extension.VariantVector; +import org.apache.arrow.variant.holders.NullableVariantHolder; +import org.apache.arrow.variant.holders.VariantHolder; +import org.apache.arrow.vector.complex.impl.AbstractFieldReader; +import org.apache.arrow.vector.holders.ExtensionHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.Field; + +public class VariantReaderImpl extends AbstractFieldReader { + private final VariantVector vector; + + public VariantReaderImpl(VariantVector vector) { + this.vector = vector; + } + + @Override + public Types.MinorType getMinorType() { + return this.vector.getMinorType(); + } + + @Override + public Field getField() { + return this.vector.getField(); + } + + @Override + public boolean isSet() { + return !this.vector.isNull(this.idx()); + } + + @Override + public void read(ExtensionHolder holder) { + if (holder instanceof VariantHolder) { + vector.get(idx(), (VariantHolder) holder); + } else if (holder instanceof NullableVariantHolder) { + vector.get(idx(), (NullableVariantHolder) holder); + } else { + throw new IllegalArgumentException( + "Unsupported holder type for VariantReader: " + holder.getClass()); + } + } + + public void read(VariantHolder h) { + this.vector.get(this.idx(), h); + } + + public void read(NullableVariantHolder h) { + this.vector.get(this.idx(), h); + } + + @Override + public Object readObject() { + return this.vector.getObject(this.idx()); + } +} diff --git a/arrow-variant/src/main/java/org/apache/arrow/variant/impl/VariantWriterImpl.java b/arrow-variant/src/main/java/org/apache/arrow/variant/impl/VariantWriterImpl.java new file mode 100644 index 0000000000..266ddb75d2 --- /dev/null +++ b/arrow-variant/src/main/java/org/apache/arrow/variant/impl/VariantWriterImpl.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant.impl; + +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.variant.Variant; +import org.apache.arrow.variant.extension.VariantVector; +import org.apache.arrow.variant.holders.NullableVariantHolder; +import org.apache.arrow.variant.holders.VariantHolder; +import org.apache.arrow.vector.complex.impl.AbstractExtensionTypeWriter; +import org.apache.arrow.vector.holders.ExtensionHolder; +import org.apache.arrow.vector.types.pojo.ArrowType; + +/** + * Writer implementation for VARIANT extension type vectors. + * + *

This writer handles writing variant data to a {@link VariantVector}. It accepts both {@link + * VariantHolder} and {@link NullableVariantHolder} objects containing metadata and value buffers + * and writes them to the appropriate position in the vector. + */ +public class VariantWriterImpl extends AbstractExtensionTypeWriter { + + private static final String UNSUPPORTED_TYPE_TEMPLATE = "Unsupported type for Variant: %s"; + + /** + * Constructs a new VariantWriterImpl for the given vector. + * + * @param vector the variant vector to write to + */ + public VariantWriterImpl(VariantVector vector) { + super(vector); + } + + /** + * Writes an extension type or variant value to the vector. + * + *

This method handles {@link ExtensionHolder} by delegating to {@link #write(ExtensionHolder)} + * and {@link Variant} by delegating to {@link #writeVariant(Variant)}. + * + * @param object the object to write, must be an {@link ExtensionHolder} or {@link Variant} + * @throws IllegalArgumentException if the object is not an {@link ExtensionHolder} or {@link + * Variant} + */ + @Override + public void writeExtension(Object object) { + if (object instanceof ExtensionHolder) { + write((ExtensionHolder) object); + } else if (object instanceof Variant) { + writeVariant((Variant) object); + } else { + throw new IllegalArgumentException( + String.format(UNSUPPORTED_TYPE_TEMPLATE, object.getClass().getName())); + } + } + + private void writeVariant(Variant variant) { + java.nio.ByteBuffer metadataBuffer = variant.getMetadataBuffer(); + java.nio.ByteBuffer valueBuffer = variant.getValueBuffer(); + int metadataLength = metadataBuffer.remaining(); + int valueLength = valueBuffer.remaining(); + try (ArrowBuf metadataBuf = vector.getAllocator().buffer(metadataLength); + ArrowBuf valueBuf = vector.getAllocator().buffer(valueLength)) { + metadataBuf.setBytes(0, metadataBuffer.duplicate()); + valueBuf.setBytes(0, valueBuffer.duplicate()); + NullableVariantHolder holder = new NullableVariantHolder(); + holder.isSet = 1; + holder.metadataBuffer = metadataBuf; + holder.metadataStart = 0; + holder.metadataEnd = metadataLength; + holder.valueBuffer = valueBuf; + holder.valueStart = 0; + holder.valueEnd = valueLength; + vector.setSafe(getPosition(), holder); + vector.setValueCount(getPosition() + 1); + } + } + + @Override + public void writeExtension(Object value, ArrowType type) { + writeExtension(value); + } + + /** + * Writes a variant holder to the vector at the current position. + * + *

The holder can be either a {@link VariantHolder} (non-nullable, always set) or a {@link + * NullableVariantHolder} (nullable, may be null). The data is written using {@link + * VariantVector#setSafe(int, NullableVariantHolder)} which handles buffer allocation and copying. + * + * @param extensionHolder the variant holder to write, must be a {@link VariantHolder} or {@link + * NullableVariantHolder} + * @throws IllegalArgumentException if the holder is neither a {@link VariantHolder} nor a {@link + * NullableVariantHolder} + */ + @Override + public void write(ExtensionHolder extensionHolder) { + if (extensionHolder instanceof VariantHolder) { + vector.setSafe(getPosition(), (VariantHolder) extensionHolder); + } else if (extensionHolder instanceof NullableVariantHolder) { + vector.setSafe(getPosition(), (NullableVariantHolder) extensionHolder); + } else { + throw new IllegalArgumentException( + String.format(UNSUPPORTED_TYPE_TEMPLATE, extensionHolder.getClass().getName())); + } + vector.setValueCount(getPosition() + 1); + } +} diff --git a/arrow-variant/src/test/java/org/apache/arrow/variant/TestVariant.java b/arrow-variant/src/test/java/org/apache/arrow/variant/TestVariant.java new file mode 100644 index 0000000000..bc46a68616 --- /dev/null +++ b/arrow-variant/src/test/java/org/apache/arrow/variant/TestVariant.java @@ -0,0 +1,439 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.util.UUID; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.parquet.variant.VariantBuilder; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class TestVariant { + + private BufferAllocator allocator; + + @BeforeEach + void beforeEach() { + allocator = new RootAllocator(); + } + + @AfterEach + void afterEach() { + allocator.close(); + } + + static Variant buildVariant(VariantBuilder builder) { + org.apache.parquet.variant.Variant parquetVariant = builder.build(); + ByteBuffer valueBuf = parquetVariant.getValueBuffer(); + ByteBuffer metaBuf = parquetVariant.getMetadataBuffer(); + byte[] valueBytes = new byte[valueBuf.remaining()]; + byte[] metaBytes = new byte[metaBuf.remaining()]; + valueBuf.get(valueBytes); + metaBuf.get(metaBytes); + return new Variant(metaBytes, valueBytes); + } + + public static Variant variantString(String value) { + VariantBuilder builder = new VariantBuilder(); + builder.appendString(value); + return buildVariant(builder); + } + + @Test + void testConstructionWithArrowBuf() { + VariantBuilder builder = new VariantBuilder(); + builder.appendInt(42); + Variant source = buildVariant(builder); + int metaLen = source.getMetadataBuffer().remaining(); + int valueLen = source.getValueBuffer().remaining(); + + try (ArrowBuf metadataArrowBuf = allocator.buffer(metaLen + 2); + ArrowBuf valueArrowBuf = allocator.buffer(valueLen + 3)) { + metadataArrowBuf.setBytes(2, source.getMetadataBuffer()); + valueArrowBuf.setBytes(3, source.getValueBuffer()); + + Variant variant = + new Variant(metadataArrowBuf, 2, 2 + metaLen, valueArrowBuf, 3, 3 + valueLen); + + assertEquals(Variant.Type.INT, variant.getType()); + assertEquals(42, variant.getInt()); + } + } + + @Test + void testNullType() { + VariantBuilder builder = new VariantBuilder(); + builder.appendNull(); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.NULL, variant.getType()); + } + + @Test + void testBooleanType() { + VariantBuilder builder = new VariantBuilder(); + builder.appendBoolean(true); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.BOOLEAN, variant.getType()); + assertTrue(variant.getBoolean()); + + builder = new VariantBuilder(); + builder.appendBoolean(false); + variant = buildVariant(builder); + + assertEquals(Variant.Type.BOOLEAN, variant.getType()); + assertFalse(variant.getBoolean()); + } + + @Test + void testByteType() { + VariantBuilder builder = new VariantBuilder(); + builder.appendByte((byte) 42); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.BYTE, variant.getType()); + assertEquals((byte) 42, variant.getByte()); + } + + @Test + void testShortType() { + VariantBuilder builder = new VariantBuilder(); + builder.appendShort((short) 1234); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.SHORT, variant.getType()); + assertEquals((short) 1234, variant.getShort()); + } + + @Test + void testIntType() { + VariantBuilder builder = new VariantBuilder(); + builder.appendInt(123456); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.INT, variant.getType()); + assertEquals(123456, variant.getInt()); + } + + @Test + void testLongType() { + VariantBuilder builder = new VariantBuilder(); + builder.appendLong(9876543210L); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.LONG, variant.getType()); + assertEquals(9876543210L, variant.getLong()); + } + + @Test + void testFloatType() { + VariantBuilder builder = new VariantBuilder(); + builder.appendFloat(3.14f); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.FLOAT, variant.getType()); + assertEquals(3.14f, variant.getFloat(), 0.001f); + } + + @Test + void testDoubleType() { + VariantBuilder builder = new VariantBuilder(); + builder.appendDouble(3.14159265359); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.DOUBLE, variant.getType()); + assertEquals(3.14159265359, variant.getDouble(), 0.0000001); + } + + @Test + void testStringType() { + VariantBuilder builder = new VariantBuilder(); + builder.appendString("hello world"); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.STRING, variant.getType()); + assertEquals("hello world", variant.getString()); + } + + @Test + void testDecimalType() { + VariantBuilder builder = new VariantBuilder(); + builder.appendDecimal(new BigDecimal("123.456")); + Variant variant = buildVariant(builder); + + assertTrue( + variant.getType() == Variant.Type.DECIMAL4 + || variant.getType() == Variant.Type.DECIMAL8 + || variant.getType() == Variant.Type.DECIMAL16); + assertEquals(new BigDecimal("123.456"), variant.getDecimal()); + } + + @Test + void testBinaryType() { + VariantBuilder builder = new VariantBuilder(); + byte[] data = new byte[] {1, 2, 3, 4, 5}; + builder.appendBinary(ByteBuffer.wrap(data)); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.BINARY, variant.getType()); + ByteBuffer result = variant.getBinary(); + byte[] resultBytes = new byte[result.remaining()]; + result.get(resultBytes); + assertArrayEquals(data, resultBytes); + } + + @Test + void testUuidType() { + VariantBuilder builder = new VariantBuilder(); + UUID uuid = UUID.randomUUID(); + builder.appendUUID(uuid); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.UUID, variant.getType()); + assertEquals(uuid, variant.getUUID()); + } + + @Test + void testDateType() { + VariantBuilder builder = new VariantBuilder(); + int daysSinceEpoch = 19000; + builder.appendDate(daysSinceEpoch); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.DATE, variant.getType()); + } + + @Test + void testTimestampTzType() { + VariantBuilder builder = new VariantBuilder(); + long micros = System.currentTimeMillis() * 1000; + builder.appendTimestampTz(micros); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.TIMESTAMP_TZ, variant.getType()); + } + + @Test + void testTimestampNtzType() { + VariantBuilder builder = new VariantBuilder(); + long micros = System.currentTimeMillis() * 1000; + builder.appendTimestampNtz(micros); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.TIMESTAMP_NTZ, variant.getType()); + } + + @Test + void testTimeType() { + VariantBuilder builder = new VariantBuilder(); + long micros = 12345678L; + builder.appendTime(micros); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.TIME, variant.getType()); + } + + @Test + void testObjectType() { + VariantBuilder builder = new VariantBuilder(); + var objBuilder = builder.startObject(); + objBuilder.appendKey("name"); + objBuilder.appendString("test"); + objBuilder.appendKey("value"); + objBuilder.appendInt(42); + builder.endObject(); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.OBJECT, variant.getType()); + assertEquals(2, variant.numObjectElements()); + + Variant nameField = variant.getFieldByKey("name"); + assertNotNull(nameField); + assertEquals(Variant.Type.STRING, nameField.getType()); + assertEquals("test", nameField.getString()); + + Variant valueField = variant.getFieldByKey("value"); + assertNotNull(valueField); + assertEquals(Variant.Type.INT, valueField.getType()); + assertEquals(42, valueField.getInt()); + + assertNull(variant.getFieldByKey("nonexistent")); + + // Empty object + builder = new VariantBuilder(); + builder.startObject(); + builder.endObject(); + Variant emptyObj = buildVariant(builder); + assertEquals(Variant.Type.OBJECT, emptyObj.getType()); + assertEquals(0, emptyObj.numObjectElements()); + } + + @Test + void testObjectFieldAtIndex() { + VariantBuilder builder = new VariantBuilder(); + var objBuilder = builder.startObject(); + objBuilder.appendKey("alpha"); + objBuilder.appendInt(1); + objBuilder.appendKey("beta"); + objBuilder.appendInt(2); + builder.endObject(); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.OBJECT, variant.getType()); + assertEquals(2, variant.numObjectElements()); + + Variant.ObjectField field0 = variant.getFieldAtIndex(0); + assertNotNull(field0); + assertNotNull(field0.key); + assertNotNull(field0.value); + + Variant.ObjectField field1 = variant.getFieldAtIndex(1); + assertNotNull(field1); + assertNotNull(field1.key); + assertNotNull(field1.value); + } + + @Test + void testArrayType() { + VariantBuilder builder = new VariantBuilder(); + var arrayBuilder = builder.startArray(); + arrayBuilder.appendInt(1); + arrayBuilder.appendInt(2); + arrayBuilder.appendInt(3); + builder.endArray(); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.ARRAY, variant.getType()); + assertEquals(3, variant.numArrayElements()); + + Variant elem0 = variant.getElementAtIndex(0); + assertNotNull(elem0); + assertEquals(Variant.Type.INT, elem0.getType()); + assertEquals(1, elem0.getInt()); + + Variant elem1 = variant.getElementAtIndex(1); + assertEquals(2, elem1.getInt()); + + Variant elem2 = variant.getElementAtIndex(2); + assertEquals(3, elem2.getInt()); + + assertNull(variant.getElementAtIndex(-1)); + assertNull(variant.getElementAtIndex(3)); + + // Empty array + builder = new VariantBuilder(); + builder.startArray(); + builder.endArray(); + Variant emptyArr = buildVariant(builder); + assertEquals(Variant.Type.ARRAY, emptyArr.getType()); + assertEquals(0, emptyArr.numArrayElements()); + } + + @Test + void testNestedStructure() { + VariantBuilder builder = new VariantBuilder(); + var objBuilder = builder.startObject(); + objBuilder.appendKey("items"); + var arrayBuilder = objBuilder.startArray(); + arrayBuilder.appendString("a"); + arrayBuilder.appendString("b"); + objBuilder.endArray(); + builder.endObject(); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.OBJECT, variant.getType()); + Variant items = variant.getFieldByKey("items"); + assertNotNull(items); + assertEquals(Variant.Type.ARRAY, items.getType()); + assertEquals(2, items.numArrayElements()); + assertEquals("a", items.getElementAtIndex(0).getString()); + assertEquals("b", items.getElementAtIndex(1).getString()); + } + + @Test + void testEquals() { + VariantBuilder builder1 = new VariantBuilder(); + builder1.appendString("test"); + Variant variant1 = buildVariant(builder1); + + VariantBuilder builder2 = new VariantBuilder(); + builder2.appendString("test"); + Variant variant2 = buildVariant(builder2); + + VariantBuilder builder3 = new VariantBuilder(); + builder3.appendString("different"); + Variant variant3 = buildVariant(builder3); + + assertEquals(variant1, variant1); + assertEquals(variant1, variant2); + assertNotEquals(variant1, variant3); + assertNotEquals(variant1, null); + assertNotEquals(variant1, "not a variant"); + } + + @Test + void testHashCode() { + VariantBuilder builder1 = new VariantBuilder(); + builder1.appendInt(42); + Variant variant1 = buildVariant(builder1); + + VariantBuilder builder2 = new VariantBuilder(); + builder2.appendInt(42); + Variant variant2 = buildVariant(builder2); + + assertEquals(variant1.hashCode(), variant2.hashCode()); + } + + @Test + void testToString() { + VariantBuilder builder = new VariantBuilder(); + builder.appendString("test"); + Variant variant = buildVariant(builder); + + String str = variant.toString(); + assertNotNull(str); + assertTrue(str.contains("type=")); + } + + @Test + void testTypeEnumsMatch() { + for (Variant.Type arrowType : Variant.Type.values()) { + org.apache.parquet.variant.Variant.Type parquetType = + org.apache.parquet.variant.Variant.Type.valueOf(arrowType.name()); + assertEquals(arrowType, Variant.Type.fromParquet(parquetType)); + } + for (org.apache.parquet.variant.Variant.Type parquetType : + org.apache.parquet.variant.Variant.Type.values()) { + Variant.Type arrowType = Variant.Type.valueOf(parquetType.name()); + assertEquals(parquetType.name(), arrowType.name()); + } + } +} diff --git a/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantExtensionType.java b/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantExtensionType.java new file mode 100644 index 0000000000..f3213d523a --- /dev/null +++ b/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantExtensionType.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant.extension; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.channels.SeekableByteChannel; +import java.nio.channels.WritableByteChannel; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.Collections; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.variant.TestVariant; +import org.apache.arrow.variant.Variant; +import org.apache.arrow.vector.ExtensionTypeVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.compare.Range; +import org.apache.arrow.vector.compare.RangeEqualsVisitor; +import org.apache.arrow.vector.complex.StructVector; +import org.apache.arrow.vector.complex.writer.BaseWriter; +import org.apache.arrow.vector.ipc.ArrowFileReader; +import org.apache.arrow.vector.ipc.ArrowFileWriter; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ArrowType.ExtensionType; +import org.apache.arrow.vector.types.pojo.ExtensionTypeRegistry; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.arrow.vector.util.VectorBatchAppender; +import org.apache.arrow.vector.validate.ValidateVectorVisitor; +import org.junit.jupiter.api.Test; + +public class TestVariantExtensionType { + + private static void ensureRegistered(ArrowType.ExtensionType type) { + if (ExtensionTypeRegistry.lookup(type.extensionName()) == null) { + ExtensionTypeRegistry.register(type); + } + } + + @Test + public void roundtripVariant() throws IOException { + ensureRegistered(VariantType.INSTANCE); + final Schema schema = + new Schema(Collections.singletonList(Field.nullable("a", VariantType.INSTANCE))); + try (final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); + final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { + VariantVector vector = (VariantVector) root.getVector("a"); + vector.allocateNew(); + + vector.setSafe(0, TestVariant.variantString("hello")); + vector.setSafe(1, TestVariant.variantString("world")); + vector.setValueCount(2); + root.setRowCount(2); + + final File file = File.createTempFile("varianttest", ".arrow"); + try (final WritableByteChannel channel = + FileChannel.open(Paths.get(file.getAbsolutePath()), StandardOpenOption.WRITE); + final ArrowFileWriter writer = new ArrowFileWriter(root, null, channel)) { + writer.start(); + writer.writeBatch(); + writer.end(); + } + + try (final SeekableByteChannel channel = + Files.newByteChannel(Paths.get(file.getAbsolutePath())); + final ArrowFileReader reader = new ArrowFileReader(channel, allocator)) { + reader.loadNextBatch(); + final VectorSchemaRoot readerRoot = reader.getVectorSchemaRoot(); + assertEquals(root.getSchema(), readerRoot.getSchema()); + + final Field field = readerRoot.getSchema().getFields().get(0); + final VariantType expectedType = VariantType.INSTANCE; + assertEquals( + field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_NAME), + expectedType.extensionName()); + assertEquals( + field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_METADATA), + expectedType.serialize()); + + final ExtensionTypeVector deserialized = + (ExtensionTypeVector) readerRoot.getFieldVectors().get(0); + assertEquals(vector.getValueCount(), deserialized.getValueCount()); + for (int i = 0; i < vector.getValueCount(); i++) { + assertEquals(vector.isNull(i), deserialized.isNull(i)); + if (!vector.isNull(i)) { + assertEquals(vector.getObject(i), deserialized.getObject(i)); + } + } + } + } + } + + @Test + public void readVariantAsUnderlyingType() throws IOException { + ensureRegistered(VariantType.INSTANCE); + final Schema schema = + new Schema(Collections.singletonList(VariantVector.createVariantField("a"))); + try (final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); + final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { + VariantVector vector = (VariantVector) root.getVector("a"); + vector.allocateNew(); + + vector.setSafe(0, TestVariant.variantString("hello")); + vector.setValueCount(1); + root.setRowCount(1); + + final File file = File.createTempFile("varianttest", ".arrow"); + try (final WritableByteChannel channel = + FileChannel.open(Paths.get(file.getAbsolutePath()), StandardOpenOption.WRITE); + final ArrowFileWriter writer = new ArrowFileWriter(root, null, channel)) { + writer.start(); + writer.writeBatch(); + writer.end(); + } + + ExtensionTypeRegistry.unregister(VariantType.INSTANCE); + + try (final SeekableByteChannel channel = + Files.newByteChannel(Paths.get(file.getAbsolutePath())); + final ArrowFileReader reader = new ArrowFileReader(channel, allocator)) { + reader.loadNextBatch(); + VectorSchemaRoot readRoot = reader.getVectorSchemaRoot(); + + // Verify schema properties + assertEquals(1, readRoot.getSchema().getFields().size()); + assertEquals("a", readRoot.getSchema().getFields().get(0).getName()); + assertTrue(readRoot.getSchema().getFields().get(0).getType() instanceof ArrowType.Struct); + + // Verify extension metadata is preserved + final Field field = readRoot.getSchema().getFields().get(0); + assertEquals( + VariantType.EXTENSION_NAME, + field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_NAME)); + assertEquals("", field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_METADATA)); + + // Verify vector type and row count + assertEquals(1, readRoot.getRowCount()); + FieldVector readVector = readRoot.getVector("a"); + assertEquals(StructVector.class, readVector.getClass()); + + // Verify value count matches + StructVector structVector = (StructVector) readVector; + assertEquals(vector.getValueCount(), structVector.getValueCount()); + + // Verify the underlying data can be accessed from child vectors + VarBinaryVector metadataVector = + structVector.getChild(VariantVector.METADATA_VECTOR_NAME, VarBinaryVector.class); + VarBinaryVector valueVector = + structVector.getChild(VariantVector.VALUE_VECTOR_NAME, VarBinaryVector.class); + assertNotNull(metadataVector); + assertNotNull(valueVector); + assertEquals(1, metadataVector.getValueCount()); + assertEquals(1, valueVector.getValueCount()); + } + } + } + + @Test + public void testVariantVectorCompare() { + VariantType variantType = VariantType.INSTANCE; + ExtensionTypeRegistry.register(variantType); + Variant hello = TestVariant.variantString("hello"); + Variant world = TestVariant.variantString("world"); + try (final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); + VariantVector a1 = + (VariantVector) + variantType.getNewVector("a", FieldType.nullable(variantType), allocator); + VariantVector a2 = + (VariantVector) + variantType.getNewVector("a", FieldType.nullable(variantType), allocator); + VariantVector bb = + (VariantVector) + variantType.getNewVector("a", FieldType.nullable(variantType), allocator)) { + + ValidateVectorVisitor validateVisitor = new ValidateVectorVisitor(); + validateVisitor.visit(a1, null); + + a1.allocateNew(); + a2.allocateNew(); + bb.allocateNew(); + + a1.setSafe(0, hello); + a1.setSafe(1, world); + a1.setValueCount(2); + + a2.setSafe(0, hello); + a2.setSafe(1, world); + a2.setValueCount(2); + + bb.setSafe(0, world); + bb.setSafe(1, hello); + bb.setValueCount(2); + + Range range = new Range(0, 0, a1.getValueCount()); + RangeEqualsVisitor visitor = new RangeEqualsVisitor(a1, a2); + assertTrue(visitor.rangeEquals(range)); + + visitor = new RangeEqualsVisitor(a1, bb); + assertFalse(visitor.rangeEquals(range)); + + VectorBatchAppender.batchAppend(a1, a2, bb); + assertEquals(6, a1.getValueCount()); + validateVisitor.visit(a1, null); + } + } + + @Test + public void testVariantCopyAsValueThrowsException() { + ensureRegistered(VariantType.INSTANCE); + try (BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); + VariantVector vector = new VariantVector("variant", allocator)) { + vector.allocateNew(); + vector.setSafe(0, TestVariant.variantString("hello")); + vector.setValueCount(1); + + var reader = vector.getReader(); + reader.setPosition(0); + + assertThrows( + IllegalArgumentException.class, () -> reader.copyAsValue((BaseWriter.StructWriter) null)); + } + } +} diff --git a/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantInListVector.java b/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantInListVector.java new file mode 100644 index 0000000000..8b6000bc46 --- /dev/null +++ b/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantInListVector.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant.extension; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.variant.TestVariant; +import org.apache.arrow.variant.Variant; +import org.apache.arrow.variant.holders.NullableVariantHolder; +import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.impl.UnionListReader; +import org.apache.arrow.vector.complex.impl.UnionListWriter; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.complex.writer.BaseWriter.ExtensionWriter; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class TestVariantInListVector { + + private BufferAllocator allocator; + + @BeforeEach + public void init() { + allocator = new RootAllocator(Long.MAX_VALUE); + } + + @AfterEach + public void terminate() throws Exception { + allocator.close(); + } + + @Test + public void testListVectorWithVariantExtensionType() { + final FieldType type = FieldType.nullable(VariantType.INSTANCE); + try (ListVector inVector = new ListVector("input", allocator, type, null)) { + Variant variant1 = TestVariant.variantString("hello"); + Variant variant2 = TestVariant.variantString("bye"); + + UnionListWriter writer = inVector.getWriter(); + writer.allocate(); + + writer.setPosition(0); + writer.startList(); + ExtensionWriter extensionWriter = writer.extension(VariantType.INSTANCE); + extensionWriter.writeExtension(variant1); + extensionWriter.writeExtension(variant2); + writer.endList(); + inVector.setValueCount(1); + + ArrayList resultSet = (ArrayList) inVector.getObject(0); + assertEquals(2, resultSet.size()); + assertEquals(variant1, resultSet.get(0)); + assertEquals(variant2, resultSet.get(1)); + } + } + + @Test + public void testListVectorReaderForVariantExtensionType() { + try (ListVector inVector = ListVector.empty("input", allocator)) { + Variant variant1 = TestVariant.variantString("hello"); + Variant variant2 = TestVariant.variantString("bye"); + + UnionListWriter writer = inVector.getWriter(); + writer.allocate(); + + writer.setPosition(0); + writer.startList(); + ExtensionWriter extensionWriter = writer.extension(VariantType.INSTANCE); + extensionWriter.writeExtension(variant1); + writer.endList(); + + writer.setPosition(1); + writer.startList(); + extensionWriter.writeExtension(variant2); + extensionWriter.writeExtension(variant2); + writer.endList(); + + inVector.setValueCount(2); + + UnionListReader reader = inVector.getReader(); + reader.setPosition(0); + assertTrue(reader.next()); + FieldReader variantReader = reader.reader(); + NullableVariantHolder resultHolder = new NullableVariantHolder(); + variantReader.read(resultHolder); + assertEquals(variant1, new Variant(resultHolder)); + + reader.setPosition(1); + assertTrue(reader.next()); + variantReader = reader.reader(); + variantReader.read(resultHolder); + assertEquals(variant2, new Variant(resultHolder)); + + assertTrue(reader.next()); + variantReader = reader.reader(); + variantReader.read(resultHolder); + assertEquals(variant2, new Variant(resultHolder)); + } + } + + @Test + public void testCopyFromForVariantExtensionType() { + try (ListVector inVector = ListVector.empty("input", allocator); + ListVector outVector = ListVector.empty("output", allocator)) { + Variant variant1 = TestVariant.variantString("hello"); + Variant variant2 = TestVariant.variantString("bye"); + + UnionListWriter writer = inVector.getWriter(); + writer.allocate(); + + writer.setPosition(0); + writer.startList(); + ExtensionWriter extensionWriter = writer.extension(VariantType.INSTANCE); + extensionWriter.writeExtension(variant1); + writer.endList(); + + writer.setPosition(1); + writer.startList(); + extensionWriter.writeExtension(variant2); + extensionWriter.writeExtension(variant2); + writer.endList(); + + inVector.setValueCount(2); + + outVector.allocateNew(); + outVector.copyFrom(0, 0, inVector); + outVector.copyFrom(1, 1, inVector); + outVector.setValueCount(2); + + ArrayList resultSet0 = (ArrayList) outVector.getObject(0); + assertEquals(1, resultSet0.size()); + assertEquals(variant1, resultSet0.get(0)); + + ArrayList resultSet1 = (ArrayList) outVector.getObject(1); + assertEquals(2, resultSet1.size()); + assertEquals(variant2, resultSet1.get(0)); + assertEquals(variant2, resultSet1.get(1)); + } + } + + @Test + public void testCopyValueSafeForVariantExtensionType() { + try (ListVector inVector = ListVector.empty("input", allocator)) { + Variant variant1 = TestVariant.variantString("hello"); + Variant variant2 = TestVariant.variantString("bye"); + + UnionListWriter writer = inVector.getWriter(); + writer.allocate(); + + writer.setPosition(0); + writer.startList(); + ExtensionWriter extensionWriter = writer.extension(VariantType.INSTANCE); + extensionWriter.writeExtension(variant1); + writer.endList(); + + writer.setPosition(1); + writer.startList(); + extensionWriter.writeExtension(variant2); + extensionWriter.writeExtension(variant2); + writer.endList(); + + inVector.setValueCount(2); + + try (ListVector outVector = (ListVector) inVector.getTransferPair(allocator).getTo()) { + TransferPair tp = inVector.makeTransferPair(outVector); + tp.copyValueSafe(0, 0); + tp.copyValueSafe(1, 1); + outVector.setValueCount(2); + + ArrayList resultSet0 = (ArrayList) outVector.getObject(0); + assertEquals(1, resultSet0.size()); + assertEquals(variant1, resultSet0.get(0)); + + ArrayList resultSet1 = (ArrayList) outVector.getObject(1); + assertEquals(2, resultSet1.size()); + assertEquals(variant2, resultSet1.get(0)); + assertEquals(variant2, resultSet1.get(1)); + } + } + } +} diff --git a/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantInMapVector.java b/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantInMapVector.java new file mode 100644 index 0000000000..dd925810de --- /dev/null +++ b/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantInMapVector.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant.extension; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.variant.TestVariant; +import org.apache.arrow.variant.Variant; +import org.apache.arrow.variant.holders.NullableVariantHolder; +import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.impl.UnionMapReader; +import org.apache.arrow.vector.complex.impl.UnionMapWriter; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class TestVariantInMapVector { + + private BufferAllocator allocator; + + @BeforeEach + public void init() { + allocator = new RootAllocator(Long.MAX_VALUE); + } + + @AfterEach + public void terminate() { + allocator.close(); + } + + @Test + public void testMapVectorWithVariantExtensionType() { + Variant variant1 = TestVariant.variantString("hello"); + Variant variant2 = TestVariant.variantString("world"); + try (final MapVector inVector = MapVector.empty("map", allocator, false)) { + inVector.allocateNew(); + UnionMapWriter writer = inVector.getWriter(); + writer.setPosition(0); + + writer.startMap(); + writer.startEntry(); + writer.key().bigInt().writeBigInt(0); + writer.value().extension(VariantType.INSTANCE).writeExtension(variant1, VariantType.INSTANCE); + writer.endEntry(); + writer.startEntry(); + writer.key().bigInt().writeBigInt(1); + writer.value().extension(VariantType.INSTANCE).writeExtension(variant2, VariantType.INSTANCE); + writer.endEntry(); + writer.endMap(); + + writer.setValueCount(1); + + UnionMapReader mapReader = inVector.getReader(); + mapReader.setPosition(0); + mapReader.next(); + FieldReader variantReader = mapReader.value(); + NullableVariantHolder holder = new NullableVariantHolder(); + variantReader.read(holder); + assertEquals(variant1, new Variant(holder)); + + mapReader.next(); + variantReader = mapReader.value(); + variantReader.read(holder); + assertEquals(variant2, new Variant(holder)); + } + } + + @Test + public void testCopyFromForVariantExtensionType() { + Variant variant1 = TestVariant.variantString("hello"); + Variant variant2 = TestVariant.variantString("world"); + try (final MapVector inVector = MapVector.empty("in", allocator, false); + final MapVector outVector = MapVector.empty("out", allocator, false)) { + inVector.allocateNew(); + UnionMapWriter writer = inVector.getWriter(); + writer.setPosition(0); + + writer.startMap(); + writer.startEntry(); + writer.key().bigInt().writeBigInt(0); + writer.value().extension(VariantType.INSTANCE).writeExtension(variant1, VariantType.INSTANCE); + writer.endEntry(); + writer.startEntry(); + writer.key().bigInt().writeBigInt(1); + writer.value().extension(VariantType.INSTANCE).writeExtension(variant2, VariantType.INSTANCE); + writer.endEntry(); + writer.endMap(); + + writer.setValueCount(1); + outVector.allocateNew(); + outVector.copyFrom(0, 0, inVector); + outVector.setValueCount(1); + + UnionMapReader mapReader = outVector.getReader(); + mapReader.setPosition(0); + mapReader.next(); + FieldReader variantReader = mapReader.value(); + NullableVariantHolder holder = new NullableVariantHolder(); + variantReader.read(holder); + assertEquals(variant1, new Variant(holder)); + + mapReader.next(); + variantReader = mapReader.value(); + variantReader.read(holder); + assertEquals(variant2, new Variant(holder)); + } + } +} diff --git a/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantType.java b/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantType.java new file mode 100644 index 0000000000..017e71224b --- /dev/null +++ b/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantType.java @@ -0,0 +1,308 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant.extension; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.variant.holders.NullableVariantHolder; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.dictionary.DictionaryProvider; +import org.apache.arrow.vector.ipc.ArrowStreamReader; +import org.apache.arrow.vector.ipc.ArrowStreamWriter; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ExtensionTypeRegistry; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class TestVariantType { + BufferAllocator allocator; + + @BeforeEach + void beforeEach() { + allocator = new RootAllocator(); + } + + @AfterEach + void afterEach() { + allocator.close(); + } + + @Test + void testConstants() { + assertNotNull(VariantType.INSTANCE); + } + + @Test + void testStorageType() { + VariantType type = VariantType.INSTANCE; + assertEquals(ArrowType.Struct.INSTANCE, type.storageType()); + assertInstanceOf(ArrowType.Struct.class, type.storageType()); + } + + @Test + void testExtensionName() { + VariantType type = VariantType.INSTANCE; + assertEquals("parquet.variant", type.extensionName()); + } + + @Test + void testExtensionEquals() { + VariantType type1 = VariantType.INSTANCE; + VariantType type2 = VariantType.INSTANCE; + + assertTrue(type1.extensionEquals(type2)); + } + + @Test + void testIsComplex() { + VariantType type = VariantType.INSTANCE; + assertFalse(type.isComplex()); + } + + @Test + void testSerialize() { + VariantType type = VariantType.INSTANCE; + String serialized = type.serialize(); + assertEquals("", serialized); + } + + @Test + void testDeserializeValid() { + VariantType type = VariantType.INSTANCE; + ArrowType storageType = ArrowType.Struct.INSTANCE; + + ArrowType deserialized = assertDoesNotThrow(() -> type.deserialize(storageType, "")); + assertInstanceOf(VariantType.class, deserialized); + assertEquals(VariantType.INSTANCE, deserialized); + } + + @Test + void testDeserializeInvalidStorageType() { + VariantType type = VariantType.INSTANCE; + ArrowType wrongStorageType = ArrowType.Utf8.INSTANCE; + + assertThrows(UnsupportedOperationException.class, () -> type.deserialize(wrongStorageType, "")); + } + + @Test + void testGetNewVector() { + VariantType type = VariantType.INSTANCE; + try (FieldVector vector = + type.getNewVector("variant_field", FieldType.nullable(type), allocator)) { + assertInstanceOf(VariantVector.class, vector); + assertEquals("variant_field", vector.getField().getName()); + assertEquals(type, vector.getField().getType()); + } + } + + @Test + void testGetNewVectorWithNullableFieldType() { + VariantType type = VariantType.INSTANCE; + FieldType nullableFieldType = FieldType.nullable(type); + + try (FieldVector vector = type.getNewVector("nullable_variant", nullableFieldType, allocator)) { + assertInstanceOf(VariantVector.class, vector); + assertEquals("nullable_variant", vector.getField().getName()); + assertTrue(vector.getField().isNullable()); + } + } + + @Test + void testGetNewVectorWithNonNullableFieldType() { + VariantType type = VariantType.INSTANCE; + FieldType nonNullableFieldType = FieldType.notNullable(type); + + try (FieldVector vector = + type.getNewVector("non_nullable_variant", nonNullableFieldType, allocator)) { + assertInstanceOf(VariantVector.class, vector); + assertEquals("non_nullable_variant", vector.getField().getName()); + } + } + + @Test + void testIpcRoundTrip() { + VariantType type = VariantType.INSTANCE; + + Schema schema = new Schema(Collections.singletonList(Field.nullable("variant", type))); + byte[] serialized = schema.serializeAsMessage(); + Schema deserialized = Schema.deserializeMessage(ByteBuffer.wrap(serialized)); + assertEquals(schema, deserialized); + } + + @Test + void testVectorIpcRoundTrip() throws IOException { + VariantType type = VariantType.INSTANCE; + + try (FieldVector vector = type.getNewVector("field", FieldType.nullable(type), allocator); + ArrowBuf metadataBuf1 = allocator.buffer(10); + ArrowBuf valueBuf1 = allocator.buffer(10); + ArrowBuf metadataBuf2 = allocator.buffer(10); + ArrowBuf valueBuf2 = allocator.buffer(10)) { + VariantVector variantVector = (VariantVector) vector; + + byte[] metadata1 = new byte[] {1, 2, 3}; + byte[] value1 = new byte[] {4, 5, 6, 7}; + metadataBuf1.setBytes(0, metadata1); + valueBuf1.setBytes(0, value1); + + byte[] metadata2 = new byte[] {8, 9}; + byte[] value2 = new byte[] {10, 11, 12}; + metadataBuf2.setBytes(0, metadata2); + valueBuf2.setBytes(0, value2); + + NullableVariantHolder holder1 = new NullableVariantHolder(); + holder1.isSet = 1; + holder1.metadataStart = 0; + holder1.metadataEnd = metadata1.length; + holder1.metadataBuffer = metadataBuf1; + holder1.valueStart = 0; + holder1.valueEnd = value1.length; + holder1.valueBuffer = valueBuf1; + + NullableVariantHolder holder2 = new NullableVariantHolder(); + holder2.isSet = 1; + holder2.metadataStart = 0; + holder2.metadataEnd = metadata2.length; + holder2.metadataBuffer = metadataBuf2; + holder2.valueStart = 0; + holder2.valueEnd = value2.length; + holder2.valueBuffer = valueBuf2; + + variantVector.setSafe(0, holder1); + variantVector.setNull(1); + variantVector.setSafe(2, holder2); + variantVector.setValueCount(3); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (VectorSchemaRoot root = new VectorSchemaRoot(Collections.singletonList(variantVector)); + ArrowStreamWriter writer = + new ArrowStreamWriter(root, new DictionaryProvider.MapDictionaryProvider(), baos)) { + writer.start(); + writer.writeBatch(); + } + + try (ArrowStreamReader reader = + new ArrowStreamReader(new ByteArrayInputStream(baos.toByteArray()), allocator)) { + assertTrue(reader.loadNextBatch()); + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + assertEquals(3, root.getRowCount()); + assertEquals( + new Schema(Collections.singletonList(variantVector.getField())), root.getSchema()); + + VariantVector actual = assertInstanceOf(VariantVector.class, root.getVector("field")); + assertFalse(actual.isNull(0)); + assertTrue(actual.isNull(1)); + assertFalse(actual.isNull(2)); + + NullableVariantHolder result1 = new NullableVariantHolder(); + actual.get(0, result1); + assertEquals(1, result1.isSet); + assertEquals(metadata1.length, result1.metadataEnd - result1.metadataStart); + assertEquals(value1.length, result1.valueEnd - result1.valueStart); + + assertNull(actual.getObject(1)); + + NullableVariantHolder result2 = new NullableVariantHolder(); + actual.get(2, result2); + assertEquals(1, result2.isSet); + assertEquals(metadata2.length, result2.metadataEnd - result2.metadataStart); + assertEquals(value2.length, result2.valueEnd - result2.valueStart); + } + } + } + + @Test + void testSingleton() { + VariantType type1 = VariantType.INSTANCE; + VariantType type2 = VariantType.INSTANCE; + + // Same instance + assertSame(type1, type2); + assertTrue(type1.extensionEquals(type2)); + } + + @Test + void testExtensionTypeRegistry() { + // VariantType should be automatically registered via static initializer + ArrowType.ExtensionType registeredType = + ExtensionTypeRegistry.lookup(VariantType.EXTENSION_NAME); + assertNotNull(registeredType); + assertInstanceOf(VariantType.class, registeredType); + assertEquals(VariantType.INSTANCE, registeredType); + } + + @Test + void testFieldMetadata() { + Map metadata = new HashMap<>(); + metadata.put("key1", "value1"); + metadata.put("key2", "value2"); + + FieldType fieldType = new FieldType(true, VariantType.INSTANCE, null, metadata); + try (VariantVector vector = new VariantVector("test", allocator)) { + Field field = new Field("test", fieldType, VariantVector.createVariantChildFields()); + + // Field metadata includes both custom metadata and extension type metadata + Map fieldMetadata = field.getMetadata(); + assertEquals("value1", fieldMetadata.get("key1")); + assertEquals("value2", fieldMetadata.get("key2")); + // Extension type metadata is also present + assertTrue(fieldMetadata.containsKey("ARROW:extension:name")); + assertTrue(fieldMetadata.containsKey("ARROW:extension:metadata")); + } + } + + @Test + void testFieldChildren() { + try (VariantVector vector = new VariantVector("test", allocator)) { + Field field = vector.getField(); + + assertNotNull(field.getChildren()); + assertEquals(2, field.getChildren().size()); + + Field metadataField = field.getChildren().get(0); + assertEquals(VariantVector.METADATA_VECTOR_NAME, metadataField.getName()); + assertEquals(ArrowType.Binary.INSTANCE, metadataField.getType()); + + Field valueField = field.getChildren().get(1); + assertEquals(VariantVector.VALUE_VECTOR_NAME, valueField.getName()); + assertEquals(ArrowType.Binary.INSTANCE, valueField.getType()); + } + } +} diff --git a/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantVector.java b/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantVector.java new file mode 100644 index 0000000000..1c172e304f --- /dev/null +++ b/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantVector.java @@ -0,0 +1,844 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant.extension; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.variant.Variant; +import org.apache.arrow.variant.holders.NullableVariantHolder; +import org.apache.arrow.variant.holders.VariantHolder; +import org.apache.arrow.variant.impl.VariantReaderImpl; +import org.apache.arrow.variant.impl.VariantWriterImpl; +import org.apache.arrow.vector.holders.ExtensionHolder; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** Tests for VariantVector, VariantWriterImpl, and VariantReaderImpl. */ +class TestVariantVector { + + private BufferAllocator allocator; + + @BeforeEach + void beforeEach() { + allocator = new RootAllocator(); + } + + @AfterEach + void afterEach() { + allocator.close(); + } + + private VariantHolder createHolder( + ArrowBuf metadataBuf, byte[] metadata, ArrowBuf valueBuf, byte[] value) { + VariantHolder holder = new VariantHolder(); + holder.metadataStart = 0; + holder.metadataEnd = metadata.length; + holder.metadataBuffer = metadataBuf; + holder.valueStart = 0; + holder.valueEnd = value.length; + holder.valueBuffer = valueBuf; + return holder; + } + + private NullableVariantHolder createNullableHolder( + ArrowBuf metadataBuf, byte[] metadata, ArrowBuf valueBuf, byte[] value) { + NullableVariantHolder holder = new NullableVariantHolder(); + holder.isSet = 1; + holder.metadataStart = 0; + holder.metadataEnd = metadata.length; + holder.metadataBuffer = metadataBuf; + holder.valueStart = 0; + holder.valueEnd = value.length; + holder.valueBuffer = valueBuf; + return holder; + } + + private NullableVariantHolder createNullHolder() { + NullableVariantHolder holder = new NullableVariantHolder(); + holder.isSet = 0; + return holder; + } + + // ========== Basic Vector Tests ========== + + @Test + void testVectorCreation() { + try (VariantVector vector = new VariantVector("test", allocator)) { + assertNotNull(vector); + assertEquals("test", vector.getField().getName()); + assertNotNull(vector.getMetadataVector()); + assertNotNull(vector.getValueVector()); + } + } + + @Test + void testSetAndGet() { + try (VariantVector vector = new VariantVector("test", allocator); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1, 2, 3}; + byte[] value = new byte[] {4, 5, 6, 7}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + NullableVariantHolder holder = createNullableHolder(metadataBuf, metadata, valueBuf, value); + + vector.setSafe(0, holder); + vector.setValueCount(1); + + // Retrieve and verify + NullableVariantHolder result = new NullableVariantHolder(); + vector.get(0, result); + + assertEquals(1, result.isSet); + assertEquals(metadata.length, result.metadataEnd - result.metadataStart); + assertEquals(value.length, result.valueEnd - result.valueStart); + + byte[] actualMetadata = new byte[metadata.length]; + byte[] actualValue = new byte[value.length]; + result.metadataBuffer.getBytes(result.metadataStart, actualMetadata); + result.valueBuffer.getBytes(result.valueStart, actualValue); + + assertArrayEquals(metadata, actualMetadata); + assertArrayEquals(value, actualValue); + } + } + + @Test + void testSetNull() { + try (VariantVector vector = new VariantVector("test", allocator)) { + NullableVariantHolder holder = createNullHolder(); + + vector.setSafe(0, holder); + vector.setValueCount(1); + + assertTrue(vector.isNull(0)); + + NullableVariantHolder result = new NullableVariantHolder(); + vector.get(0, result); + assertEquals(0, result.isSet); + } + } + + @Test + void testMultipleValues() { + try (VariantVector vector = new VariantVector("test", allocator); + ArrowBuf metadataBuf1 = allocator.buffer(10); + ArrowBuf valueBuf1 = allocator.buffer(10); + ArrowBuf metadataBuf2 = allocator.buffer(10); + ArrowBuf valueBuf2 = allocator.buffer(10)) { + + byte[] metadata1 = new byte[] {1, 2}; + byte[] value1 = new byte[] {3, 4, 5}; + metadataBuf1.setBytes(0, metadata1); + valueBuf1.setBytes(0, value1); + + NullableVariantHolder holder1 = + createNullableHolder(metadataBuf1, metadata1, valueBuf1, value1); + + byte[] metadata2 = new byte[] {6, 7, 8}; + byte[] value2 = new byte[] {9, 10}; + metadataBuf2.setBytes(0, metadata2); + valueBuf2.setBytes(0, value2); + + NullableVariantHolder holder2 = + createNullableHolder(metadataBuf2, metadata2, valueBuf2, value2); + + vector.setSafe(0, holder1); + vector.setSafe(1, holder2); + vector.setValueCount(2); + + // Verify first value + NullableVariantHolder result1 = new NullableVariantHolder(); + vector.get(0, result1); + assertEquals(1, result1.isSet); + + byte[] actualMetadata1 = new byte[metadata1.length]; + byte[] actualValue1 = new byte[value1.length]; + result1.metadataBuffer.getBytes(result1.metadataStart, actualMetadata1); + result1.valueBuffer.getBytes(result1.valueStart, actualValue1); + assertArrayEquals(metadata1, actualMetadata1); + assertArrayEquals(value1, actualValue1); + + // Verify second value + NullableVariantHolder result2 = new NullableVariantHolder(); + vector.get(1, result2); + assertEquals(1, result2.isSet); + + byte[] actualMetadata2 = new byte[metadata2.length]; + byte[] actualValue2 = new byte[value2.length]; + result2.metadataBuffer.getBytes(result2.metadataStart, actualMetadata2); + result2.valueBuffer.getBytes(result2.valueStart, actualValue2); + assertArrayEquals(metadata2, actualMetadata2); + assertArrayEquals(value2, actualValue2); + } + } + + @Test + void testNonNullableHolder() { + try (VariantVector vector = new VariantVector("test", allocator); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1, 2, 3}; + byte[] value = new byte[] {4, 5, 6}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + VariantHolder holder = createHolder(metadataBuf, metadata, valueBuf, value); + + vector.setSafe(0, holder); + vector.setValueCount(1); + + assertFalse(vector.isNull(0)); + + NullableVariantHolder result = new NullableVariantHolder(); + vector.get(0, result); + assertEquals(1, result.isSet); + } + } + + // ========== Writer Tests ========== + + @Test + void testWriteWithVariantHolder() { + try (VariantVector vector = new VariantVector("test", allocator); + VariantWriterImpl writer = new VariantWriterImpl(vector); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1, 2}; + byte[] value = new byte[] {3, 4, 5}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + VariantHolder holder = createHolder(metadataBuf, metadata, valueBuf, value); + + writer.setPosition(0); + writer.write(holder); + + assertEquals(1, vector.getValueCount()); + assertFalse(vector.isNull(0)); + } + } + + @Test + void testWriteWithNullableVariantHolder() { + try (VariantVector vector = new VariantVector("test", allocator); + VariantWriterImpl writer = new VariantWriterImpl(vector); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1, 2}; + byte[] value = new byte[] {3, 4, 5}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + NullableVariantHolder holder = createNullableHolder(metadataBuf, metadata, valueBuf, value); + + writer.setPosition(0); + writer.write(holder); + + assertEquals(1, vector.getValueCount()); + assertFalse(vector.isNull(0)); + } + } + + @Test + void testWriteWithNullableVariantHolderNull() { + try (VariantVector vector = new VariantVector("test", allocator); + VariantWriterImpl writer = new VariantWriterImpl(vector)) { + + NullableVariantHolder holder = createNullHolder(); + + writer.setPosition(0); + writer.write(holder); + + assertEquals(1, vector.getValueCount()); + assertTrue(vector.isNull(0)); + } + } + + @Test + void testWriteExtensionWithUnsupportedType() { + try (VariantVector vector = new VariantVector("test", allocator); + VariantWriterImpl writer = new VariantWriterImpl(vector)) { + + writer.setPosition(0); + + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> writer.writeExtension("invalid-type")); + + assertTrue(exception.getMessage().contains("Unsupported type for Variant")); + } + } + + @Test + void testWriteWithUnsupportedHolder() { + try (VariantVector vector = new VariantVector("test", allocator); + VariantWriterImpl writer = new VariantWriterImpl(vector)) { + + ExtensionHolder unsupportedHolder = + new ExtensionHolder() { + @Override + public ArrowType type() { + return VariantType.INSTANCE; + } + }; + + writer.setPosition(0); + + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> writer.write(unsupportedHolder)); + + assertTrue(exception.getMessage().contains("Unsupported type for Variant")); + } + } + + // ========== Reader Tests ========== + + @Test + void testReaderReadWithNullableVariantHolder() { + try (VariantVector vector = new VariantVector("test", allocator); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1, 2, 3}; + byte[] value = new byte[] {4, 5, 6}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + NullableVariantHolder holder = createNullableHolder(metadataBuf, metadata, valueBuf, value); + + vector.setSafe(0, holder); + vector.setValueCount(1); + + VariantReaderImpl reader = (VariantReaderImpl) vector.getReader(); + reader.setPosition(0); + + NullableVariantHolder result = new NullableVariantHolder(); + reader.read(result); + + assertEquals(1, result.isSet); + assertEquals(metadata.length, result.metadataEnd - result.metadataStart); + assertEquals(value.length, result.valueEnd - result.valueStart); + } + } + + @Test + void testReaderReadWithNullableVariantHolderNull() { + try (VariantVector vector = new VariantVector("test", allocator)) { + vector.setNull(0); + vector.setValueCount(1); + + VariantReaderImpl reader = (VariantReaderImpl) vector.getReader(); + reader.setPosition(0); + + NullableVariantHolder holder = new NullableVariantHolder(); + reader.read(holder); + + assertEquals(0, holder.isSet); + } + } + + @Test + void testReaderIsSet() { + try (VariantVector vector = new VariantVector("test", allocator); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1}; + byte[] value = new byte[] {2}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + NullableVariantHolder holder = createNullableHolder(metadataBuf, metadata, valueBuf, value); + + vector.setSafe(0, holder); + vector.setNull(1); + vector.setValueCount(2); + + VariantReaderImpl reader = (VariantReaderImpl) vector.getReader(); + + reader.setPosition(0); + assertTrue(reader.isSet()); + + reader.setPosition(1); + assertFalse(reader.isSet()); + } + } + + @Test + void testReaderGetMinorType() { + try (VariantVector vector = new VariantVector("test", allocator)) { + VariantReaderImpl reader = (VariantReaderImpl) vector.getReader(); + assertEquals(vector.getMinorType(), reader.getMinorType()); + } + } + + @Test + void testReaderGetField() { + try (VariantVector vector = new VariantVector("test", allocator)) { + VariantReaderImpl reader = (VariantReaderImpl) vector.getReader(); + assertEquals(vector.getField(), reader.getField()); + assertEquals("test", reader.getField().getName()); + } + } + + @Test + void testReaderReadWithNonNullableVariantHolder() { + try (VariantVector vector = new VariantVector("test", allocator); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1, 2, 3}; + byte[] value = new byte[] {4, 5, 6}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + NullableVariantHolder holder = createNullableHolder(metadataBuf, metadata, valueBuf, value); + + vector.setSafe(0, holder); + vector.setValueCount(1); + + VariantReaderImpl reader = (VariantReaderImpl) vector.getReader(); + reader.setPosition(0); + + VariantHolder result = new VariantHolder(); + reader.read(result); + + // Verify the data was read correctly + byte[] actualMetadata = new byte[metadata.length]; + byte[] actualValue = new byte[value.length]; + result.metadataBuffer.getBytes(result.metadataStart, actualMetadata); + result.valueBuffer.getBytes(result.valueStart, actualValue); + + assertArrayEquals(metadata, actualMetadata); + assertArrayEquals(value, actualValue); + assertEquals(1, result.isSet); + } + } + + // ========== Transfer Pair Tests ========== + + @Test + void testTransferPair() { + try (VariantVector fromVector = new VariantVector("from", allocator); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1, 2, 3}; + byte[] value = new byte[] {4, 5, 6, 7}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + NullableVariantHolder holder = createNullableHolder(metadataBuf, metadata, valueBuf, value); + + fromVector.setSafe(0, holder); + fromVector.setValueCount(1); + + org.apache.arrow.vector.util.TransferPair transferPair = + fromVector.getTransferPair(allocator); + VariantVector toVector = (VariantVector) transferPair.getTo(); + + transferPair.transfer(); + + assertEquals(0, fromVector.getValueCount()); + assertEquals(1, toVector.getValueCount()); + + NullableVariantHolder result = new NullableVariantHolder(); + toVector.get(0, result); + assertEquals(1, result.isSet); + + byte[] actualMetadata = new byte[metadata.length]; + byte[] actualValue = new byte[value.length]; + result.metadataBuffer.getBytes(result.metadataStart, actualMetadata); + result.valueBuffer.getBytes(result.valueStart, actualValue); + + assertArrayEquals(metadata, actualMetadata); + assertArrayEquals(value, actualValue); + + toVector.close(); + } + } + + @Test + void testSplitAndTransfer() { + try (VariantVector fromVector = new VariantVector("from", allocator); + ArrowBuf metadataBuf1 = allocator.buffer(10); + ArrowBuf valueBuf1 = allocator.buffer(10); + ArrowBuf metadataBuf2 = allocator.buffer(10); + ArrowBuf valueBuf2 = allocator.buffer(10); + ArrowBuf metadataBuf3 = allocator.buffer(10); + ArrowBuf valueBuf3 = allocator.buffer(10)) { + + byte[] metadata1 = new byte[] {1}; + byte[] value1 = new byte[] {2, 3}; + metadataBuf1.setBytes(0, metadata1); + valueBuf1.setBytes(0, value1); + + byte[] metadata2 = new byte[] {4, 5}; + byte[] value2 = new byte[] {6}; + metadataBuf2.setBytes(0, metadata2); + valueBuf2.setBytes(0, value2); + + byte[] metadata3 = new byte[] {7, 8, 9}; + byte[] value3 = new byte[] {10, 11, 12}; + metadataBuf3.setBytes(0, metadata3); + valueBuf3.setBytes(0, value3); + + NullableVariantHolder holder1 = + createNullableHolder(metadataBuf1, metadata1, valueBuf1, value1); + NullableVariantHolder holder2 = + createNullableHolder(metadataBuf2, metadata2, valueBuf2, value2); + NullableVariantHolder holder3 = + createNullableHolder(metadataBuf3, metadata3, valueBuf3, value3); + + fromVector.setSafe(0, holder1); + fromVector.setSafe(1, holder2); + fromVector.setSafe(2, holder3); + fromVector.setValueCount(3); + + org.apache.arrow.vector.util.TransferPair transferPair = + fromVector.getTransferPair(allocator); + VariantVector toVector = (VariantVector) transferPair.getTo(); + + // Split and transfer indices 1-2 (middle and last) + transferPair.splitAndTransfer(1, 2); + + assertEquals(2, toVector.getValueCount()); + + // Verify transferred values + NullableVariantHolder result1 = new NullableVariantHolder(); + toVector.get(0, result1); + assertEquals(1, result1.isSet); + + byte[] actualMetadata1 = new byte[metadata2.length]; + byte[] actualValue1 = new byte[value2.length]; + result1.metadataBuffer.getBytes(result1.metadataStart, actualMetadata1); + result1.valueBuffer.getBytes(result1.valueStart, actualValue1); + assertArrayEquals(metadata2, actualMetadata1); + assertArrayEquals(value2, actualValue1); + + NullableVariantHolder result2 = new NullableVariantHolder(); + toVector.get(1, result2); + assertEquals(1, result2.isSet); + + byte[] actualMetadata2 = new byte[metadata3.length]; + byte[] actualValue2 = new byte[value3.length]; + result2.metadataBuffer.getBytes(result2.metadataStart, actualMetadata2); + result2.valueBuffer.getBytes(result2.valueStart, actualValue2); + assertArrayEquals(metadata3, actualMetadata2); + assertArrayEquals(value3, actualValue2); + + toVector.close(); + } + } + + @Test + void testCopyValueSafe() { + try (VariantVector fromVector = new VariantVector("from", allocator); + VariantVector toVector = new VariantVector("to", allocator); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1, 2}; + byte[] value = new byte[] {3, 4, 5}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + NullableVariantHolder holder = createNullableHolder(metadataBuf, metadata, valueBuf, value); + + fromVector.setSafe(0, holder); + fromVector.setValueCount(1); + + org.apache.arrow.vector.util.TransferPair transferPair = + fromVector.makeTransferPair(toVector); + + transferPair.copyValueSafe(0, 0); + toVector.setValueCount(1); + + // Verify the value was copied + NullableVariantHolder result = new NullableVariantHolder(); + toVector.get(0, result); + assertEquals(1, result.isSet); + + byte[] actualMetadata = new byte[metadata.length]; + byte[] actualValue = new byte[value.length]; + result.metadataBuffer.getBytes(result.metadataStart, actualMetadata); + result.valueBuffer.getBytes(result.valueStart, actualValue); + + assertArrayEquals(metadata, actualMetadata); + assertArrayEquals(value, actualValue); + + // Original vector should still have the value + NullableVariantHolder originalResult = new NullableVariantHolder(); + fromVector.get(0, originalResult); + assertEquals(1, originalResult.isSet); + } + } + + @Test + void testGetTransferPairWithField() { + try (VariantVector fromVector = new VariantVector("from", allocator); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1}; + byte[] value = new byte[] {2}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + NullableVariantHolder holder = createNullableHolder(metadataBuf, metadata, valueBuf, value); + + fromVector.setSafe(0, holder); + fromVector.setValueCount(1); + + org.apache.arrow.vector.util.TransferPair transferPair = + fromVector.getTransferPair(fromVector.getField(), allocator); + VariantVector toVector = (VariantVector) transferPair.getTo(); + + transferPair.transfer(); + + assertEquals(1, toVector.getValueCount()); + assertEquals(fromVector.getField().getName(), toVector.getField().getName()); + + toVector.close(); + } + } + + // ========== Copy Operations Tests ========== + + @Test + void testCopyFrom() { + try (VariantVector fromVector = new VariantVector("from", allocator); + VariantVector toVector = new VariantVector("to", allocator); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1, 2, 3}; + byte[] value = new byte[] {4, 5}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + NullableVariantHolder holder = createNullableHolder(metadataBuf, metadata, valueBuf, value); + + fromVector.setSafe(0, holder); + fromVector.setValueCount(1); + + toVector.allocateNew(); + toVector.copyFrom(0, 0, fromVector); + toVector.setValueCount(1); + + NullableVariantHolder result = new NullableVariantHolder(); + toVector.get(0, result); + assertEquals(1, result.isSet); + + byte[] actualMetadata = new byte[metadata.length]; + byte[] actualValue = new byte[value.length]; + result.metadataBuffer.getBytes(result.metadataStart, actualMetadata); + result.valueBuffer.getBytes(result.valueStart, actualValue); + + assertArrayEquals(metadata, actualMetadata); + assertArrayEquals(value, actualValue); + } + } + + @Test + void testCopyFromSafe() { + try (VariantVector fromVector = new VariantVector("from", allocator); + VariantVector toVector = new VariantVector("to", allocator); + ArrowBuf metadataBuf1 = allocator.buffer(10); + ArrowBuf valueBuf1 = allocator.buffer(10); + ArrowBuf metadataBuf2 = allocator.buffer(10); + ArrowBuf valueBuf2 = allocator.buffer(10)) { + + byte[] metadata1 = new byte[] {1}; + byte[] value1 = new byte[] {2, 3}; + metadataBuf1.setBytes(0, metadata1); + valueBuf1.setBytes(0, value1); + + NullableVariantHolder holder1 = + createNullableHolder(metadataBuf1, metadata1, valueBuf1, value1); + + byte[] metadata2 = new byte[] {4, 5}; + byte[] value2 = new byte[] {6}; + metadataBuf2.setBytes(0, metadata2); + valueBuf2.setBytes(0, value2); + + NullableVariantHolder holder2 = + createNullableHolder(metadataBuf2, metadata2, valueBuf2, value2); + + fromVector.setSafe(0, holder1); + fromVector.setSafe(1, holder2); + fromVector.setValueCount(2); + + // Copy without pre-allocating toVector + for (int i = 0; i < 2; i++) { + toVector.copyFromSafe(i, i, fromVector); + } + toVector.setValueCount(2); + + // Verify both values + NullableVariantHolder result1 = new NullableVariantHolder(); + toVector.get(0, result1); + assertEquals(1, result1.isSet); + + byte[] actualMetadata1 = new byte[metadata1.length]; + byte[] actualValue1 = new byte[value1.length]; + result1.metadataBuffer.getBytes(result1.metadataStart, actualMetadata1); + result1.valueBuffer.getBytes(result1.valueStart, actualValue1); + assertArrayEquals(metadata1, actualMetadata1); + assertArrayEquals(value1, actualValue1); + + NullableVariantHolder result2 = new NullableVariantHolder(); + toVector.get(1, result2); + assertEquals(1, result2.isSet); + + byte[] actualMetadata2 = new byte[metadata2.length]; + byte[] actualValue2 = new byte[value2.length]; + result2.metadataBuffer.getBytes(result2.metadataStart, actualMetadata2); + result2.valueBuffer.getBytes(result2.valueStart, actualValue2); + assertArrayEquals(metadata2, actualMetadata2); + assertArrayEquals(value2, actualValue2); + } + } + + @Test + void testCopyFromWithNulls() { + try (VariantVector fromVector = new VariantVector("from", allocator); + VariantVector toVector = new VariantVector("to", allocator); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1}; + byte[] value = new byte[] {2}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + NullableVariantHolder holder = createNullableHolder(metadataBuf, metadata, valueBuf, value); + + fromVector.setSafe(0, holder); + fromVector.setNull(1); + fromVector.setSafe(2, holder); + fromVector.setValueCount(3); + + toVector.allocateNew(); + for (int i = 0; i < 3; i++) { + toVector.copyFromSafe(i, i, fromVector); + } + toVector.setValueCount(3); + + assertFalse(toVector.isNull(0)); + assertTrue(toVector.isNull(1)); + assertFalse(toVector.isNull(2)); + } + } + + // ========== GetObject Tests ========== + + @Test + void testGetObject() { + try (VariantVector vector = new VariantVector("test", allocator); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1, 2}; + byte[] value = new byte[] {3, 4, 5}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + NullableVariantHolder holder = createNullableHolder(metadataBuf, metadata, valueBuf, value); + + vector.setSafe(0, holder); + vector.setValueCount(1); + + Object obj = vector.getObject(0); + assertNotNull(obj); + assertTrue(obj instanceof Variant); + assertEquals(new Variant(metadata, value), obj); + } + } + + @Test + void testGetObjectNull() { + try (VariantVector vector = new VariantVector("test", allocator)) { + vector.setNull(0); + vector.setValueCount(1); + + Object obj = vector.getObject(0); + assertNull(obj); + } + } + + // ========== Allocate and Capacity Tests ========== + + @Test + void testAllocateNew() { + try (VariantVector vector = new VariantVector("test", allocator)) { + vector.allocateNew(); + assertTrue(vector.getValueCapacity() > 0); + } + } + + @Test + void testSetInitialCapacity() { + try (VariantVector vector = new VariantVector("test", allocator)) { + vector.setInitialCapacity(100); + vector.allocateNew(); + assertTrue(vector.getValueCapacity() >= 100); + } + } + + @Test + void testClearAndReuse() { + try (VariantVector vector = new VariantVector("test", allocator); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1}; + byte[] value = new byte[] {2}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + NullableVariantHolder holder = createNullableHolder(metadataBuf, metadata, valueBuf, value); + + vector.setSafe(0, holder); + vector.setValueCount(1); + + assertFalse(vector.isNull(0)); + + vector.clear(); + vector.allocateNew(); + + // After clear, vector should be empty + assertEquals(0, vector.getValueCount()); + } + } +} diff --git a/bom/pom.xml b/bom/pom.xml index 9efde53243..b631f9366d 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -194,6 +194,11 @@ under the License. arrow-tools ${project.version} + + org.apache.arrow + arrow-variant + ${project.version} + diff --git a/pom.xml b/pom.xml index d64df1ade3..9ffaf6b60d 100644 --- a/pom.xml +++ b/pom.xml @@ -68,6 +68,7 @@ under the License. bom format memory + arrow-variant vector tools adapter/jdbc @@ -104,6 +105,7 @@ under the License. 3.4.2 25.2.10 1.12.1 + 1.17.0 5.17.0 2 diff --git a/vector/src/main/codegen/templates/AbstractFieldReader.java b/vector/src/main/codegen/templates/AbstractFieldReader.java index 556fb576ce..789295e959 100644 --- a/vector/src/main/codegen/templates/AbstractFieldReader.java +++ b/vector/src/main/codegen/templates/AbstractFieldReader.java @@ -29,9 +29,9 @@ * Source code generated using FreeMarker template ${.template_name} */ @SuppressWarnings("unused") -abstract class AbstractFieldReader extends AbstractBaseReader implements FieldReader{ +public abstract class AbstractFieldReader extends AbstractBaseReader implements FieldReader{ - AbstractFieldReader(){ + protected AbstractFieldReader(){ super(); } From 9d6237ed4761b0ae923499a24d1545bae6218add Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Feb 2026 07:49:34 +0100 Subject: [PATCH 285/408] MINOR: Bump org.mockito:mockito-bom from 5.17.0 to 5.21.0 (#1000) Bumps [org.mockito:mockito-bom](https://github.com/mockito/mockito) from 5.17.0 to 5.21.0.

Release notes

Sourced from org.mockito:mockito-bom's releases.

v5.21.0

Changelog generated by Shipkit Changelog Gradle Plugin

5.21.0

v5.20.0

Changelog generated by Shipkit Changelog Gradle Plugin

5.20.0

v5.19.0

Changelog generated by Shipkit Changelog Gradle Plugin

5.19.0

... (truncated)

Commits
  • 09d2230 Bump graalvm/setup-graalvm from 1.4.3 to 1.4.4 (#3768)
  • df3e0cc Bump graalvm/setup-graalvm from 1.4.2 to 1.4.3 (#3767)
  • 04a6e9f Bump actions/checkout from 5 to 6 (#3765)
  • 756a3cf Add description of matchers to potential mismatch (#3760)
  • 58ba445 Forbid mocking WeakReference with inline mock maker (#3759)
  • 966d600 Bump actions/upload-artifact from 4 to 5 (#3756)
  • 632bf7b Bump graalvm/setup-graalvm from 1.4.1 to 1.4.2 (#3755)
  • 8564b43 Fix primitives support in GenericArrayReturnType for Android (#3753)
  • bf3a809 Bump graalvm/setup-graalvm from 1.4.0 to 1.4.1 (#3744)
  • cffddd4 Bump gradle/actions from 4 to 5 (#3743)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.mockito:mockito-bom&package-manager=maven&previous-version=5.17.0&new-version=5.21.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9ffaf6b60d..1f57039015 100644 --- a/pom.xml +++ b/pom.xml @@ -106,7 +106,7 @@ under the License. 25.2.10 1.12.1 1.17.0 - 5.17.0 + 5.21.0 2 10.23.0 From 32984435ed1ffbe6ce1578e075da6eeb012d6bbb Mon Sep 17 00:00:00 2001 From: Aleksei Starikov Date: Tue, 17 Feb 2026 13:59:36 +0100 Subject: [PATCH 286/408] GH-130: Fix AutoCloseables to work with @Nullable structures (#1017) ## What's Changed `AutoCloseables` supposes to work with nullable `Iterables`, `varargs`, and `collection of nulls`. The PR introduces: - `@Nullable` annotation for all public methods in `AutoCloseables` (only private `flatten` method doesn't support null `Iterable`) - `null` checks to prevent NPEs --- The change is backward compatible. Only possible NPEs are prevented. --- Closes #130 . --- .../org/apache/arrow/util/AutoCloseables.java | 61 ++-- .../apache/arrow/util/TestAutoCloseables.java | 268 ++++++++++++++++++ 2 files changed, 311 insertions(+), 18 deletions(-) create mode 100644 memory/memory-core/src/test/java/org/apache/arrow/util/TestAutoCloseables.java diff --git a/memory/memory-core/src/main/java/org/apache/arrow/util/AutoCloseables.java b/memory/memory-core/src/main/java/org/apache/arrow/util/AutoCloseables.java index a39004a9d0..ba5a539a87 100644 --- a/memory/memory-core/src/main/java/org/apache/arrow/util/AutoCloseables.java +++ b/memory/memory-core/src/main/java/org/apache/arrow/util/AutoCloseables.java @@ -22,7 +22,9 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.stream.Stream; import java.util.stream.StreamSupport; +import org.checkerframework.checker.nullness.qual.Nullable; /** Utilities for AutoCloseable classes. */ public final class AutoCloseables { @@ -33,7 +35,8 @@ private AutoCloseables() {} * Returns a new {@link AutoCloseable} that calls {@link #close(Iterable)} on autoCloseables * when close is called. */ - public static AutoCloseable all(final Collection autoCloseables) { + public static AutoCloseable all( + final @Nullable Collection autoCloseables) { return new AutoCloseable() { @Override public void close() throws Exception { @@ -48,7 +51,10 @@ public void close() throws Exception { * @param t the throwable to add suppressed exception to * @param autoCloseables the closeables to close */ - public static void close(Throwable t, AutoCloseable... autoCloseables) { + public static void close(Throwable t, @Nullable AutoCloseable... autoCloseables) { + if (autoCloseables == null) { + return; + } close(t, Arrays.asList(autoCloseables)); } @@ -58,7 +64,8 @@ public static void close(Throwable t, AutoCloseable... autoCloseables) { * @param t the throwable to add suppressed exception to * @param autoCloseables the closeables to close */ - public static void close(Throwable t, Iterable autoCloseables) { + public static void close( + Throwable t, @Nullable Iterable autoCloseables) { try { close(autoCloseables); } catch (Exception e) { @@ -71,7 +78,10 @@ public static void close(Throwable t, Iterable autoClos * * @param autoCloseables the closeables to close */ - public static void close(AutoCloseable... autoCloseables) throws Exception { + public static void close(@Nullable AutoCloseable... autoCloseables) throws Exception { + if (autoCloseables == null) { + return; + } close(Arrays.asList(autoCloseables)); } @@ -80,7 +90,8 @@ public static void close(AutoCloseable... autoCloseables) throws Exception { * * @param ac the closeables to close */ - public static void close(Iterable ac) throws Exception { + public static void close(@Nullable Iterable ac) + throws Exception { // this method can be called on a single object if it implements Iterable // like for example VectorContainer make sure we handle that properly if (ac == null) { @@ -111,12 +122,17 @@ public static void close(Iterable ac) throws Exception /** Calls {@link #close(Iterable)} on the flattened list of closeables. */ @SafeVarargs - public static void close(Iterable... closeables) throws Exception { + public static void close(@Nullable Iterable... closeables) + throws Exception { + if (closeables == null) { + return; + } close(flatten(closeables)); } @SafeVarargs - private static Iterable flatten(Iterable... closeables) { + private static Iterable flatten( + Iterable... closeables) { return new Iterable() { // Cast from Iterable to Iterable is safe in this // context @@ -127,16 +143,18 @@ public Iterator iterator() { return Arrays.stream(closeables) .flatMap( (Iterable i) -> - StreamSupport.stream( - ((Iterable) i).spliterator(), /* parallel= */ false)) + i == null + ? Stream.empty() + : StreamSupport.stream( + ((Iterable) i).spliterator(), /* parallel= */ false)) .iterator(); } }; } /** Converts ac to a {@link Iterable} filtering out any null values. */ - public static Iterable iter(AutoCloseable... ac) { - if (ac.length == 0) { + public static Iterable iter(@Nullable AutoCloseable... ac) { + if (ac == null || ac.length == 0) { return Collections.emptyList(); } else { final List nonNullAc = new ArrayList<>(); @@ -153,10 +171,11 @@ public static Iterable iter(AutoCloseable... ac) { public static class RollbackCloseable implements AutoCloseable { private boolean commit = false; - private List closeables; + private final List closeables; - public RollbackCloseable(AutoCloseable... closeables) { - this.closeables = new ArrayList<>(Arrays.asList(closeables)); + public RollbackCloseable(@Nullable AutoCloseable... closeables) { + this.closeables = + closeables == null ? new ArrayList<>() : new ArrayList<>(Arrays.asList(closeables)); } public T add(T t) { @@ -165,12 +184,18 @@ public T add(T t) { } /** Add all of list to the rollback list. */ - public void addAll(AutoCloseable... list) { + public void addAll(@Nullable AutoCloseable... list) { + if (list == null) { + return; + } closeables.addAll(Arrays.asList(list)); } /** Add all of list to the rollback list. */ - public void addAll(Iterable list) { + public void addAll(@Nullable Iterable list) { + if (list == null) { + return; + } for (AutoCloseable ac : list) { closeables.add(ac); } @@ -189,7 +214,7 @@ public void close() throws Exception { } /** Creates an {@link RollbackCloseable} from the given closeables. */ - public static RollbackCloseable rollbackable(AutoCloseable... closeables) { + public static RollbackCloseable rollbackable(@Nullable AutoCloseable... closeables) { return new RollbackCloseable(closeables); } @@ -203,7 +228,7 @@ public static RollbackCloseable rollbackable(AutoCloseable... closeables) { * @throws RuntimeException if an Exception occurs; the Exception is wrapped by the * RuntimeException */ - public static void closeNoChecked(final AutoCloseable autoCloseable) { + public static void closeNoChecked(final @Nullable AutoCloseable autoCloseable) { if (autoCloseable != null) { try { autoCloseable.close(); diff --git a/memory/memory-core/src/test/java/org/apache/arrow/util/TestAutoCloseables.java b/memory/memory-core/src/test/java/org/apache/arrow/util/TestAutoCloseables.java new file mode 100644 index 0000000000..ba5b78178a --- /dev/null +++ b/memory/memory-core/src/test/java/org/apache/arrow/util/TestAutoCloseables.java @@ -0,0 +1,268 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import org.junit.jupiter.api.Test; + +public class TestAutoCloseables { + + /** Closeable that records that it was closed and can optionally throw. */ + private static final class TrackCloseable implements AutoCloseable { + private boolean closed; + private final Exception toThrow; + + TrackCloseable() { + this.toThrow = null; + } + + TrackCloseable(Exception toThrow) { + this.toThrow = toThrow; + } + + @Override + public void close() throws Exception { + closed = true; + if (toThrow != null) { + throw toThrow; + } + } + + boolean isClosed() { + return closed; + } + } + + @Test + public void testCloseVarargsIgnoresNulls() throws Exception { + TrackCloseable a = new TrackCloseable(); + TrackCloseable b = new TrackCloseable(); + AutoCloseables.close(a, null, b); + assertTrue(a.isClosed()); + assertTrue(b.isClosed()); + } + + @Test + public void testCloseVarargsThrowsFirstExceptionAndSuppressesRest() throws Exception { + Exception e1 = new Exception("first"); + Exception e2 = new Exception("second"); + TrackCloseable c1 = new TrackCloseable(e1); + TrackCloseable c2 = new TrackCloseable(e2); + Exception thrown = assertThrows(Exception.class, () -> AutoCloseables.close(c1, c2)); + assertEquals("first", thrown.getMessage()); + assertTrue(Arrays.asList(thrown.getSuppressed()).contains(e2)); + } + + @Test + public void testCloseIterableNullIterableReturns() throws Exception { + AutoCloseables.close((List) null); // no exception + } + + @Test + public void testCloseIterableIgnoresNullElements() throws Exception { + TrackCloseable a = new TrackCloseable(); + TrackCloseable b = new TrackCloseable(); + List list = Arrays.asList(a, null, b); + AutoCloseables.close(list); + assertTrue(a.isClosed()); + assertTrue(b.isClosed()); + } + + @Test + public void testCloseIterableWhenIterableIsAlsoAutoCloseable() throws Exception { + TrackCloseable iter = new TrackCloseable(); + TrackCloseable inner = new TrackCloseable(); + // When the Iterable itself implements AutoCloseable (e.g. VectorContainer), + // close(Iterable) calls close() on it and does not iterate over elements + class IterableCloseable implements Iterable, AutoCloseable { + @Override + @SuppressWarnings("unchecked") + public Iterator iterator() { + return (Iterator) Collections.singletonList(inner); + } + + @Override + public void close() throws Exception { + iter.close(); + } + } + AutoCloseables.close(new IterableCloseable()); + assertTrue(iter.isClosed()); + assertFalse(inner.isClosed()); + } + + @Test + public void testCloseIterableVarargsWithNullIterables() throws Exception { + TrackCloseable a = new TrackCloseable(); + TrackCloseable b = new TrackCloseable(); + TrackCloseable c = new TrackCloseable(); + List list1 = Arrays.asList(null, a, b); + List list2 = Collections.singletonList(c); + AutoCloseables.close(list1, null, list2); + assertTrue(a.isClosed()); + assertTrue(b.isClosed()); + assertTrue(c.isClosed()); + } + + @Test + public void testCloseThrowableSuppressesException() { + Exception e = new Exception("from close"); + TrackCloseable c = new TrackCloseable(e); + Exception main = new Exception("main"); + AutoCloseables.close(main, c); + assertTrue(c.isClosed()); + assertEquals(1, main.getSuppressed().length); + assertEquals(e, main.getSuppressed()[0]); + } + + @Test + public void testCloseThrowableWithNullCloseables() { + Exception main = new Exception("main"); + AutoCloseables.close(main, (AutoCloseable) null); + assertEquals(0, main.getSuppressed().length); + + AutoCloseables.close(main, (AutoCloseable[]) null); // no exception + } + + @Test + public void testIterFiltersNulls() { + TrackCloseable a = new TrackCloseable(); + TrackCloseable b = new TrackCloseable(); + Iterable it = AutoCloseables.iter(a, null, b); + List list = new ArrayList<>(); + it.forEach(list::add); + assertEquals(2, list.size()); + assertTrue(list.contains(a)); + assertTrue(list.contains(b)); + } + + @Test + public void testIterEmptyVarargs() { + Iterable it = AutoCloseables.iter(); + List list = new ArrayList<>(); + it.forEach(list::add); + assertTrue(list.isEmpty()); + } + + @Test + public void testIterWithNull() { + AutoCloseables.iter((AutoCloseable) null); // no exception + } + + @Test + public void testCloseNoCheckedWithNull() { + AutoCloseables.closeNoChecked(null); // no exception + } + + @Test + public void testCloseNoCheckedWrapsException() { + Exception e = new Exception("close failed"); + TrackCloseable c = new TrackCloseable(e); + RuntimeException re = + assertThrows(RuntimeException.class, () -> AutoCloseables.closeNoChecked(c)); + assertSame(re.getCause(), e); + assertTrue(re.getMessage().contains("close failed")); + } + + @Test + public void testNoop() throws Exception { + AutoCloseable noop = AutoCloseables.noop(); + assertSame(noop, AutoCloseables.noop()); + noop.close(); // no exception + } + + @Test + public void testAllClosesCollectionOnClose() throws Exception { + TrackCloseable a = new TrackCloseable(); + TrackCloseable b = new TrackCloseable(); + List list = Arrays.asList(a, b); + AutoCloseable all = AutoCloseables.all(list); + assertFalse(a.isClosed()); + assertFalse(b.isClosed()); + all.close(); + assertTrue(a.isClosed()); + assertTrue(b.isClosed()); + } + + @Test + public void testAllWithNullCollection() throws Exception { + AutoCloseable all = AutoCloseables.all(null); + all.close(); // no exception + } + + @Test + public void testRollbackCloseableClosesWhenNotCommitted() throws Exception { + TrackCloseable a = new TrackCloseable(); + TrackCloseable b = new TrackCloseable(); + AutoCloseables.RollbackCloseable rb = AutoCloseables.rollbackable(a, b); + rb.close(); + assertTrue(a.isClosed()); + assertTrue(b.isClosed()); + } + + @Test + public void testRollbackCloseableDoesNotCloseWhenCommitted() throws Exception { + TrackCloseable a = new TrackCloseable(); + TrackCloseable b = new TrackCloseable(); + AutoCloseables.RollbackCloseable rb = AutoCloseables.rollbackable(a, b); + rb.commit(); + rb.close(); + assertFalse(a.isClosed()); + assertFalse(b.isClosed()); + } + + @Test + public void testRollbackCloseableAddAndAddAll() throws Exception { + TrackCloseable a = new TrackCloseable(); + TrackCloseable b = new TrackCloseable(); + TrackCloseable c = new TrackCloseable(); + TrackCloseable d = new TrackCloseable(); + AutoCloseables.RollbackCloseable rb = AutoCloseables.rollbackable(a); + rb.add(b); + rb.addAll(c, d); + rb.addAll((AutoCloseable[]) null); // null varargs shouldn't fail + rb.addAll((List) null); // null Iterable shouldn't fail + rb.close(); + assertTrue(a.isClosed()); + assertTrue(b.isClosed()); + assertTrue(c.isClosed()); + assertTrue(d.isClosed()); + } + + @Test + public void testRollbackCloseableWithNull() throws Exception { + AutoCloseables.rollbackable((AutoCloseable) null); // no exception + } + + @Test + public void testRollbackCloseableWithNulls() throws Exception { + TrackCloseable a = new TrackCloseable(); + AutoCloseables.RollbackCloseable rb = AutoCloseables.rollbackable(a, null); + rb.close(); + assertTrue(a.isClosed()); + } +} From 46211fccf642d6dcfcebc31081817da1529d6a35 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Feb 2026 14:06:40 +0100 Subject: [PATCH 287/408] MINOR: Bump com.gradle:develocity-maven-extension from 2.3.1 to 2.3.3 (#1001) Bumps com.gradle:develocity-maven-extension from 2.3.1 to 2.3.3. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.gradle:develocity-maven-extension&package-manager=maven&previous-version=2.3.1&new-version=2.3.3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .mvn/extensions.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index b136e95f43..0e25cc84f8 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -23,7 +23,7 @@ com.gradle develocity-maven-extension - 2.3.1 + 2.3.4 com.gradle From 9fd4973d12abbd9e9d14098c2acd41d97a6c4407 Mon Sep 17 00:00:00 2001 From: Aleksei Starikov Date: Tue, 17 Feb 2026 14:15:00 +0100 Subject: [PATCH 288/408] GH-470: [Vector] Fix ListViewVector.getElementEndIndex(index) method (#1019) ## What's Changed [First commit](https://github.com/apache/arrow-java/commit/a758cadb17c3d50c08a139a3e2ddced71215ccf5) changes logic: - The PR fixes a bug in the `ListViewVector.getElementEndIndex(index)` method . Before: ``` public int getElementEndIndex(int index) { return sizeBuffer.getInt(index * OFFSET_WIDTH); } ``` After: ``` public int getElementEndIndex(int index) { return offsetBuffer.getInt(index * OFFSET_WIDTH) + sizeBuffer.getInt(index * SIZE_WIDTH); } ``` [Second commit](https://github.com/apache/arrow-java/commit/aec29750202ff1aac73e7fdc9c2020b3dcf72696) doesn't change logic: - Fixes a bug of usage `sizeBuffer` with `OFFSET_WIDTH` (`hashCode` method) and `offsetBuffer` with `SIZE_WIDTH` (`setSize` method). It doesn't introduce real changes in the logic as `OFFSET_WIDTH` == `SIZE_WIDTH` == 4 - Plus small refactoring of ListViewVector to avoid code duplication and similar issues in the future. --- It's a bug fix. --- Closes #470. --- .../arrow/vector/complex/ListViewVector.java | 68 +++--- .../arrow/vector/TestListViewVector.java | 215 ++++++++++++++---- 2 files changed, 204 insertions(+), 79 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java index 8711db5e0f..d41f61e291 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java @@ -226,8 +226,8 @@ private void setReaderAndWriterIndex() { sizeBuffer.writerIndex(0); } else { validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); - offsetBuffer.writerIndex(valueCount * OFFSET_WIDTH); - sizeBuffer.writerIndex(valueCount * SIZE_WIDTH); + offsetBuffer.writerIndex((long) valueCount * OFFSET_WIDTH); + sizeBuffer.writerIndex((long) valueCount * SIZE_WIDTH); } } @@ -445,14 +445,22 @@ public int hashCode(int index, ArrowBufHasher hasher) { return ArrowBufPointer.NULL_HASH_CODE; } int hash = 0; - final int start = offsetBuffer.getInt(index * OFFSET_WIDTH); - final int end = sizeBuffer.getInt(index * OFFSET_WIDTH); + final int start = getElementStartIndex(index); + final int end = getElementEndIndex(index); for (int i = start; i < end; i++) { hash = ByteFunctionHelpers.combineHash(hash, vector.hashCode(i, hasher)); } return hash; } + private void setElementOffsetBuffer(int index, int value) { + offsetBuffer.setInt((long) index * OFFSET_WIDTH, value); + } + + private void setElementSizeBuffer(int index, int value) { + sizeBuffer.setInt((long) index * SIZE_WIDTH, value); + } + private class TransferImpl implements TransferPair { ListViewVector to; @@ -498,7 +506,6 @@ public void splitAndTransfer(int startIndex, int length) { valueCount); to.clear(); if (length > 0) { - final int startPoint = offsetBuffer.getInt((long) startIndex * OFFSET_WIDTH); // we have to scan by index since there are out-of-order offsets to.offsetBuffer = to.allocateBuffers((long) length * OFFSET_WIDTH); to.sizeBuffer = to.allocateBuffers((long) length * SIZE_WIDTH); @@ -507,9 +514,9 @@ public void splitAndTransfer(int startIndex, int length) { int maxOffsetAndSizeSum = -1; int minOffsetValue = -1; for (int i = 0; i < length; i++) { - final int offsetValue = offsetBuffer.getInt((long) (startIndex + i) * OFFSET_WIDTH); - final int sizeValue = sizeBuffer.getInt((long) (startIndex + i) * SIZE_WIDTH); - to.sizeBuffer.setInt((long) i * SIZE_WIDTH, sizeValue); + final int offsetValue = getElementStartIndex(startIndex + i); + final int sizeValue = getElementSize(startIndex + i); + to.setElementSizeBuffer(i, sizeValue); if (maxOffsetAndSizeSum < offsetValue + sizeValue) { maxOffsetAndSizeSum = offsetValue + sizeValue; } @@ -520,9 +527,9 @@ public void splitAndTransfer(int startIndex, int length) { /* splitAndTransfer the offset buffer */ for (int i = 0; i < length; i++) { - final int offsetValue = offsetBuffer.getInt((long) (startIndex + i) * OFFSET_WIDTH); + final int offsetValue = getElementStartIndex(startIndex + i); final int relativeOffset = offsetValue - minOffsetValue; - to.offsetBuffer.setInt((long) i * OFFSET_WIDTH, relativeOffset); + to.setElementOffsetBuffer(i, relativeOffset); } /* splitAndTransfer the validity buffer */ @@ -678,8 +685,8 @@ public List getObject(int index) { if (isSet(index) == 0) { return null; } - final int start = offsetBuffer.getInt(index * OFFSET_WIDTH); - final int end = start + sizeBuffer.getInt((index) * SIZE_WIDTH); + final int start = getElementStartIndex(index); + final int end = getElementEndIndex(index); final ValueVector vv = getDataVector(); final List vals = new JsonStringArrayList<>(end - start); for (int i = start; i < end; i++) { @@ -711,7 +718,7 @@ public boolean isEmpty(int index) { if (isNull(index)) { return true; } else { - return sizeBuffer.getInt(index * SIZE_WIDTH) == 0; + return getElementSize(index) == 0; } } @@ -722,10 +729,7 @@ public boolean isEmpty(int index) { * @return 1 if element at given index is not null, 0 otherwise */ public int isSet(int index) { - final int byteIndex = index >> 3; - final byte b = validityBuffer.getByte(byteIndex); - final int bitIndex = index & 7; - return (b >> bitIndex) & 0x01; + return BitVectorHelper.get(validityBuffer, index); } /** @@ -775,8 +779,8 @@ public void setNull(int index) { reallocValidityAndSizeAndOffsetBuffers(); } - offsetBuffer.setInt(index * OFFSET_WIDTH, 0); - sizeBuffer.setInt(index * SIZE_WIDTH, 0); + setElementOffsetBuffer(index, 0); + setElementSizeBuffer(index, 0); BitVectorHelper.unsetBit(validityBuffer, index); } @@ -794,11 +798,11 @@ public int startNewValue(int index) { if (index > 0) { final int prevOffset = getMaxViewEndChildVectorByIndex(index); - offsetBuffer.setInt(index * OFFSET_WIDTH, prevOffset); + setElementOffsetBuffer(index, prevOffset); } BitVectorHelper.setBit(validityBuffer, index); - return offsetBuffer.getInt(index * OFFSET_WIDTH); + return getElementStartIndex(index); } /** @@ -836,9 +840,9 @@ private void validateInvariants(int offset, int size) { * @param value value to set */ public void setOffset(int index, int value) { - validateInvariants(value, sizeBuffer.getInt(index * SIZE_WIDTH)); + validateInvariants(value, getElementSize(index)); - offsetBuffer.setInt(index * OFFSET_WIDTH, value); + setElementOffsetBuffer(index, value); } /** @@ -848,9 +852,9 @@ public void setOffset(int index, int value) { * @param value value to set */ public void setSize(int index, int value) { - validateInvariants(offsetBuffer.getInt(index * SIZE_WIDTH), value); + validateInvariants(getElementStartIndex(index), value); - sizeBuffer.setInt(index * SIZE_WIDTH, value); + setElementSizeBuffer(index, value); } /** @@ -886,12 +890,16 @@ public void setValueCount(int valueCount) { @Override public int getElementStartIndex(int index) { - return offsetBuffer.getInt(index * OFFSET_WIDTH); + return offsetBuffer.getInt((long) index * OFFSET_WIDTH); + } + + private int getElementSize(int index) { + return sizeBuffer.getInt((long) index * SIZE_WIDTH); } @Override public int getElementEndIndex(int index) { - return sizeBuffer.getInt(index * OFFSET_WIDTH); + return getElementStartIndex(index) + getElementSize(index); } @Override @@ -948,8 +956,8 @@ public double getDensity() { @Override public void validate() { for (int i = 0; i < valueCount; i++) { - final int offset = offsetBuffer.getInt(i * OFFSET_WIDTH); - final int size = sizeBuffer.getInt(i * SIZE_WIDTH); + final int offset = getElementStartIndex(i); + final int size = getElementSize(i); validateInvariants(offset, size); } } @@ -961,6 +969,6 @@ public void validate() { * @param size number of elements in the list that was written */ public void endValue(int index, int size) { - sizeBuffer.setInt(index * SIZE_WIDTH, size); + setElementSizeBuffer(index, size); } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestListViewVector.java b/vector/src/test/java/org/apache/arrow/vector/TestListViewVector.java index 2f282e1988..8ab0edb145 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestListViewVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestListViewVector.java @@ -1550,55 +1550,7 @@ public void testOverwriteWithNull() { public void testOutOfOrderOffset1() { // [[12, -7, 25], null, [0, -127, 127, 50], [], [50, 12]] try (ListViewVector listViewVector = ListViewVector.empty("listview", allocator)) { - // Allocate buffers in listViewVector by calling `allocateNew` method. - listViewVector.allocateNew(); - - // Initialize the child vector using `initializeChildrenFromFields` method. - - FieldType fieldType = new FieldType(true, new ArrowType.Int(16, true), null, null); - Field field = new Field("child-vector", fieldType, null); - listViewVector.initializeChildrenFromFields(Collections.singletonList(field)); - - // Set values in the child vector. - FieldVector fieldVector = listViewVector.getDataVector(); - fieldVector.clear(); - - SmallIntVector childVector = (SmallIntVector) fieldVector; - - childVector.allocateNew(7); - - childVector.set(0, 0); - childVector.set(1, -127); - childVector.set(2, 127); - childVector.set(3, 50); - childVector.set(4, 12); - childVector.set(5, -7); - childVector.set(6, 25); - - childVector.setValueCount(7); - - // Set validity, offset and size buffers using `setValidity`, - // `setOffset` and `setSize` methods. - listViewVector.setValidity(0, 1); - listViewVector.setValidity(1, 0); - listViewVector.setValidity(2, 1); - listViewVector.setValidity(3, 1); - listViewVector.setValidity(4, 1); - - listViewVector.setOffset(0, 4); - listViewVector.setOffset(1, 7); - listViewVector.setOffset(2, 0); - listViewVector.setOffset(3, 0); - listViewVector.setOffset(4, 3); - - listViewVector.setSize(0, 3); - listViewVector.setSize(1, 0); - listViewVector.setSize(2, 4); - listViewVector.setSize(3, 0); - listViewVector.setSize(4, 2); - - // Set value count using `setValueCount` method. - listViewVector.setValueCount(5); + initializeListViewVectorAsInSpecification(listViewVector); final ArrowBuf offSetBuffer = listViewVector.getOffsetBuffer(); final ArrowBuf sizeBuffer = listViewVector.getSizeBuffer(); @@ -2217,6 +2169,105 @@ public void testRangeChildVector2() { } } + @Test + public void testGetElementStartIndexAndEndIndexOrderedOffsetsNoIntersection() { + /* + values = [10, 20, 30, 40, 50] + offsets = [0, 3] + sizes = [3, 2] + vector: [[10, 20, 30], [40, 50]] + */ + try (ListViewVector listViewVector = ListViewVector.empty("sourceVector", allocator)) { + initializeListViewVector( + listViewVector, List.of(10, 20, 30, 40, 50), List.of(1, 1), List.of(0, 3), List.of(3, 2)); + + assertEquals(0, listViewVector.getElementStartIndex(0)); + assertEquals(3, listViewVector.getElementEndIndex(0)); + assertEquals(3, listViewVector.getElementStartIndex(1)); + assertEquals(5, listViewVector.getElementEndIndex(1)); + + final FieldVector dataVec = listViewVector.getDataVector(); + int elemIndex = 0; + int start = listViewVector.getElementStartIndex(elemIndex); + int end = listViewVector.getElementEndIndex(elemIndex); + List list = listViewVector.getObject(elemIndex); + assertEquals(end - start, list.size()); + for (int j = 0; j < list.size(); j++) { + assertEquals(((SmallIntVector) dataVec).get(start + j), list.get(j)); + } + } + } + + @Test + public void testGetElementStartIndexAndEndIndexNotOrderedOffsetsNoIntersection() { + /* + values = [1, 2, 3, 4, 5, 6] + validity = [1, 1, 1] + offsets = [4, 2, 0] + sizes = [2, 2, 2] + vector: [[5, 6], [3, 4], [1, 2]] + */ + try (ListViewVector listViewVector = ListViewVector.empty("sourceVector", allocator)) { + initializeListViewVector( + listViewVector, + List.of(1, 2, 3, 4, 5, 6), + List.of(1, 1, 1), + List.of(4, 2, 0), + List.of(2, 2, 2)); + + assertEquals(4, listViewVector.getElementStartIndex(0)); + assertEquals(6, listViewVector.getElementEndIndex(0)); + assertEquals(2, listViewVector.getElementStartIndex(1)); + assertEquals(4, listViewVector.getElementEndIndex(1)); + assertEquals(0, listViewVector.getElementStartIndex(2)); + assertEquals(2, listViewVector.getElementEndIndex(2)); + } + } + + @Test + public void testGetElementStartIndexAndEndIndexOrderedOffsetsWithIntersection() { + /* + values = [1, 2, 3, 4, 5] + validity = [1, 1, 1] + offsets = [0, 1, 4] + sizes = [2, 3, 1] + vector: [[1, 2], [2, 3, 4], [5]] + */ + try (ListViewVector listViewVector = ListViewVector.empty("sourceVector", allocator)) { + initializeListViewVector( + listViewVector, + List.of(1, 2, 3, 4, 5), + List.of(1, 1, 1), + List.of(0, 1, 4), + List.of(2, 3, 1)); + + assertEquals(0, listViewVector.getElementStartIndex(0)); + assertEquals(2, listViewVector.getElementEndIndex(0)); + assertEquals(1, listViewVector.getElementStartIndex(1)); + assertEquals(4, listViewVector.getElementEndIndex(1)); + assertEquals(4, listViewVector.getElementStartIndex(2)); + assertEquals(5, listViewVector.getElementEndIndex(2)); + } + } + + @Test + public void testGetElementStartIndexAndEndIndexOrderedOffsetsAsInSpecification() { + try (ListViewVector listViewVector = ListViewVector.empty("sourceVector", allocator)) { + initializeListViewVectorAsInSpecification(listViewVector); + + assertEquals(4, listViewVector.getElementStartIndex(0)); + assertEquals(7, listViewVector.getElementEndIndex(0)); + assertEquals(7, listViewVector.getElementStartIndex(1)); + assertEquals(7, listViewVector.getElementEndIndex(1)); + assertEquals(0, listViewVector.getElementStartIndex(2)); + assertEquals(4, listViewVector.getElementEndIndex(2)); + assertEquals(0, listViewVector.getElementStartIndex(3)); + assertEquals(0, listViewVector.getElementEndIndex(3)); + assertEquals(3, listViewVector.getElementStartIndex(4)); + assertEquals(5, listViewVector.getElementEndIndex(4)); + } + } + private void writeIntValues(UnionListViewWriter writer, int[] values) { writer.startListView(); for (int v : values) { @@ -2224,4 +2275,70 @@ private void writeIntValues(UnionListViewWriter writer, int[] values) { } writer.endListView(); } + + /** + * ListViewVector from the specification. + */ + private void initializeListViewVectorAsInSpecification(ListViewVector listViewVector) { + /* + values = [0, -127, 127, 50, 12, -7, 25] + validity = [1, 1, 1, 0, 1] (reversed) + offsets = [4, 7, 0, 0, 3] + sizes = [3, 0, 4, 0, 2] + vector: [[12, -7, 25], null, [0, -127, 127, 50], [], [50, 12]] + */ + initializeListViewVector( + listViewVector, + List.of(0, -127, 127, 50, 12, -7, 25), + List.of(1, 1, 1, 0, 1), + List.of(4, 7, 0, 0, 3), + List.of(3, 0, 4, 0, 2)); + } + + private void initializeListViewVector( + ListViewVector listViewVector, + List values, + List validity, + List offsets, + List sizes) { + // Allocate buffers in listViewVector by calling `allocateNew` method. + assert offsets.size() == sizes.size(); + listViewVector.allocateNew(); + + // Initialize the child vector using `initializeChildrenFromFields` method. + FieldType fieldType = new FieldType(true, new ArrowType.Int(16, true), null, null); + Field field = new Field("child-vector", fieldType, null); + listViewVector.initializeChildrenFromFields(Collections.singletonList(field)); + + // Set values in the child vector. + FieldVector fieldVector = listViewVector.getDataVector(); + fieldVector.clear(); + + SmallIntVector childVector = (SmallIntVector) fieldVector; + childVector.allocateNew(values.size()); + for (int i = 0; i < values.size(); i++) { + childVector.set(i, values.get(i)); + } + childVector.setValueCount(values.size()); + + // Set validity, offset and size buffers using `setValidity`, + // `setOffset` and `setSize` methods. + List reversedValidity = new ArrayList<>(validity); + Collections.reverse(reversedValidity); + for (int i = 0; i < reversedValidity.size(); i++) { + listViewVector.setValidity(i, reversedValidity.get(i)); + } + + for (int i = 0; i < offsets.size(); i++) { + listViewVector.setOffset(i, offsets.get(i)); + } + + for (int i = 0; i < sizes.size(); i++) { + listViewVector.setSize(i, sizes.get(i)); + } + + // Set value count using `setValueCount` method. + listViewVector.setValueCount(offsets.size()); + } } From de97138845abbd12ad253170b5738c4ec3d45473 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Feb 2026 21:29:08 +0100 Subject: [PATCH 289/408] MINOR: Bump logback.version from 1.5.26 to 1.5.27 (#999) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps `logback.version` from 1.5.26 to 1.5.27. Updates `ch.qos.logback:logback-classic` from 1.5.26 to 1.5.27
Release notes

Sourced from ch.qos.logback:logback-classic's releases.

Logback 1.5.27

2026-01-30 Release of logback version 1.5.27

• Updated license to Eclipse Public License version 2.0 from version 1.0, retaining the GPL 2.1 dual-license.

• Fixed missing MDC data transmitted by SocketAppender reported in issues/1010 by Lars Vogel.

• Removed all Receiver classes and components which were already disabled for several years.

• Refactored file scanning code for improved clarity.

• In SizeAndTimeBasedRollingPolicy modified totalSizeCap and maxFileSize comparison to taking into account file compression. This fixes issues/1007.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit 3618eb01aad6672f9cd250dccf7546a69cbe982f associated with the tag v_1.5.27. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits
  • 3618eb0 increase timeout delay to 2000 millis
  • db150c3 prepare release 1.5.27
  • 0370b13 fix missing MDC transmission in SocketAppender. Fixes issues/1010
  • 8100acd remove RemoteAppender*
  • 2b67210 remove Receiver related classes
  • d84b586 remove ReceiverModelHandler - project still builds indicating no active usage
  • 44049ed remove support for receivers in SerializedModelConfigurator and JoranConfigur...
  • 56085d8 fix test
  • e7764f4 refactor file change scanning for clarity
  • e56a12f bump assertj version
  • Additional commits viewable in compare view

Updates `ch.qos.logback:logback-core` from 1.5.26 to 1.5.27
Release notes

Sourced from ch.qos.logback:logback-core's releases.

Logback 1.5.27

2026-01-30 Release of logback version 1.5.27

• Updated license to Eclipse Public License version 2.0 from version 1.0, retaining the GPL 2.1 dual-license.

• Fixed missing MDC data transmitted by SocketAppender reported in issues/1010 by Lars Vogel.

• Removed all Receiver classes and components which were already disabled for several years.

• Refactored file scanning code for improved clarity.

• In SizeAndTimeBasedRollingPolicy modified totalSizeCap and maxFileSize comparison to taking into account file compression. This fixes issues/1007.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit 3618eb01aad6672f9cd250dccf7546a69cbe982f associated with the tag v_1.5.27. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits
  • 3618eb0 increase timeout delay to 2000 millis
  • db150c3 prepare release 1.5.27
  • 0370b13 fix missing MDC transmission in SocketAppender. Fixes issues/1010
  • 8100acd remove RemoteAppender*
  • 2b67210 remove Receiver related classes
  • d84b586 remove ReceiverModelHandler - project still builds indicating no active usage
  • 44049ed remove support for receivers in SerializedModelConfigurator and JoranConfigur...
  • 56085d8 fix test
  • e7764f4 refactor file change scanning for clarity
  • e56a12f bump assertj version
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1f57039015..c118f6aaa4 100644 --- a/pom.xml +++ b/pom.xml @@ -113,7 +113,7 @@ under the License. true 2.42.0 3.53.0 - 1.5.26 + 1.5.32 none -Xdoclint:none From b209fa2ee8ca9840530b404e1448f90d5ae0889d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Feb 2026 21:29:33 +0100 Subject: [PATCH 290/408] MINOR: [CI] Bump docker/login-action from 3.6.0 to 3.7.0 (#996) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [docker/login-action](https://github.com/docker/login-action) from 3.6.0 to 3.7.0.
Release notes

Sourced from docker/login-action's releases.

v3.7.0

Full Changelog: https://github.com/docker/login-action/compare/v3.6.0...v3.7.0

Commits
  • c94ce9f Merge pull request #915 from docker/dependabot/npm_and_yarn/lodash-4.17.23
  • 8339c95 Merge pull request #912 from docker/scope
  • c83e932 build(deps): bump lodash from 4.17.21 to 4.17.23
  • b268aa5 chore: update generated content
  • a603229 documentation for scope input
  • 7567f92 Add scope input to set scopes for the authentication token
  • 0567fa5 Merge pull request #914 from dphi/add-support-for-amazonaws.eu
  • f6ef577 feat: add support for AWS European Sovereign Cloud ECR registries
  • 916386b Merge pull request #911 from crazy-max/ensure-redact
  • 5b3f94a chore: update generated content
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=docker/login-action&package-manager=github_actions&previous-version=3.6.0&new-version=3.7.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 37b2209966..41ea193809 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -127,7 +127,7 @@ jobs: with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing - - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ghcr.io username: ${{ github.actor }} From 5adfb7e32922de6bc7d929a248911904aab80abd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Feb 2026 07:11:51 +0100 Subject: [PATCH 291/408] MINOR: Bump commons-codec:commons-codec from 1.20.0 to 1.21.0 (#997) Bumps [commons-codec:commons-codec](https://github.com/apache/commons-codec) from 1.20.0 to 1.21.0.
Changelog

Sourced from commons-codec:commons-codec's changelog.

Apache Commons Codec 1.21.0 Release Notes

The Apache Commons Codec team is pleased to announce the release of Apache Commons Codec 1.21.0.

The Apache Commons Codec component contains encoders and decoders for formats such as Base16, Base32, Base64, digest, and Hexadecimal. In addition to these widely used encoders and decoders, the codec package also maintains a collection of phonetic encoding utilities.

This is a feature and maintenance release. Java 8 or later is required.

New features

  • CODEC-333: Add distinct Base64 decoding for standard and URL-safe formats. Thanks to Aleksandr Beliakov, Gary Gregory.

Fixed Bugs

  •  Fix oak leaf icon references in overview.html when running
`mvn clean javadoc:javadoc`. Thanks to Gary Gregory.

  •  Fix Apache RAT plugin console warnings. Thanks to Gary
Gregory.

  •  Fix malformed Javadoc comments. Thanks to Gary Gregory.

Changes

  •  Bump org.apache.commons:commons-parent from 91 to 96
[#415](https://github.com/apache/commons-codec/issues/415),
[#418](https://github.com/apache/commons-codec/issues/418). Thanks to
Gary Gregory, Dependabot.

  •  Bump commons-io:commons-io from 2.20.0 to 2.21.0. Thanks to
Gary Gregory.

  •  Bump org.apache.commons:commons-lang3 from 3.19.0 to 3.20.0.
Thanks to Gary Gregory, Dependabot.

For complete information on Apache Commons Codec, including instructions on how to submit bug reports, patches, or suggestions for improvement, see the Apache Commons Codec website:

https://commons.apache.org/proper/commons-codec/

Download page: https://commons.apache.org/proper/commons-codec/download_codec.cgi

Commits
  • 91c4404 Prepare for the release candidate 1.21.0 RC1
  • 21fe1d7 Prepare for the next release candidate
  • d4ea4d0 Bump actions/checkout from 6.0.1 to 6.0.2
  • e30b1f6 Bump actions/setup-java from 5.1.0 to 5.2.0
  • 2e4891c Bump org.apache.commons:commons-parent from 95 to 96
  • d02c003 Use a URL to a prettier page: https://www.ietf.org/rfc/rfc2045
  • 3c961b8 Checkstyle
  • 99cf6b7 Javadoc and exception messages: "base 32" -> "Base32".
  • 2df7b9a Javadoc and exception messages: "base 64" -> "Base64".
  • 0643fdd Javadoc 8 doesn't know how to find this link
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=commons-codec:commons-codec&package-manager=maven&previous-version=1.20.0&new-version=1.21.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- vector/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vector/pom.xml b/vector/pom.xml index 89e9779008..b24f37d5f9 100644 --- a/vector/pom.xml +++ b/vector/pom.xml @@ -60,7 +60,7 @@ under the License. commons-codec commons-codec - 1.20.0 + 1.21.0 org.apache.arrow From 6b6d16a42fdcb46dae728ef0795559e1606c2372 Mon Sep 17 00:00:00 2001 From: Aleksei Starikov Date: Sun, 22 Feb 2026 14:41:02 +0100 Subject: [PATCH 292/408] GH-139: [Flight] Stop return null from MetadataAdapter.getAll(String) and getAllByte(String) (#1016) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What's Changed `CallHeaders` has 3 implementations: - FlightCallHeaders - ErrorFlightMetadata - MetadataAdapter **Before this change:** `MetadataAdapter` could return `null` from `getAll(String)` and `getAllByte(String)` when there were no values for the key, because gRPC’s `Metadata.getAll()` returns `null` in that case. This was undocumented and forced callers to null-check. **After this change:** All 3 implementations consistently return an `empty iterable` (never `null`) when the key is absent or has no values. The contract is documented on the interface and covered by tests for each implementation. --- **This contains breaking changes.** `MetadataAdapter.getAll(String)` and `getAllByte(String)` return empty iterator instead of null. --- Closes #139. --- .../org/apache/arrow/flight/CallHeaders.java | 14 ++++++-- .../arrow/flight/ServerSessionMiddleware.java | 26 +++++++------- .../flight/client/ClientCookieMiddleware.java | 5 +-- .../arrow/flight/grpc/MetadataAdapter.java | 9 +++-- .../apache/arrow/flight/TestCallOptions.java | 11 ++++++ .../arrow/flight/TestErrorMetadata.java | 11 ++++++ .../flight/grpc/TestMetadataAdapter.java | 36 +++++++++++++++++++ 7 files changed, 90 insertions(+), 22 deletions(-) create mode 100644 flight/flight-core/src/test/java/org/apache/arrow/flight/grpc/TestMetadataAdapter.java diff --git a/flight/flight-core/src/main/java/org/apache/arrow/flight/CallHeaders.java b/flight/flight-core/src/main/java/org/apache/arrow/flight/CallHeaders.java index f4f6486a3c..0939d232cf 100644 --- a/flight/flight-core/src/main/java/org/apache/arrow/flight/CallHeaders.java +++ b/flight/flight-core/src/main/java/org/apache/arrow/flight/CallHeaders.java @@ -26,10 +26,20 @@ public interface CallHeaders { /** Get the value of a metadata key. If multiple values are present, then get the last one. */ byte[] getByte(String key); - /** Get all values present for the given metadata key. */ + /** + * Get all values present for the given metadata key. + * + * @param key the metadata key + * @return an iterable of all values for the key. Returns an empty iterable if no value to return. + */ Iterable getAll(String key); - /** Get all values present for the given metadata key. */ + /** + * Get all values present for the given metadata key. + * + * @param key the metadata key + * @return an iterable of all values for the key. Returns an empty iterable if no value to return. + */ Iterable getAllByte(String key); /** diff --git a/flight/flight-core/src/main/java/org/apache/arrow/flight/ServerSessionMiddleware.java b/flight/flight-core/src/main/java/org/apache/arrow/flight/ServerSessionMiddleware.java index 47fd6f1366..5ec01b9c83 100644 --- a/flight/flight-core/src/main/java/org/apache/arrow/flight/ServerSessionMiddleware.java +++ b/flight/flight-core/src/main/java/org/apache/arrow/flight/ServerSessionMiddleware.java @@ -80,20 +80,18 @@ public ServerSessionMiddleware onCallStarted( String sessionId = null; final Iterable it = incomingHeaders.getAll("cookie"); - if (it != null) { - findIdCookie: - for (final String headerValue : it) { - for (final String cookie : headerValue.split(" ;")) { - final String[] cookiePair = cookie.split("="); - if (cookiePair.length != 2) { - // Soft failure: Ignore invalid cookie list field - break; - } - - if (sessionCookieName.equals(cookiePair[0]) && cookiePair[1].length() > 0) { - sessionId = cookiePair[1]; - break findIdCookie; - } + findIdCookie: + for (final String headerValue : it) { + for (final String cookie : headerValue.split(" ;")) { + final String[] cookiePair = cookie.split("="); + if (cookiePair.length != 2) { + // Soft failure: Ignore invalid cookie list field + break; + } + + if (sessionCookieName.equals(cookiePair[0]) && cookiePair[1].length() > 0) { + sessionId = cookiePair[1]; + break findIdCookie; } } } diff --git a/flight/flight-core/src/main/java/org/apache/arrow/flight/client/ClientCookieMiddleware.java b/flight/flight-core/src/main/java/org/apache/arrow/flight/client/ClientCookieMiddleware.java index e5eb934001..b33e6b7ecc 100644 --- a/flight/flight-core/src/main/java/org/apache/arrow/flight/client/ClientCookieMiddleware.java +++ b/flight/flight-core/src/main/java/org/apache/arrow/flight/client/ClientCookieMiddleware.java @@ -100,10 +100,7 @@ public void onBeforeSendingHeaders(CallHeaders outgoingHeaders) { @Override public void onHeadersReceived(CallHeaders incomingHeaders) { - final Iterable setCookieHeaders = incomingHeaders.getAll(SET_COOKIE_HEADER); - if (setCookieHeaders != null) { - factory.updateCookies(setCookieHeaders); - } + factory.updateCookies(incomingHeaders.getAll(SET_COOKIE_HEADER)); } @Override diff --git a/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/MetadataAdapter.java b/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/MetadataAdapter.java index a1de16ede6..64a0769d63 100644 --- a/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/MetadataAdapter.java +++ b/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/MetadataAdapter.java @@ -18,6 +18,7 @@ import io.grpc.Metadata; import java.nio.charset.StandardCharsets; +import java.util.Collections; import java.util.HashSet; import java.util.Set; import java.util.stream.Collectors; @@ -53,13 +54,17 @@ public byte[] getByte(String key) { @Override public Iterable getAll(String key) { - return this.metadata.getAll(Metadata.Key.of(key, Metadata.ASCII_STRING_MARSHALLER)); + final Iterable all = + this.metadata.getAll(Metadata.Key.of(key, Metadata.ASCII_STRING_MARSHALLER)); + return all != null ? all : Collections.emptyList(); } @Override public Iterable getAllByte(String key) { if (key.endsWith(Metadata.BINARY_HEADER_SUFFIX)) { - return this.metadata.getAll(Metadata.Key.of(key, Metadata.BINARY_BYTE_MARSHALLER)); + final Iterable all = + this.metadata.getAll(Metadata.Key.of(key, Metadata.BINARY_BYTE_MARSHALLER)); + return all != null ? all : Collections.emptyList(); } return StreamSupport.stream(getAll(key).spliterator(), false) .map(String::getBytes) diff --git a/flight/flight-core/src/test/java/org/apache/arrow/flight/TestCallOptions.java b/flight/flight-core/src/test/java/org/apache/arrow/flight/TestCallOptions.java index a54ce69812..8aef9c69a1 100644 --- a/flight/flight-core/src/test/java/org/apache/arrow/flight/TestCallOptions.java +++ b/flight/flight-core/src/test/java/org/apache/arrow/flight/TestCallOptions.java @@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -110,6 +111,16 @@ public void mixedProperties() { testHeaders(headers); } + @Test + public void getAllReturnsEmptyIterableForMissingKey() { + FlightCallHeaders headers = new FlightCallHeaders(); + + assertNotNull(headers.getAll("missing")); + assertFalse(headers.getAll("missing").iterator().hasNext()); + assertNotNull(headers.getAllByte("missing-bin")); + assertFalse(headers.getAllByte("missing-bin").iterator().hasNext()); + } + private void testHeaders(CallHeaders headers) { try (BufferAllocator a = new RootAllocator(Long.MAX_VALUE); HeaderProducer producer = new HeaderProducer(); diff --git a/flight/flight-core/src/test/java/org/apache/arrow/flight/TestErrorMetadata.java b/flight/flight-core/src/test/java/org/apache/arrow/flight/TestErrorMetadata.java index a9a3e355bc..214614defd 100644 --- a/flight/flight-core/src/test/java/org/apache/arrow/flight/TestErrorMetadata.java +++ b/flight/flight-core/src/test/java/org/apache/arrow/flight/TestErrorMetadata.java @@ -20,6 +20,7 @@ import static org.apache.arrow.flight.Location.forGrpcInsecure; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -119,6 +120,16 @@ public void testFlightMetadata() throws Exception { } } + @Test + public void getAllReturnsEmptyIterableForMissingKey() { + ErrorFlightMetadata metadata = new ErrorFlightMetadata(); + + assertNotNull(metadata.getAll("missing")); + assertFalse(metadata.getAll("missing").iterator().hasNext()); + assertNotNull(metadata.getAllByte("missing-bin")); + assertFalse(metadata.getAllByte("missing-bin").iterator().hasNext()); + } + private static class StatusRuntimeExceptionProducer extends NoOpFlightProducer { private final PerfOuterClass.Perf perf; diff --git a/flight/flight-core/src/test/java/org/apache/arrow/flight/grpc/TestMetadataAdapter.java b/flight/flight-core/src/test/java/org/apache/arrow/flight/grpc/TestMetadataAdapter.java new file mode 100644 index 0000000000..b0f5dcfcfc --- /dev/null +++ b/flight/flight-core/src/test/java/org/apache/arrow/flight/grpc/TestMetadataAdapter.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.flight.grpc; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import io.grpc.Metadata; +import org.junit.jupiter.api.Test; + +public class TestMetadataAdapter { + + @Test + public void getAllReturnsEmptyIterableForMissingKey() { + MetadataAdapter headers = new MetadataAdapter(new Metadata()); + + assertNotNull(headers.getAll("missing")); + assertFalse(headers.getAll("missing").iterator().hasNext()); + assertNotNull(headers.getAllByte("missing-bin")); + assertFalse(headers.getAllByte("missing-bin").iterator().hasNext()); + } +} From 6e73f7f563b28eec4218d8afabdfaa0ef41e714a Mon Sep 17 00:00:00 2001 From: Ashish Date: Sun, 22 Feb 2026 09:29:47 -0800 Subject: [PATCH 293/408] MINOR: Fix minor issue with README (#1026) ## What's Changed Please fill in a description of the changes here. The PR fixes minor documentation issue, where commands needed to be adjusted to new repo. These were found while setting up the environment. AI was **NOT** used to generate the PR Closes #NNN. --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b0715aadf1..0196536514 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ a version of your choosing. $ flatc --version flatc version 25.1.24 -$ grep "dep.fbs.version" java/pom.xml +$ grep "dep.fbs.version" pom.xml 25.1.24 ``` @@ -60,10 +60,10 @@ $ grep "dep.fbs.version" java/pom.xml cd $ARROW_HOME # remove the existing files -rm -rf java/format/src +rm -rf format/src # regenerate from the .fbs files -flatc --java -o java/format/src/main/java format/*.fbs +flatc --java -o format/src/main/java arrow-format/*.fbs # prepend license header mvn spotless:apply -pl :arrow-format From 0e54b379ff871084139679f1bc501faf05996e18 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Feb 2026 09:45:46 +0100 Subject: [PATCH 294/408] MINOR: Bump commons-cli:commons-cli from 1.9.0 to 1.11.0 (#1028) Bumps [commons-cli:commons-cli](https://github.com/apache/commons-cli) from 1.9.0 to 1.11.0.
Changelog

Sourced from commons-cli:commons-cli's changelog.

Apache Commons CLI 1.11.0 Release Notes

The Apache Commons CLI team is pleased to announce the release of Apache Commons CLI 1.11.0.

Apache Commons CLI provides a simple API for presenting, processing, and validating a Command Line Interface.

This is a feature and maintenance release. Java 8 or later is required.

New Features

  •  Add CommandLine.getOptionCount() to measure option
repetition [#396](https://github.com/apache/commons-cli/issues/396).
Thanks to David Larochette, Gary Gregory.

Fixed Bugs

  • CLI-351: Multiple trailing BREAK_CHAR_SET characters cause infinite loop in HelpFormatter. Thanks to Damien Carbonne, Claude Warren, Gary Gregory.
  • CLI-351: Fix issue with groups not being reported in help output. #411. Thanks to Damien Carbonne, Claude Warren, Gary Gregory.

Updates

  •  Bump org.apache.commons:commons-parent from 85 to 91
[#393](https://github.com/apache/commons-cli/issues/393). Thanks to Gary
Gregory, Dependabot.

  •  Bump commons-io:commons-io from 2.20.0 to 2.21.0. Thanks to
Gary Gregory.

Historical list of changes: https://commons.apache.org/proper/commons-cli/changes.html

For complete information on Apache Commons CLI, including instructions on how to submit bug reports, patches, or suggestions for improvement, see the Apache Commons CLI website:

https://commons.apache.org/proper/commons-cli/

Download page: https://commons.apache.org/proper/commons-cli/download_cli.cgi

Have fun! The Apache Commons Team

Apache Commons CLI 1.11.0 Release Notes

The Apache Commons CLI team is pleased to announce the release of Apache Commons CLI 1.11.0.

Apache Commons CLI provides a simple API for presenting, processing, and validating a Command Line Interface.

This is a feature and maintenance release. Java 8 or later is required.

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=commons-cli:commons-cli&package-manager=maven&previous-version=1.9.0&new-version=1.11.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-integration-tests/pom.xml | 2 +- flight/flight-sql/pom.xml | 2 +- tools/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flight/flight-integration-tests/pom.xml b/flight/flight-integration-tests/pom.xml index 78a2d08ee1..f0f10ada43 100644 --- a/flight/flight-integration-tests/pom.xml +++ b/flight/flight-integration-tests/pom.xml @@ -58,7 +58,7 @@ under the License. commons-cli commons-cli - 1.9.0 + 1.11.0 org.slf4j diff --git a/flight/flight-sql/pom.xml b/flight/flight-sql/pom.xml index 56c47f64dd..a5954819c3 100644 --- a/flight/flight-sql/pom.xml +++ b/flight/flight-sql/pom.xml @@ -119,7 +119,7 @@ under the License. commons-cli commons-cli - 1.9.0 + 1.11.0 true diff --git a/tools/pom.xml b/tools/pom.xml index cb9a161308..d43adb1fdf 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -54,7 +54,7 @@ under the License. commons-cli commons-cli - 1.9.0 + 1.11.0 ch.qos.logback From 2e76de1d2542811843b7dc262cdae0e20102b034 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Feb 2026 09:46:15 +0100 Subject: [PATCH 295/408] MINOR: Bump org.codehaus.mojo:versions-maven-plugin from 2.20.0 to 2.21.0 (#1029) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.codehaus.mojo:versions-maven-plugin](https://github.com/mojohaus/versions) from 2.20.0 to 2.21.0.
Release notes

Sourced from org.codehaus.mojo:versions-maven-plugin's releases.

2.21.0

🚀 New features and improvements

🐛 Bug Fixes

  • #1331: Fix NPE in restrictionForUnchangedSegment if actual version is null (#1332) @​andrzejj0
  • #1310: Corrected UseDepVersionMojo + handling a similar case in SetMojo, SetScmTagMojo, UpdateChildModulesMojo (#1322) @​andrzejj0
  • UseDepVersionMoto should process all projects on the project list (#1320) @​andrzejj0
  • Fixed #1317: Regression coming from ArtifactVersions::filter when currentVersion is null and ignoredVersions is not null (#1319) @​andrzejj0

📝 Documentation updates

📦 Dependency updates

2.20.1

🐛 Bug Fixes

Commits
  • 1cdedea [maven-release-plugin] prepare release 2.21.0
  • b947957 Fix README typos in Contributing section
  • b85c0a8 Bump project version to 2.21.0-SNAPSHOT
  • 7ae3767 Bump byteBuddyVersion from 1.18.3 to 1.18.4 (#1335)
  • 38afa9f Bump org.apache.maven.plugin-testing:maven-plugin-testing-harness
  • 39af6a2 Bump org.codehaus.plexus:plexus-archiver from 4.10.4 to 4.11.0
  • f51b9d5 #1331: Fix NPE in restrictionForUnchangedSegment if actual version is null (#...
  • 8d209b3 Bump org.codehaus.mojo:mojo-parent from 94 to 95 (#1330)
  • 4929d48 Bump byteBuddyVersion from 1.18.2 to 1.18.3 (#1329)
  • cb84d01 Add versions.skip parameter to skip plugin execution (#1328)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.codehaus.mojo:versions-maven-plugin&package-manager=maven&previous-version=2.20.0&new-version=2.21.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- bom/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index b631f9366d..0de43a1217 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -213,7 +213,7 @@ under the License. org.codehaus.mojo versions-maven-plugin - 2.20.0 + 2.21.0 diff --git a/pom.xml b/pom.xml index c118f6aaa4..7bc88675aa 100644 --- a/pom.xml +++ b/pom.xml @@ -512,7 +512,7 @@ under the License. org.codehaus.mojo versions-maven-plugin - 2.20.0 + 2.21.0 pl.project13.maven From 39b0593ac53c1444d0eb05f698fc3c0aa300f30b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Feb 2026 09:47:30 +0100 Subject: [PATCH 296/408] MINOR: Bump com.google.api.grpc:proto-google-common-protos from 2.63.2 to 2.66.0 (#1034) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [com.google.api.grpc:proto-google-common-protos](https://github.com/googleapis/sdk-platform-java) from 2.63.2 to 2.66.0.
Release notes

Sourced from com.google.api.grpc:proto-google-common-protos's releases.

v2.66.0

2.66.0 (2026-01-23)

Features

Dependencies

v2.65.1

2.65.1 (2026-01-13)

Documentation

  • Update docs for GoogleCredentialsProvider#setScopesToApply (#4057) (0a9962f)

v2.65.0

2.65.0 (2026-01-12)

Features

Bug Fixes

  • add api_version breadcrumb to client docs (#4018) (a2b2179)
  • Create a single S2AChannelCredentials per application (#3989) (3758b43)
  • provide API to share the same background executor for channel po… (#4030) (178182c)

Dependencies

Documentation

... (truncated)

Changelog

Sourced from com.google.api.grpc:proto-google-common-protos's changelog.

2.66.0 (2026-01-23)

Features

Dependencies

2.65.1 (2026-01-13)

Documentation

  • Update docs for GoogleCredentialsProvider#setScopesToApply (#4057) (0a9962f)

2.65.0 (2026-01-12)

Features

Bug Fixes

  • add api_version breadcrumb to client docs (#4018) (a2b2179)
  • Create a single S2AChannelCredentials per application (#3989) (3758b43)
  • provide API to share the same background executor for channel po… (#4030) (178182c)

Dependencies

Documentation

2.64.2 (2025-12-10)

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.api.grpc:proto-google-common-protos&package-manager=maven&previous-version=2.63.2&new-version=2.66.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml index b1f755844e..f1d58a0cad 100644 --- a/flight/flight-core/pom.xml +++ b/flight/flight-core/pom.xml @@ -134,7 +134,7 @@ under the License. com.google.api.grpc proto-google-common-protos - 2.63.2 + 2.66.0 test From 394755bc612f9457446396e4374f2871bbc99d9d Mon Sep 17 00:00:00 2001 From: Issac Garcia Date: Thu, 26 Feb 2026 13:20:01 +0100 Subject: [PATCH 297/408] GH-1007: fix: does not break class loading if direct buffer allocator is not available (#1008) ## What's Changed The Direct Buffer is not always needed to use Arrow memory, however, we cannot load MemoryUtil class if we don't set: ``` --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED ``` Which is not always needed/possible. This fix proposes to catch the `InaccessibleObjectException` to not avoiding the load of the class. The directBuffer is, in any case not available and a `UnsupportedOperationException` will be throw as it is in the existing code Closes #1007 . --- .../apache/arrow/memory/util/MemoryUtil.java | 23 ++++++++++++++++--- .../org/apache/arrow/memory/TestOpens.java | 17 +++++--------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java b/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java index 91bd7cd905..be0749a215 100644 --- a/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java +++ b/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java @@ -18,6 +18,7 @@ import java.lang.reflect.Constructor; import java.lang.reflect.Field; +import java.lang.reflect.InaccessibleObjectException; import java.lang.reflect.InvocationTargetException; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -81,9 +82,18 @@ public Object run() { BYTE_ARRAY_BASE_OFFSET = UNSAFE.arrayBaseOffset(byte[].class); // get the offset of the address field in a java.nio.Buffer object + long maybeOffset; Field addressField = java.nio.Buffer.class.getDeclaredField("address"); - addressField.setAccessible(true); - BYTE_BUFFER_ADDRESS_OFFSET = UNSAFE.objectFieldOffset(addressField); + try { + addressField.setAccessible(true); + maybeOffset = UNSAFE.objectFieldOffset(addressField); + } catch (InaccessibleObjectException e) { + maybeOffset = -1; + logger.debug( + "Cannot access the address field of java.nio.Buffer. DirectBuffer operations wont be available", + e); + } + BYTE_BUFFER_ADDRESS_OFFSET = maybeOffset; Constructor directBufferConstructor; long address = -1; @@ -109,6 +119,9 @@ public Object run() { } catch (SecurityException e) { logger.debug("Cannot get constructor for direct buffer allocation", e); return e; + } catch (InaccessibleObjectException e) { + logger.debug("Cannot get constructor for direct buffer allocation", e); + return e; } } }); @@ -156,7 +169,11 @@ public Object run() { * @return address of the underlying memory. */ public static long getByteBufferAddress(ByteBuffer buf) { - return UNSAFE.getLong(buf, BYTE_BUFFER_ADDRESS_OFFSET); + if (BYTE_BUFFER_ADDRESS_OFFSET != -1) { + return UNSAFE.getLong(buf, BYTE_BUFFER_ADDRESS_OFFSET); + } + throw new UnsupportedOperationException( + "Byte buffer address cannot be obtained because sun.misc.Unsafe or java.nio.DirectByteBuffer.(long, int) is not available"); } private MemoryUtil() {} diff --git a/memory/memory-core/src/test/java/org/apache/arrow/memory/TestOpens.java b/memory/memory-core/src/test/java/org/apache/arrow/memory/TestOpens.java index b5e0a71e7e..f74bf63f82 100644 --- a/memory/memory-core/src/test/java/org/apache/arrow/memory/TestOpens.java +++ b/memory/memory-core/src/test/java/org/apache/arrow/memory/TestOpens.java @@ -20,32 +20,27 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.condition.JRE.JAVA_16; +import org.apache.arrow.memory.util.MemoryUtil; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledForJreRange; public class TestOpens { - /** Instantiating the RootAllocator should poke MemoryUtil and fail. */ + /** Accessing MemoryUtil.directBuffer should fail as add-opens is not configured. */ @Test @EnabledForJreRange(min = JAVA_16) public void testMemoryUtilFailsLoudly() { // This test is configured by Maven to run WITHOUT add-opens. So this should fail on JDK16+ // (where JEP396 means that add-opens is required to access JDK internals). // The test will likely fail in your IDE if it doesn't correctly pick this up. - Throwable e = - assertThrows( - Throwable.class, - () -> { - BufferAllocator allocator = new RootAllocator(); - allocator.close(); - }); + Throwable e = assertThrows(Throwable.class, () -> MemoryUtil.directBuffer(0, 10)); boolean found = false; while (e != null) { - e = e.getCause(); - if (e instanceof RuntimeException - && e.getMessage().contains("Failed to initialize MemoryUtil")) { + if (e instanceof UnsupportedOperationException + && e.getMessage().contains("java.nio.DirectByteBuffer.(long, int) not available")) { found = true; break; } + e = e.getCause(); } assertTrue(found, "Expected exception was not thrown"); } From 1a99518180b6a89fe17a9af5ce12f6956b5c6c27 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Mar 2026 13:15:36 +0100 Subject: [PATCH 298/408] MINOR: [CI] Bump actions/upload-artifact from 6.0.0 to 7.0.0 (#1045) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 6.0.0 to 7.0.0.
Release notes

Sourced from actions/upload-artifact's releases.

v7.0.0

v7 What's new

Direct Uploads

Adds support for uploading single files directly (unzipped). Callers can set the new archive parameter to false to skip zipping the file during upload. Right now, we only support single files. The action will fail if the glob passed resolves to multiple files. The name parameter is also ignored with this setting. Instead, the name of the artifact will be the name of the uploaded file.

ESM

To support new versions of the @actions/* packages, we've upgraded the package to ESM.

What's Changed

New Contributors

Full Changelog: https://github.com/actions/upload-artifact/compare/v6...v7.0.0

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/upload-artifact&package-manager=github_actions&previous-version=6.0.0&new-version=7.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 41ea193809..4ec89c2204 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -71,7 +71,7 @@ jobs: run: | dev/release/run_rat.sh "${TAR_GZ}" - name: Upload source archive - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: release-source path: | @@ -148,7 +148,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf jni-linux-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: jni-linux-${{ matrix.platform.arch }} path: jni-linux-${{ matrix.platform.arch }}.tar.gz @@ -278,7 +278,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf jni-macos-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: jni-macos-${{ matrix.platform.arch }} path: jni-macos-${{ matrix.platform.arch }}.tar.gz @@ -356,7 +356,7 @@ jobs: shell: bash run: tar -cvzf jni-windows-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: jni-windows-${{ matrix.platform.arch }} path: jni-windows-${{ matrix.platform.arch }}.tar.gz @@ -428,12 +428,12 @@ jobs: cp -a target/site/apidocs reference tar -cvzf reference.tar.gz reference - name: Upload binaries - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: release-binaries path: binaries/* - name: Upload docs - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: reference path: reference.tar.gz @@ -471,7 +471,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf html.tar.gz -C docs/build html - name: Upload artifacts - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: release-html path: html.tar.gz From 9c497848f9e7a531aae1095009257217d993c24b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Mar 2026 13:19:43 +0100 Subject: [PATCH 299/408] MINOR: Bump checker.framework.version from 3.53.0 to 3.53.1 (#1046) Bumps `checker.framework.version` from 3.53.0 to 3.53.1. Updates `org.checkerframework:checker-qual` from 3.53.0 to 3.53.1
Release notes

Sourced from org.checkerframework:checker-qual's releases.

Checker Framework 3.53.1

Version 3.53.1 (2026-02-02)

Closed issues

#4858, #6141, #6620, #7360, #7388.

Changelog

Sourced from org.checkerframework:checker-qual's changelog.

Version 3.53.1 (2026-02-02)

Closed issues

#4858, #6141, #6620, #7360, #7388.

Commits

Updates `org.checkerframework:checker` from 3.53.0 to 3.53.1
Release notes

Sourced from org.checkerframework:checker's releases.

Checker Framework 3.53.1

Version 3.53.1 (2026-02-02)

Closed issues

#4858, #6141, #6620, #7360, #7388.

Changelog

Sourced from org.checkerframework:checker's changelog.

Version 3.53.1 (2026-02-02)

Closed issues

#4858, #6141, #6620, #7360, #7388.

Commits

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7bc88675aa..2917d6cb9b 100644 --- a/pom.xml +++ b/pom.xml @@ -112,7 +112,7 @@ under the License. 10.23.0 true 2.42.0 - 3.53.0 + 3.53.1 1.5.32 none -Xdoclint:none From 41acbdc681c4792b01655066bf7230f4a07c2ef2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Mar 2026 13:26:12 +0100 Subject: [PATCH 300/408] MINOR: [CI] Bump actions/download-artifact from 7.0.0 to 8.0.0 (#1047) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 7.0.0 to 8.0.0.
Release notes

Sourced from actions/download-artifact's releases.

v8.0.0

v8 - What's new

Direct downloads

To support direct uploads in actions/upload-artifact, the action will no longer attempt to unzip all downloaded files. Instead, the action checks the Content-Type header ahead of unzipping and skips non-zipped files. Callers wishing to download a zipped file as-is can also set the new skip-decompress parameter to false.

Enforced checks (breaking)

A previous release introduced digest checks on the download. If a download hash didn't match the expected hash from the server, the action would log a warning. Callers can now configure the behavior on mismatch with the digest-mismatch parameter. To be secure by default, we are now defaulting the behavior to error which will fail the workflow run.

ESM

To support new versions of the @actions/* packages, we've upgraded the package to ESM.

What's Changed

Full Changelog: https://github.com/actions/download-artifact/compare/v7...v8.0.0

Commits
  • 70fc10c Merge pull request #461 from actions/danwkennedy/digest-mismatch-behavior
  • f258da9 Add change docs
  • ccc058e Fix linting issues
  • bd7976b Add a setting to specify what to do on hash mismatch and default it to error
  • ac21fcf Merge pull request #460 from actions/danwkennedy/download-no-unzip
  • 15999bf Add note about package bumps
  • 974686e Bump the version to v8 and add release notes
  • fbe48b1 Update test names to make it clearer what they do
  • 96bf374 One more test fix
  • b8c4819 Fix skip decompress test
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/download-artifact&package-manager=github_actions&previous-version=7.0.0&new-version=8.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 4ec89c2204..8039f4c598 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -101,7 +101,7 @@ jobs: packages: write steps: - name: Download source archive - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: release-source - name: Extract source archive @@ -168,7 +168,7 @@ jobs: MACOSX_DEPLOYMENT_TARGET: "14.0" steps: - name: Download source archive - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: release-source - name: Extract source archive @@ -296,7 +296,7 @@ jobs: arch: "x86_64" steps: - name: Download source archive - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: release-source - name: Extract source archive @@ -369,7 +369,7 @@ jobs: - jni-windows steps: - name: Download artifacts - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: path: artifacts - name: Decompress artifacts @@ -450,11 +450,11 @@ jobs: with: cache: 'pip' - name: Download source archive - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: release-source - name: Download Javadocs - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: reference - name: Extract source archive @@ -519,7 +519,7 @@ jobs: cp ../.asf.yaml ./ git add .nojekyll .asf.yaml - name: Download - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: release-html - name: Extract @@ -555,7 +555,7 @@ jobs: - ubuntu-latest steps: - name: Download release artifacts - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: pattern: release-* - name: Verify @@ -589,7 +589,7 @@ jobs: contents: write steps: - name: Download release artifacts - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: pattern: release-* path: artifacts From 5dfd2595080a15d3d6ff3e40d8de57af4bdd7858 Mon Sep 17 00:00:00 2001 From: Logan Riggs Date: Wed, 4 Mar 2026 12:03:18 -0800 Subject: [PATCH 301/408] GH-1038: Trim object memory for ArrowBuf (#1044) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What's Changed A significant number of ArrowBuf and BufferLedger objects are created during certain workloads. Saving several bytes per instance could add up to significant memory savings and reduced memory allocation expense and garbage collection. The id field, which was a sequential value used when logging object information, is replaced with an identity hash code. This should still allow enough information for debugging without the memory overhead. There may be possible duplicate values but it shouldn't matter for logging purposes. Atomic fields can be replaced by a primitive and a static updater which saves several bytes per instance. ### ArrowBuf | Component | Before | After | Savings | |-----------|--------|-------|---------| | `idGenerator` (static) | `AtomicLong` | Removed | 24 bytes globally | | `id` field (per instance) | `long` (8 bytes) | Removed | **8 bytes per instance** | | `getId()` | Returns `id` field | Returns `System.identityHashCode(this)` | — | ### BufferLedger | Component | Before | After | Savings | |-----------|--------|-------|---------| | `LEDGER_ID_GENERATOR` (static) | `AtomicLong` | Removed | 24 bytes globally | | `ledgerId` (per instance) | `long` (8 bytes) | Removed | **8 bytes per instance** | | `bufRefCnt` | `AtomicInteger` (24 bytes) | `volatile int` + static updater | **20 bytes per instance** | ### Total Savings | Scale | ArrowBuf | BufferLedger | Combined | |-------|----------|--------------|----------| | 100K | 800 KB | 2.8 MB | **~3.6 MB** | | 1M | 8 MB | 28 MB | **~36 MB** | | 10M | 80 MB | 280 MB | **~360 MB** | ### Benchmarking I ran the added benchmark before and after the metadata trimming. **Metadata Trimmed** | Benchmark | Mode | Score | Error |Units| |-------|----------|--------------|----------|----------| |MemoryFootprintBenchmarks.measureAllocationPerformance | avgt | 456.831 |± 36.059 | us/op| |MemoryFootprintBenchmarks.measureArrowBufMemoryFootprint | ss | 161.085 |± 35.596| ms/op| |Created 100000 ArrowBuf instances. Heap memory used | sum | 35631520 bytes (33.98 MB) |0 |bytes| |Average memory per ArrowBuf| sum | 356.32 bytes |0 |bytes| **Previous Object Layout** | Benchmark | Mode | Score | Error |Units| |-------|----------|--------------|----------|----------| |MemoryFootprintBenchmarks.measureAllocationPerformance | avgt | 466.171 |± 16.233 | us/op| |MemoryFootprintBenchmarks.measureArrowBufMemoryFootprint | ss | 176.790 |± 17.943 |ms/op| |Created 100000 ArrowBuf instances. Heap memory used | sum | 38817480 bytes (37.02 MB) |0 |bytes| |Average memory per ArrowBuf| sum | 388.17 bytes |0 |bytes| Closes #1038. --- .../org/apache/arrow/memory/Accountant.java | 42 ++-- .../org/apache/arrow/memory/ArrowBuf.java | 19 +- .../org/apache/arrow/memory/BufferLedger.java | 28 +-- .../memory/MemoryFootprintBenchmarks.java | 213 ++++++++++++++++++ 4 files changed, 263 insertions(+), 39 deletions(-) create mode 100644 performance/src/main/java/org/apache/arrow/memory/MemoryFootprintBenchmarks.java diff --git a/memory/memory-core/src/main/java/org/apache/arrow/memory/Accountant.java b/memory/memory-core/src/main/java/org/apache/arrow/memory/Accountant.java index 5d052c2cde..d4d76f57f4 100644 --- a/memory/memory-core/src/main/java/org/apache/arrow/memory/Accountant.java +++ b/memory/memory-core/src/main/java/org/apache/arrow/memory/Accountant.java @@ -16,7 +16,7 @@ */ package org.apache.arrow.memory; -import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicLongFieldUpdater; import org.apache.arrow.util.Preconditions; import org.checkerframework.checker.nullness.qual.Nullable; @@ -37,16 +37,24 @@ class Accountant implements AutoCloseable { */ protected final long reservation; - private final AtomicLong peakAllocation = new AtomicLong(); + // AtomicLongFieldUpdaters for memory accounting fields to reduce memory overhead + private static final AtomicLongFieldUpdater PEAK_ALLOCATION_UPDATER = + AtomicLongFieldUpdater.newUpdater(Accountant.class, "peakAllocation"); + private static final AtomicLongFieldUpdater ALLOCATION_LIMIT_UPDATER = + AtomicLongFieldUpdater.newUpdater(Accountant.class, "allocationLimit"); + private static final AtomicLongFieldUpdater LOCALLY_HELD_MEMORY_UPDATER = + AtomicLongFieldUpdater.newUpdater(Accountant.class, "locallyHeldMemory"); + + private volatile long peakAllocation = 0; /** * Maximum local memory that can be held. This can be externally updated. Changing it won't cause * past memory to change but will change responses to future allocation efforts */ - private final AtomicLong allocationLimit = new AtomicLong(); + private volatile long allocationLimit = 0; /** Currently allocated amount of memory. */ - private final AtomicLong locallyHeldMemory = new AtomicLong(); + private volatile long locallyHeldMemory = 0; public Accountant( @Nullable Accountant parent, String name, long reservation, long maxAllocation) { @@ -64,7 +72,7 @@ public Accountant( this.parent = parent; this.name = name; this.reservation = reservation; - this.allocationLimit.set(maxAllocation); + ALLOCATION_LIMIT_UPDATER.set(this, maxAllocation); if (reservation != 0) { Preconditions.checkArgument(parent != null, "parent must not be null"); @@ -117,12 +125,12 @@ private AllocationOutcome.Status allocateBytesInternal(long size) { } private void updatePeak() { - final long currentMemory = locallyHeldMemory.get(); + final long currentMemory = locallyHeldMemory; while (true) { - final long previousPeak = peakAllocation.get(); + final long previousPeak = peakAllocation; if (currentMemory > previousPeak) { - if (!peakAllocation.compareAndSet(previousPeak, currentMemory)) { + if (!PEAK_ALLOCATION_UPDATER.compareAndSet(this, previousPeak, currentMemory)) { // peak allocation changed underneath us. try again. continue; } @@ -166,7 +174,7 @@ private AllocationOutcome.Status allocate( final boolean incomingUpdatePeak, final boolean forceAllocation, @Nullable AllocationOutcomeDetails details) { - final long oldLocal = locallyHeldMemory.getAndAdd(size); + final long oldLocal = LOCALLY_HELD_MEMORY_UPDATER.getAndAdd(this, size); final long newLocal = oldLocal + size; // Borrowed from Math.addExact (but avoid exception here) // Overflow if result has opposite sign of both arguments @@ -174,7 +182,7 @@ private AllocationOutcome.Status allocate( // failure final boolean overflow = ((oldLocal ^ newLocal) & (size ^ newLocal)) < 0; final long beyondReservation = newLocal - reservation; - final boolean beyondLimit = overflow || newLocal > allocationLimit.get(); + final boolean beyondLimit = overflow || newLocal > allocationLimit; final boolean updatePeak = forceAllocation || (incomingUpdatePeak && !beyondLimit); if (details != null) { @@ -214,7 +222,7 @@ private AllocationOutcome.Status allocate( public void releaseBytes(long size) { // reduce local memory. all memory released above reservation should be released up the tree. - final long newSize = locallyHeldMemory.addAndGet(-size); + final long newSize = LOCALLY_HELD_MEMORY_UPDATER.addAndGet(this, -size); Preconditions.checkArgument(newSize >= 0, "Accounted size went negative."); @@ -255,7 +263,7 @@ public String getName() { * @return Limit in bytes. */ public long getLimit() { - return allocationLimit.get(); + return allocationLimit; } /** @@ -274,7 +282,7 @@ public long getInitReservation() { * @param newLimit The limit in bytes. */ public void setLimit(long newLimit) { - allocationLimit.set(newLimit); + ALLOCATION_LIMIT_UPDATER.set(this, newLimit); } /** @@ -284,7 +292,7 @@ public void setLimit(long newLimit) { * @return Currently allocate memory in bytes. */ public long getAllocatedMemory() { - return locallyHeldMemory.get(); + return locallyHeldMemory; } /** @@ -293,17 +301,17 @@ public long getAllocatedMemory() { * @return The peak allocated memory in bytes. */ public long getPeakMemoryAllocation() { - return peakAllocation.get(); + return peakAllocation; } public long getHeadroom() { - long localHeadroom = allocationLimit.get() - locallyHeldMemory.get(); + long localHeadroom = allocationLimit - locallyHeldMemory; if (parent == null) { return localHeadroom; } // Amount of reserved memory left on top of what parent has - long reservedHeadroom = Math.max(0, reservation - locallyHeldMemory.get()); + long reservedHeadroom = Math.max(0, reservation - locallyHeldMemory); return Math.min(localHeadroom, parent.getHeadroom() + reservedHeadroom); } } diff --git a/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java b/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java index b8012fe643..9712be34d7 100644 --- a/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java +++ b/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java @@ -24,7 +24,6 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.ReadOnlyBufferException; -import java.util.concurrent.atomic.AtomicLong; import org.apache.arrow.memory.BaseAllocator.Verbosity; import org.apache.arrow.memory.util.CommonUtil; import org.apache.arrow.memory.util.HistoricalLog; @@ -57,9 +56,8 @@ public final class ArrowBuf implements AutoCloseable { private static final int DOUBLE_SIZE = Double.BYTES; private static final int LONG_SIZE = Long.BYTES; - private static final AtomicLong idGenerator = new AtomicLong(0); private static final int LOG_BYTES_PER_ROW = 10; - private final long id = idGenerator.incrementAndGet(); + private final ReferenceManager referenceManager; private final @Nullable BufferManager bufferManager; private final long addr; @@ -67,7 +65,8 @@ public final class ArrowBuf implements AutoCloseable { private long writerIndex; private final @Nullable HistoricalLog historicalLog = BaseAllocator.DEBUG - ? new HistoricalLog(BaseAllocator.DEBUG_LOG_LENGTH, "ArrowBuf[%d]", id) + ? new HistoricalLog( + BaseAllocator.DEBUG_LOG_LENGTH, "ArrowBuf[%d]", System.identityHashCode(this)) : null; private volatile long capacity; @@ -218,7 +217,8 @@ public long memoryAddress() { @Override public String toString() { - return String.format("ArrowBuf[%d], address:%d, capacity:%d", id, memoryAddress(), capacity); + return String.format( + "ArrowBuf[%d], address:%d, capacity:%d", getId(), memoryAddress(), capacity); } @Override @@ -1080,12 +1080,15 @@ public String toHexString(final long start, final int length) { } /** - * Get the integer id assigned to this ArrowBuf for debugging purposes. + * Get the id assigned to this ArrowBuf for debugging purposes. + * + *

Returns {@link System#identityHashCode(Object)} which provides a unique identifier for this + * buffer without any per-instance memory overhead. * - * @return integer id + * @return the identity hash code for this buffer */ public long getId() { - return id; + return System.identityHashCode(this); } /** diff --git a/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferLedger.java b/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferLedger.java index b562a421e7..eb90efcbb5 100644 --- a/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferLedger.java +++ b/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferLedger.java @@ -17,8 +17,7 @@ package org.apache.arrow.memory; import java.util.IdentityHashMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import org.apache.arrow.memory.util.CommonUtil; import org.apache.arrow.memory.util.HistoricalLog; import org.apache.arrow.util.Preconditions; @@ -32,12 +31,13 @@ public class BufferLedger implements ValueWithKeyIncluded, ReferenceManager { private final @Nullable IdentityHashMap buffers = BaseAllocator.DEBUG ? new IdentityHashMap<>() : null; - private static final AtomicLong LEDGER_ID_GENERATOR = new AtomicLong(0); - // unique ID assigned to each ledger - private final long ledgerId = LEDGER_ID_GENERATOR.incrementAndGet(); - private final AtomicInteger bufRefCnt = new AtomicInteger(0); // start at zero so we can - // manage request for retain - // correctly + + // AtomicIntegerFieldUpdater for bufRefCnt to reduce memory overhead + private static final AtomicIntegerFieldUpdater BUF_REF_CNT_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(BufferLedger.class, "bufRefCnt"); + // start at zero so we can manage request for retain correctly + private volatile int bufRefCnt = 0; + private final long lCreationTime = System.nanoTime(); private final BufferAllocator allocator; private final AllocationManager allocationManager; @@ -78,7 +78,7 @@ public BufferAllocator getAllocator() { */ @Override public int getRefCount() { - return bufRefCnt.get(); + return bufRefCnt; } /** @@ -86,7 +86,7 @@ public int getRefCount() { * ArrowBufs managed by this ledger will share the ref count. */ void increment() { - bufRefCnt.incrementAndGet(); + BUF_REF_CNT_UPDATER.incrementAndGet(this); } /** @@ -144,7 +144,7 @@ private int decrement(int decrement) { allocator.assertOpen(); final int outcome; synchronized (allocationManager) { - outcome = bufRefCnt.addAndGet(-decrement); + outcome = BUF_REF_CNT_UPDATER.addAndGet(this, -decrement); if (outcome == 0) { lDestructionTime = System.nanoTime(); // refcount of this reference manager has dropped to 0 @@ -174,7 +174,7 @@ public void retain(int increment) { if (historicalLog != null) { historicalLog.recordEvent("retain(%d)", increment); } - final int originalReferenceCount = bufRefCnt.getAndAdd(increment); + final int originalReferenceCount = BUF_REF_CNT_UPDATER.getAndAdd(this, increment); Preconditions.checkArgument(originalReferenceCount > 0); } @@ -472,13 +472,13 @@ public long getAccountedSize() { void print(StringBuilder sb, int indent, BaseAllocator.Verbosity verbosity) { CommonUtil.indent(sb, indent) .append("ledger[") - .append(ledgerId) + .append(System.identityHashCode(this)) .append("] allocator: ") .append(allocator.getName()) .append("), isOwning: ") .append(", size: ") .append(", references: ") - .append(bufRefCnt.get()) + .append(bufRefCnt) .append(", life: ") .append(lCreationTime) .append("..") diff --git a/performance/src/main/java/org/apache/arrow/memory/MemoryFootprintBenchmarks.java b/performance/src/main/java/org/apache/arrow/memory/MemoryFootprintBenchmarks.java new file mode 100644 index 0000000000..395ba13b9d --- /dev/null +++ b/performance/src/main/java/org/apache/arrow/memory/MemoryFootprintBenchmarks.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.memory; + +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryUsage; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +/** + * Benchmarks for memory footprint of Arrow memory objects. + * + *

This benchmark measures the heap memory overhead of creating many ArrowBuf instances. The + * optimizations using AtomicFieldUpdater instead of AtomicLong/AtomicInteger objects should reduce + * memory overhead significantly. + * + *

Expected savings per instance: - ArrowBuf: 8 bytes (id field removed) - BufferLedger: 28 bytes + * (20 from AtomicInteger + 8 from ledgerId) - Accountant: 48 bytes (3 × 16 bytes from AtomicLong + * objects) + * + *

For 1M ArrowBuf instances, this should save approximately 8 MB of heap memory. + */ +@State(Scope.Benchmark) +@Fork( + value = 1, + jvmArgs = {"-Xms2g", "-Xmx2g"}) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +public class MemoryFootprintBenchmarks { + + /** Number of ArrowBuf instances to create for memory footprint measurement. */ + private static final int NUM_BUFFERS = 100_000; + + /** Size in bytes of each buffer allocation. */ + private static final int BUFFER_SIZE = 1024; + + /** Root allocator used for all buffer allocations in the benchmark. */ + private RootAllocator allocator; + + /** Array to hold references to allocated buffers, preventing garbage collection. */ + private ArrowBuf[] buffers; + + /** JMX bean for querying heap memory usage statistics. */ + private MemoryMXBean memoryBean; + + /** + * Sets up the benchmark state before each trial. + * + *

Initializes the memory monitoring bean, creates a root allocator with sufficient capacity, + * and allocates the buffer reference array. + */ + @Setup(Level.Trial) + public void setup() { + memoryBean = ManagementFactory.getMemoryMXBean(); + allocator = new RootAllocator((long) NUM_BUFFERS * BUFFER_SIZE); + buffers = new ArrowBuf[NUM_BUFFERS]; + } + + /** + * Cleans up buffers after each benchmark invocation. + * + *

Closes all allocated buffers to prevent memory leaks and ensure each iteration starts with a + * clean slate. This is critical for the memory footprint benchmark which allocates many buffers + * that would otherwise accumulate across warmup and measurement iterations. + */ + @TearDown(Level.Invocation) + public void tearDown() { + for (int i = 0; i < NUM_BUFFERS; i++) { + if (buffers[i] != null) { + buffers[i].close(); + buffers[i] = null; + } + } + } + + /** + * Cleans up the allocator after the trial completes. + * + *

Closes the root allocator to release all resources after all warmup and measurement + * iterations are complete. + */ + @TearDown(Level.Trial) + public void tearDownTrial() { + allocator.close(); + } + + /** + * Benchmark that measures heap memory usage when creating many ArrowBuf instances. + * + *

This benchmark creates {@value #NUM_BUFFERS} ArrowBuf instances and measures the heap memory + * used. With the AtomicFieldUpdater optimizations, we expect to save approximately 800 KB of heap + * memory (8 bytes × 100,000 instances) just from removing the id field in ArrowBuf. + * + *

The benchmark performs garbage collection before and after allocation to ensure accurate + * measurement of heap memory delta. Results are printed to stdout for analysis. + * + * @return the total heap memory used by the allocated buffers in bytes + */ + @Benchmark + @BenchmarkMode(Mode.SingleShotTime) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public long measureArrowBufMemoryFootprint() { + // Force GC before measurement + System.gc(); + System.gc(); + System.gc(); + + MemoryUsage heapBefore = memoryBean.getHeapMemoryUsage(); + long usedBefore = heapBefore.getUsed(); + + // Allocate buffers + for (int i = 0; i < NUM_BUFFERS; i++) { + buffers[i] = allocator.buffer(BUFFER_SIZE); + } + + // Force GC to get accurate measurement + System.gc(); + System.gc(); + System.gc(); + + MemoryUsage heapAfter = memoryBean.getHeapMemoryUsage(); + long usedAfter = heapAfter.getUsed(); + + long memoryUsed = usedAfter - usedBefore; + + // Print memory usage for analysis + System.out.printf( + "Created %d ArrowBuf instances. Heap memory used: %d bytes (%.2f MB)%n", + NUM_BUFFERS, memoryUsed, memoryUsed / (1024.0 * 1024.0)); + System.out.printf( + "Average memory per ArrowBuf: %.2f bytes%n", (double) memoryUsed / NUM_BUFFERS); + + return memoryUsed; + } + + /** + * Benchmark that measures allocation and deallocation performance. + * + *

This complements the memory footprint benchmark by measuring the time it takes to allocate + * and deallocate 1,000 buffers in a tight loop. This helps identify any performance regressions + * introduced by memory optimizations. + * + *

Uses a local buffer array to avoid interference with the shared {@link #buffers} array used + * by other benchmarks. + */ + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + public void measureAllocationPerformance() { + ArrowBuf[] localBuffers = new ArrowBuf[1000]; + + for (int i = 0; i < 1000; i++) { + localBuffers[i] = allocator.buffer(BUFFER_SIZE); + } + + for (int i = 0; i < 1000; i++) { + localBuffers[i].close(); + } + } + + /** + * Main entry point for running the benchmarks standalone. + * + *

This allows running the benchmarks directly from the command line or IDE without using the + * Maven JMH plugin. Example usage: + * + * 

{@code
+   * java -cp target/benchmarks.jar org.apache.arrow.memory.MemoryFootprintBenchmarks
+   * }
+ * + * @param args command line arguments (not used) + * @throws RunnerException if the benchmark runner encounters an error + */ + public static void main(String[] args) throws RunnerException { + Options opt = + new OptionsBuilder() + .include(MemoryFootprintBenchmarks.class.getSimpleName()) + .forks(1) + .build(); + + new Runner(opt).run(); + } +} From 7cbf15994ae7c82fbcb178dded6b3f128219d9ad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Mar 2026 21:18:49 +0100 Subject: [PATCH 302/408] MINOR: Bump org.codehaus.mojo:build-helper-maven-plugin from 3.6.0 to 3.6.1 (#1049) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.codehaus.mojo:build-helper-maven-plugin](https://github.com/mojohaus/build-helper-maven-plugin) from 3.6.0 to 3.6.1.
Release notes

Sourced from org.codehaus.mojo:build-helper-maven-plugin's releases.

3.6.1

📝 Documentation updates

👻 Maintenance

📦 Dependency updates

Commits
  • 908df59 [maven-release-plugin] prepare release 3.6.1
  • faafd8f Use common release-drafter configuration
  • a91b402 Rename Goals to Plugin Documentation in the site menu
  • 1e9136d Bump org.codehaus.mojo:mojo-parent from 87 to 91
  • 8700ddc Bump org.apache.maven.shared:file-management from 3.1.0 to 3.2.0
  • ab2c635 Bump org.codehaus.mojo:mojo-parent from 86 to 87
  • 611ce40 Typos.
  • 02d2b8e Bump org.codehaus.mojo:mojo-parent from 85 to 86
  • d742e5c Update site.xml to Doxia 2
  • 80b89b8 Bump org.codehaus.plexus:plexus-utils from 4.0.1 to 4.0.2
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.codehaus.mojo:build-helper-maven-plugin&package-manager=maven&previous-version=3.6.0&new-version=3.6.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2917d6cb9b..6b7003f31a 100644 --- a/pom.xml +++ b/pom.xml @@ -497,7 +497,7 @@ under the License. org.codehaus.mojo build-helper-maven-plugin - 3.6.0 + 3.6.1 org.codehaus.mojo From 15f50796b5603100702560fad4b4c843f4fa379c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Mon, 9 Mar 2026 15:12:41 +0100 Subject: [PATCH 303/408] MINOR: Fix flaky TestBasicAuth memory leak by waiting for async buffer release (#1058) ## What's Changed gRPC/Netty releases Arrow buffers asynchronously after server shutdown. Poll briefly for the allocator's memory to drain before closing it, preventing spurious "Memory was leaked" errors in CI. The fix adds a brief polling loop to wait for the allocator's memory to drain before closing it. --- .../java/org/apache/arrow/flight/auth/TestBasicAuth.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/flight/flight-core/src/test/java/org/apache/arrow/flight/auth/TestBasicAuth.java b/flight/flight-core/src/test/java/org/apache/arrow/flight/auth/TestBasicAuth.java index 0c63785c88..0f202ba2d9 100644 --- a/flight/flight-core/src/test/java/org/apache/arrow/flight/auth/TestBasicAuth.java +++ b/flight/flight-core/src/test/java/org/apache/arrow/flight/auth/TestBasicAuth.java @@ -178,6 +178,12 @@ public static void shutdown() throws Exception { AutoCloseables.close(server); allocator.getChildAllocators().forEach(BufferAllocator::close); + + // gRPC/Netty may still be releasing Arrow buffers asynchronously after server shutdown. + // Poll briefly to allow in-flight buffer releases to complete before closing the allocator. + for (int i = 0; i < 20 && allocator.getAllocatedMemory() > 0; i++) { + Thread.sleep(100); + } AutoCloseables.close(allocator); } } From 2f39438afd4a2c8bf7ba63b7e3aa726c680036e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 16:32:14 +0100 Subject: [PATCH 304/408] MINOR: Bump org.apache.orc:orc-core from 2.2.2 to 2.3.0 (#1056) Bumps org.apache.orc:orc-core from 2.2.2 to 2.3.0. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.orc:orc-core&package-manager=maven&previous-version=2.2.2&new-version=2.3.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- adapter/orc/pom.xml | 2 +- dataset/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/adapter/orc/pom.xml b/adapter/orc/pom.xml index 89d45e155c..c96ab36119 100644 --- a/adapter/orc/pom.xml +++ b/adapter/orc/pom.xml @@ -61,7 +61,7 @@ under the License. org.apache.orc orc-core - 2.2.2 + 2.3.0 test diff --git a/dataset/pom.xml b/dataset/pom.xml index 686a234358..1852c6eddc 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -130,7 +130,7 @@ under the License. org.apache.orc orc-core - 2.2.2 + 2.3.0 test From 07c5f48a16230275cb502b94ffe4a3ca70f9adad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Mon, 9 Mar 2026 17:32:10 +0100 Subject: [PATCH 305/408] MINOR: [CI] Increase JNI macOS job timeout from 45 to 60 minutes (#1060) As MacOS executor as slightly slower than other executors, this PR increase the JNI MacOS job timeout to 60 minutes (instead of 45 minutes). --- .github/workflows/rc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 8039f4c598..b866ff75f2 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -155,7 +155,7 @@ jobs: jni-macos: name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }} runs-on: ${{ matrix.platform.runs_on }} - timeout-minutes: 45 + timeout-minutes: 60 needs: - source strategy: From a7313c22c17211ecb666e44e97158a495a176778 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 17:32:56 +0100 Subject: [PATCH 306/408] MINOR: [CI] Bump docker/login-action from 3.7.0 to 4.0.0 (#1053) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [docker/login-action](https://github.com/docker/login-action) from 3.7.0 to 4.0.0.
Release notes

Sourced from docker/login-action's releases.

v4.0.0

Full Changelog: https://github.com/docker/login-action/compare/v3.7.0...v4.0.0

Commits
  • b45d80f Merge pull request #929 from crazy-max/node24
  • 176cb9c node 24 as default runtime
  • cad8984 Merge pull request #920 from docker/dependabot/npm_and_yarn/aws-sdk-dependenc...
  • 92cbcb2 chore: update generated content
  • 5a2d6a7 build(deps): bump the aws-sdk-dependencies group with 2 updates
  • 44512b6 Merge pull request #928 from docker/dependabot/npm_and_yarn/docker/actions-to...
  • 28737a5 chore: update generated content
  • dac0793 build(deps): bump @​docker/actions-toolkit from 0.76.0 to 0.77.0
  • 62029f3 Merge pull request #919 from docker/dependabot/npm_and_yarn/actions/core-3.0.0
  • 08c8f06 chore: update generated content
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=docker/login-action&package-manager=github_actions&previous-version=3.7.0&new-version=4.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index b866ff75f2..a202777143 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -127,7 +127,7 @@ jobs: with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing - - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: registry: ghcr.io username: ${{ github.actor }} From a53339b6ba2321f51f7f10f16a1ce06b12384498 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 18:59:48 +0100 Subject: [PATCH 307/408] MINOR: Bump dep.hadoop.version from 3.4.2 to 3.4.3 (#1055) Bumps `dep.hadoop.version` from 3.4.2 to 3.4.3. Updates `org.apache.hadoop:hadoop-client-runtime` from 3.4.2 to 3.4.3 Updates `org.apache.hadoop:hadoop-client-api` from 3.4.2 to 3.4.3 Updates `org.apache.hadoop:hadoop-common` from 3.4.2 to 3.4.3 Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6b7003f31a..e91e8c888f 100644 --- a/pom.xml +++ b/pom.xml @@ -102,7 +102,7 @@ under the License. 1.78.0 4.33.4 2.21.0 - 3.4.2 + 3.4.3 25.2.10 1.12.1 1.17.0 From 7390f551267798d4670eae6b2894c527dbc90403 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Mar 2026 14:46:43 +0100 Subject: [PATCH 308/408] MINOR: Bump io.grpc:grpc-bom from 1.78.0 to 1.79.0 (#1048) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [io.grpc:grpc-bom](https://github.com/grpc/grpc-java) from 1.78.0 to 1.79.0.
Release notes

Sourced from io.grpc:grpc-bom's releases.

v1.79.0

API Changes

  • core: Delete the never-used io.grpc.internal.ReadableBuffer.readBytes(ByteBuffer) (#12580) (738782fb0). This is deeply internal and not accessible, so shouldn’t impact anything. However, Apache Arrow Java uses reflection to access private fields; GH-939: Remove reflection for gRPC buffers is swapping to gRPC’s public zero-copy APIs

  • opentelemetry: Add target attribute filter for metrics (#12587). Introduce an optional Predicate targetAttributeFilter to control how grpc.target is recorded in OpenTelemetry client metrics. When a filter is provided, targets rejected by the predicate are normalized to "other" to reduce grpc.target metric cardinality, while accepted targets are recorded as-is. If no filter is set, existing behavior is preserved. This change adds a new Builder API on GrpcOpenTelemetry to allow applications to configure the filter. 

Behavior Changes

  • core: Convert AutoConfiguredLB to an actual LB (4bbf8eee5). This is an internal refactoring, but it does improve how errors are handled for broken binaries. Previously, not being able to load pick_first would result in a channel panic. Now it is handled as a regular load balancing error

  • okhttp: Assert no pending streams before transport READY (#12566) (ed6d175fc). No pending streams should exist when the transport transitions to READY. This PR adds an assertion to help verify this invariant.

Bug Fixes

  • core: PickFirstLB should not return a subchannel during CONNECTING (228fc8ecd). Pick-first in grpc-java has behaved this way since it was created, and it was of no consequence. However, now there are some load balancing policies (mainly RLS) that will do a pick() and hope the result to be reasonably accurate for metrics.

Improvements

  • core: Improve DEADLINE_EXCEEDED message for CallCreds delays (ead532b39). Previously the error message contained “buffered_nanos” and “waiting_for_connection” for connection delays. However, we discovered the same strings were also used if waiting on CallCredentials. Now you’ll see details like “connecting_and_lb_delay”, “call_credentials_delay”, and “was_still_waiting”.

  • opentelemetry: Add Android API checking (a9f73f4c0). Previously we assumed OpenTelemetry support would not be used on Android. It did happen to be compatible with Android, but since OpenTelemetry does have some Android support, we now have a check that it remains compatible

  • core: Catch Errors when calling complex config parsing code (a535ed799). Error (and any other Throwable) is now caught and handled when parsing configuration (e.g., service config, xds). This will cause such failures to be handled gracefully instead of panicking the channel

  • core: Implement LoadBalancer.Helper.createOobChannel() with the internals of createResolvingOobChannel() (3915d029c). This API is only expected to be relevant to the gRPC-LB lookaside load balancer, and is not believed to have behavior changes. Out-of-band channel had been implemented with its own stripped-down Channel without load balancing. Reimplementing using the resolving oob channel makes it a full-fledged channel and reduces the burden when integrating new features and allows us to have a ManagedChannelBuilder to use with efforts like gRFC A110: Child Channel Options.

  • xds: Implement the proactive connection logic in RingHashLoadBalancer as outlined in gRFC A61 (#12596). Previously, the Java implementation only initialized child balancers when a ring-chosen endpoint was in TRANSIENT_FAILURE during a picker's pickSubchannel call. This PR adds the missing logic: when a child balancer reports TRANSIENT_FAILURE, the LoadBalancer now proactively initializes the first available IDLE child if no other children are currently connecting or ready.

This ensures a backup subchannel starts warming up immediately outside the RPC flow, reducing failover latency and improving overall resilience. This behavior was previously present but was inadvertently lost after #10610.

  • api: Add RFC 3986 support to DnsNameResolverProvider (#12602) (f65127cf7) Experimental RFC 3986 target URI parsing mode (disabled by default)

New Features

Dependencies 

  • protobuf: Upgrade Bazel protobuf to 33.1 (#12553) (b61a8f49c) and load java_proto_library from the protobuf repo (c7f3cdbc3)

  • protobuf: Fix build with Bazel 9 by upgrading bazel_jar_jar and grpc-proto versions (#12569)

  • Upgrade dependencies (#12588) (6422092e3) Netty to 4.1.130, error-prone annotations to 2.45.0, google-auth-library to 1.41.0, tomcat-embed-core9 to 9.0.113, tomcat-embed-core to 10.1.50, opentelemetry to 1.57.0, jetty-ee10-servlet to 12.1.5, jetty-http2-server to 12.1.5, google-cloud-logging to 3.23.9, google-auth to 1.41.0, proto-google-common-protos to 2.63.2.

... (truncated)

Commits
  • 381593f Bump version to 1.79.0
  • f93ecb0 Update README etc to reference 1.79.0
  • f6d140f xds: Normalize weights before combining endpoint and locality weights
  • c589bef core: clarify dns javadoc/test about trailing path segments
  • 65596ae core: Move 4 test cases from DnsNameResolverTest to DnsNameResolverProviderTe...
  • 59a64f0 core: Use FlagResetRule to set/restore system properties in DnsNameResolverTe...
  • c5f5ee0 opentelemetry: Add target attribute filter for metrics (#12587)
  • f65127c api: Add RFC 3986 support to DnsNameResolverProvider (#12602)
  • a535ed7 Catch Errors when calling complex parsing code
  • ebb9420 xds: Merge ClusterResolverLB into CdsLB2
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=io.grpc:grpc-bom&package-manager=maven&previous-version=1.78.0&new-version=1.79.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: JB Onofré --- .../org/apache/arrow/flight/grpc/GetReadableBuffer.java | 6 +++--- pom.xml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/GetReadableBuffer.java b/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/GetReadableBuffer.java index 45c32a86c6..fcba88d212 100644 --- a/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/GetReadableBuffer.java +++ b/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/GetReadableBuffer.java @@ -87,13 +87,13 @@ public static void readIntoBuffer( final InputStream stream, final ArrowBuf buf, final int size, final boolean fastPath) throws IOException { ReadableBuffer readableBuffer = fastPath ? getReadableBuffer(stream) : null; + byte[] heapBytes = new byte[size]; if (readableBuffer != null) { - readableBuffer.readBytes(buf.nioBuffer(0, size)); + readableBuffer.readBytes(heapBytes, 0, size); } else { - byte[] heapBytes = new byte[size]; ByteStreams.readFully(stream, heapBytes); - buf.writeBytes(heapBytes); } + buf.writeBytes(heapBytes); buf.writerIndex(size); } } diff --git a/pom.xml b/pom.xml index e91e8c888f..19625617b1 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,7 @@ under the License. 2.0.17 33.4.8-jre 4.2.9.Final - 1.78.0 + 1.79.0 4.33.4 2.21.0 3.4.3 From e349a9a837aa9e3c5a56cbdd841f4e9655fa9ab2 Mon Sep 17 00:00:00 2001 From: Logan Riggs Date: Wed, 11 Mar 2026 00:09:23 -0700 Subject: [PATCH 309/408] GH-1061: Add codegen classifier jar for arrow-vector. (#1062) ## What's Changed Add a new codegen classifier jar for arrow-vector that contains tdd and other template files. Closes #1061 . --- docs/source/overview.rst | 3 +++ vector/pom.xml | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/docs/source/overview.rst b/docs/source/overview.rst index be579c1495..1188054114 100644 --- a/docs/source/overview.rst +++ b/docs/source/overview.rst @@ -45,6 +45,9 @@ but some modules are JNI bindings to the C++ library. * - arrow-vector - An off-heap reference implementation for Arrow columnar data format. - Native + * - arrow-vector-codegen + - Template files for Arrow datatypes suitable for code generation. + - Native * - arrow-tools - Java applications for working with Arrow ValueVectors. - Native diff --git a/vector/pom.xml b/vector/pom.xml index b24f37d5f9..f46bd0e7b4 100644 --- a/vector/pom.xml +++ b/vector/pom.xml @@ -194,6 +194,34 @@ under the License. + + org.apache.maven.plugins + maven-jar-plugin + + + codegen-jar + + jar + + package + + + codegen + ${basedir}/src/main/codegen + + **/*.tdd + **/*.fmpp + **/*.ftl + + + + + From bdec833fb69f945db9f3c767715c93d09214f2b5 Mon Sep 17 00:00:00 2001 From: Pedro Matias Date: Wed, 11 Mar 2026 07:34:47 +0000 Subject: [PATCH 310/408] GH-994: Fix DatabaseMetaData NPEs when SqlInfo is unavailable (#995) ## What's Changed Multiple DatabaseMetaData methods had NPEs when the method `ArrowDatabaseMetadata.getSqlInfoAndCacheIfCacheIsEmpty(final SqlInfo sqlInfoCommand, final Class desiredType)` returned null. Now the method never returns null. If the database server does not provide the requested info, either a sensible default is returned or a SQLException is thrown. Closes #994. --- .../driver/jdbc/ArrowDatabaseMetadata.java | 33 ++++++++- .../jdbc/ArrowDatabaseMetadataTest.java | 72 +++++++++++++++++++ 2 files changed, 104 insertions(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java index 502270e1cd..0110525fea 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java @@ -45,6 +45,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.Arrays; +import java.util.Collections; import java.util.EnumMap; import java.util.HashMap; import java.util.List; @@ -86,9 +87,12 @@ import org.apache.arrow.vector.util.Text; import org.apache.calcite.avatica.AvaticaConnection; import org.apache.calcite.avatica.AvaticaDatabaseMetaData; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** Arrow Flight JDBC's implementation of {@link DatabaseMetaData}. */ public class ArrowDatabaseMetadata extends AvaticaDatabaseMetaData { + private static final Logger LOGGER = LoggerFactory.getLogger(ArrowDatabaseMetadata.class); private static final String JAVA_REGEX_SPECIALS = "[]()|^-+*?{}$\\."; private static final Charset CHARSET = StandardCharsets.UTF_8; private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; @@ -774,7 +778,34 @@ private T getSqlInfoAndCacheIfCacheIsEmpty( } } } - return desiredType.cast(cachedSqlInfo.get(sqlInfoCommand)); + T value = desiredType.cast(cachedSqlInfo.get(sqlInfoCommand)); + if (value != null) { + return value; + } + LOGGER.debug( + "SqlInfo {} not provided by server, returning default for type {}", + sqlInfoCommand.name(), + desiredType.getSimpleName()); + + // Return sensible defaults when SqlInfo is unavailable + if (desiredType == Long.class) { + return desiredType.cast(0L); + } else if (desiredType == Integer.class) { + return desiredType.cast(0); + } else if (desiredType == Boolean.class) { + return desiredType.cast(false); + } else if (desiredType == String.class) { + return desiredType.cast(""); + } else if (desiredType == Map.class) { + return desiredType.cast(Collections.emptyMap()); + } else if (desiredType == List.class) { + return desiredType.cast(Collections.emptyList()); + } + + throw new SQLException( + String.format( + "The value of the SqlInfo %s is null and it could not be cast to %s.", + sqlInfoCommand.name(), desiredType.getName())); } private Optional convertListSqlInfoToString(final List sqlInfoList) { diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java index 81579cc387..3ab1460b27 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java @@ -1543,11 +1543,83 @@ public void testEmptySqlInfo() throws Exception { try (final Connection testConnection = FLIGHT_SERVER_EMPTY_SQLINFO_TEST_RULE.getConnection(false)) { final DatabaseMetaData metaData = testConnection.getMetaData(); + assertThat(metaData.getSQLKeywords(), is("")); assertThat(metaData.getNumericFunctions(), is("")); assertThat(metaData.getStringFunctions(), is("")); assertThat(metaData.getSystemFunctions(), is("")); assertThat(metaData.getTimeDateFunctions(), is("")); + + assertThat(metaData.getMaxBinaryLiteralLength(), is(0)); + assertThat(metaData.getMaxCharLiteralLength(), is(0)); + assertThat(metaData.getMaxColumnNameLength(), is(0)); + assertThat(metaData.getMaxColumnsInGroupBy(), is(0)); + assertThat(metaData.getMaxColumnsInIndex(), is(0)); + assertThat(metaData.getMaxColumnsInOrderBy(), is(0)); + assertThat(metaData.getMaxColumnsInSelect(), is(0)); + assertThat(metaData.getMaxColumnsInTable(), is(0)); + assertThat(metaData.getMaxConnections(), is(0)); + assertThat(metaData.getMaxCursorNameLength(), is(0)); + assertThat(metaData.getMaxIndexLength(), is(0)); + assertThat(metaData.getMaxSchemaNameLength(), is(0)); + assertThat(metaData.getMaxProcedureNameLength(), is(0)); + assertThat(metaData.getMaxCatalogNameLength(), is(0)); + assertThat(metaData.getMaxRowSize(), is(0)); + assertThat(metaData.getMaxStatementLength(), is(0)); + assertThat(metaData.getMaxStatements(), is(0)); + assertThat(metaData.getMaxTableNameLength(), is(0)); + assertThat(metaData.getMaxTablesInSelect(), is(0)); + assertThat(metaData.getMaxUserNameLength(), is(0)); + + assertThat(metaData.supportsColumnAliasing(), is(false)); + assertThat(metaData.nullPlusNonNullIsNull(), is(false)); + assertThat(metaData.supportsTableCorrelationNames(), is(false)); + assertThat(metaData.supportsDifferentTableCorrelationNames(), is(false)); + assertThat(metaData.supportsExpressionsInOrderBy(), is(false)); + assertThat(metaData.supportsOrderByUnrelated(), is(false)); + assertThat(metaData.supportsLikeEscapeClause(), is(false)); + assertThat(metaData.supportsNonNullableColumns(), is(false)); + assertThat(metaData.supportsIntegrityEnhancementFacility(), is(false)); + assertThat(metaData.isCatalogAtStart(), is(false)); + assertThat(metaData.supportsSelectForUpdate(), is(false)); + assertThat(metaData.supportsStoredProcedures(), is(false)); + assertThat(metaData.supportsCorrelatedSubqueries(), is(false)); + assertThat(metaData.doesMaxRowSizeIncludeBlobs(), is(false)); + assertThat(metaData.supportsTransactions(), is(false)); + assertThat(metaData.dataDefinitionCausesTransactionCommit(), is(false)); + assertThat(metaData.dataDefinitionIgnoredInTransactions(), is(false)); + assertThat(metaData.supportsBatchUpdates(), is(false)); + assertThat(metaData.supportsSavepoints(), is(false)); + assertThat(metaData.supportsNamedParameters(), is(false)); + assertThat(metaData.locatorsUpdateCopy(), is(false)); + assertThat(metaData.supportsStoredFunctionsUsingCallSyntax(), is(false)); + assertThat(metaData.supportsGroupBy(), is(false)); + assertThat(metaData.supportsGroupByUnrelated(), is(false)); + assertThat(metaData.supportsMinimumSQLGrammar(), is(false)); + assertThat(metaData.supportsCoreSQLGrammar(), is(false)); + assertThat(metaData.supportsExtendedSQLGrammar(), is(false)); + assertThat(metaData.supportsANSI92EntryLevelSQL(), is(false)); + assertThat(metaData.supportsANSI92IntermediateSQL(), is(false)); + assertThat(metaData.supportsANSI92FullSQL(), is(false)); + assertThat(metaData.supportsOuterJoins(), is(false)); + assertThat(metaData.supportsFullOuterJoins(), is(false)); + assertThat(metaData.supportsLimitedOuterJoins(), is(false)); + assertThat(metaData.supportsSchemasInProcedureCalls(), is(false)); + assertThat(metaData.supportsSchemasInIndexDefinitions(), is(false)); + assertThat(metaData.supportsSchemasInPrivilegeDefinitions(), is(false)); + assertThat(metaData.supportsCatalogsInIndexDefinitions(), is(false)); + assertThat(metaData.supportsCatalogsInPrivilegeDefinitions(), is(false)); + assertThat(metaData.supportsPositionedDelete(), is(false)); + assertThat(metaData.supportsPositionedUpdate(), is(false)); + assertThat(metaData.supportsSubqueriesInComparisons(), is(false)); + assertThat(metaData.supportsSubqueriesInExists(), is(false)); + assertThat(metaData.supportsSubqueriesInIns(), is(false)); + assertThat(metaData.supportsSubqueriesInQuantifieds(), is(false)); + assertThat(metaData.supportsUnion(), is(false)); + assertThat(metaData.supportsUnionAll(), is(false)); + assertThat(metaData.supportsConvert(), is(false)); + + assertThat(metaData.getDefaultTransactionIsolation(), is(Connection.TRANSACTION_NONE)); } } } From c8666f28569ca7e825b45f8ca39434c12428bec6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Mar 2026 11:09:05 +0100 Subject: [PATCH 311/408] MINOR: Bump com.gradle:common-custom-user-data-maven-extension from 2.0.3 to 2.1.0 (#998) Bumps [com.gradle:common-custom-user-data-maven-extension](https://github.com/gradle/common-custom-user-data-maven-extension) from 2.0.3 to 2.1.0.
Release notes

Sourced from com.gradle:common-custom-user-data-maven-extension's releases.

2.1.0

  • [NEW] Add support for evaluating one or more Groovy scripts in the Develocity storage directory

2.0.7

  • [FIX] Added a null-safety check to handle cases where the Maven session may be null

2.0.6

  • [FIX] GitHub Actions build link doesn't include run attempt

2.0.5

  • [FIX] Add GitHub run attempt as custom value to precisely identify GitHub Action run

2.0.4

  • [FIX] Add GitHub run number as custom value to precisely identify GitHub Action run
Commits
  • 0bb5838 [maven-release-plugin] prepare release v2.1.0
  • 4b1a27c Update changes.md
  • b9010d0 Merge pull request #329 from gradle/erichaagdev/groovy-scripts-m2-directory
  • 82281ec Switch Groovy script evaluation order
  • 7cdb3cc Clarify script locations in README
  • 2c511ae Add support for evaluating one or more Groovy scripts in the Develocity stora...
  • f02dbca Update to use version 2.0.7 of the Common Custom User Data Maven Extension
  • 88e0f41 Prepare for next round of development
  • 36edaf8 [maven-release-plugin] prepare for next development iteration
  • 683a966 [maven-release-plugin] prepare release v2.0.7
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.gradle:common-custom-user-data-maven-extension&package-manager=maven&previous-version=2.0.3&new-version=2.1.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .mvn/extensions.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index 0e25cc84f8..4585435b49 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -28,6 +28,6 @@ com.gradle common-custom-user-data-maven-extension - 2.0.3 + 2.1.0 From 6ffb2d0450effccf115203ae31da10708a35dda8 Mon Sep 17 00:00:00 2001 From: Aleksei Starikov Date: Wed, 11 Mar 2026 11:23:45 +0100 Subject: [PATCH 312/408] GH-301: [Vector] Allow adding a vector at the end of VectorSchemaRoot (#1013) ## What's Changed Allow adding a vector at the end of VectorSchemaRoot in the `VectorSchemaRoot#addVector()` method. Previously, the precondition `index < fieldVectors.size()` rejected `index == fieldVectors.size()`, so appending was impossible. The precondition is now `index <= fieldVectors.size()`, and when `index == fieldVectors.size()` the new vector is appended after all existing vectors. The implementation of `VectorSchemaRoot#addVector()` is now aligned with [BaseTable#insertVector()](https://github.com/apache/arrow-java/blob/main/vector/src/main/java/org/apache/arrow/vector/table/BaseTable.java#L156) The change is backward compatible, as it extends the functionality of the `VectorSchemaRoot#addVector()` method. Closes #301. --- .../apache/arrow/vector/VectorSchemaRoot.java | 15 +++++++++----- .../arrow/vector/TestVectorSchemaRoot.java | 20 +++++++++++++++++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java b/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java index a7cb9ced72..4c1fbf761a 100644 --- a/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java +++ b/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java @@ -199,13 +199,18 @@ public FieldVector getVector(int index) { */ public VectorSchemaRoot addVector(int index, FieldVector vector) { Preconditions.checkNotNull(vector); - Preconditions.checkArgument(index >= 0 && index < fieldVectors.size()); + Preconditions.checkArgument(index >= 0 && index <= fieldVectors.size()); List newVectors = new ArrayList<>(); - for (int i = 0; i < fieldVectors.size(); i++) { - if (i == index) { - newVectors.add(vector); + if (index == fieldVectors.size()) { + newVectors.addAll(fieldVectors); + newVectors.add(vector); + } else { + for (int i = 0; i < fieldVectors.size(); i++) { + if (i == index) { + newVectors.add(vector); + } + newVectors.add(fieldVectors.get(i)); } - newVectors.add(fieldVectors.get(i)); } return new VectorSchemaRoot(newVectors); } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestVectorSchemaRoot.java b/vector/src/test/java/org/apache/arrow/vector/TestVectorSchemaRoot.java index c121d94892..bd3113f8bc 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestVectorSchemaRoot.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestVectorSchemaRoot.java @@ -171,6 +171,26 @@ public void testAddVector() { } } + @Test + public void testAddVectorAtEnd() { + try (final IntVector intVector1 = new IntVector("intVector1", allocator); + final IntVector intVector2 = new IntVector("intVector2", allocator); + final IntVector intVector3 = new IntVector("intVector3", allocator); ) { + + VectorSchemaRoot original = new VectorSchemaRoot(Arrays.asList(intVector1, intVector2)); + assertEquals(2, original.getFieldVectors().size()); + + VectorSchemaRoot newRecordBatch = original.addVector(2, intVector3); + assertEquals(3, newRecordBatch.getFieldVectors().size()); + assertEquals(intVector1, newRecordBatch.getFieldVectors().get(0)); + assertEquals(intVector2, newRecordBatch.getFieldVectors().get(1)); + assertEquals(intVector3, newRecordBatch.getFieldVectors().get(2)); + + original.close(); + newRecordBatch.close(); + } + } + @Test public void testRemoveVector() { try (final IntVector intVector1 = new IntVector("intVector1", allocator); From 18de621ff2e7a72f54d416b9ef6e6a4a96b2aa8d Mon Sep 17 00:00:00 2001 From: Pedro Matias Date: Wed, 11 Mar 2026 10:59:07 +0000 Subject: [PATCH 313/408] =?UTF-8?q?GH-1004:=20=20[JDBC]=20Fix=20NPE=20in?= =?UTF-8?q?=20ArrowFlightJdbcDriver#connect=E2=80=8B(final=20String=20url,?= =?UTF-8?q?=20final=20Properties=20info)=20=20(#1005)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What's Changed `ArrowFlightJdbcDriver.connect(final String url, final Properties info)` now properly ignores a null value for `info`, obtaining the properties solely from the URL. Closes #1004. --- .../driver/jdbc/ArrowFlightJdbcDriver.java | 4 +++- .../jdbc/ArrowFlightJdbcDriverTest.java | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriver.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriver.java index 53e6120f62..12ef8030d7 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriver.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriver.java @@ -75,7 +75,9 @@ public Logger getParentLogger() { public ArrowFlightConnection connect(final String url, final Properties info) throws SQLException { final Properties properties = new Properties(info); - properties.putAll(info); + if (info != null) { + properties.putAll(info); + } if (url != null) { final Optional> maybeProperties = getUrlsArgs(url); diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriverTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriverTest.java index ae355829d7..88fb9889b6 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriverTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriverTest.java @@ -201,6 +201,30 @@ public void testConnectWithInsensitiveCasePropertyKeys2() throws Exception { } } + /** + * Tests whether the {@link ArrowFlightJdbcDriver} can establish a successful connection to the + * Arrow Flight client when provided with null properties. + */ + @Test + public void testConnectWithNullProperties() throws Exception { + final Driver driver = new ArrowFlightJdbcDriver(); + try (Connection connection = + driver.connect( + "jdbc:arrow-flight://" + + dataSource.getConfig().getHost() + + ":" + + dataSource.getConfig().getPort() + + "?" + + "useEncryption=false" + + "&user=" + + dataSource.getConfig().getUser() + + "&password=" + + dataSource.getConfig().getPassword(), + null)) { + assertTrue(connection.isValid(300)); + } + } + /** * Tests whether an exception is thrown upon attempting to connect to a malformed URI. * From 4a7fb4ef95d9386a0a7102fb18fbe52547527c21 Mon Sep 17 00:00:00 2001 From: Aleksei Starikov Date: Wed, 11 Mar 2026 14:59:21 +0100 Subject: [PATCH 314/408] GH-552: [Vector] Add absent methods to the UnionFixedSizeListWriter (#1052) ## What's Changed Add absent methods to the `UnionFixedSizeListWriter`. 1. Aligned the `UnionFixedSizeListWriter` template with the `UnionListWriter` template, which added the following previously absent methods to the generated `UnionFixedSizeListWriter` class: ``` - duration() methods - DurationWriter duration() - DurationWriter duration(String name, org.apache.arrow.vector.types.TimeUnit unit) - DurationWriter duration(String name) - timeStampSecTZ() methods - TimeStampSecTZWriter timeStampSecTZ() - TimeStampSecTZWriter timeStampSecTZ(String name, String timezone) - TimeStampSecTZWriter timeStampSecTZ(String name) - timeStampMilliTZ() methods - TimeStampMilliTZWriter timeStampMilliTZ() - TimeStampMilliTZWriter timeStampMilliTZ(String name, String timezone) - TimeStampMilliTZWriter timeStampMilliTZ(String name) - timeStampMicroTZ() methods - TimeStampMicroTZWriter timeStampMicroTZ() - TimeStampMicroTZWriter timeStampMicroTZ(String name, String timezone) - TimeStampMicroTZWriter timeStampMicroTZ(String name) - timeStampNanoTZ() methods - TimeStampNanoTZWriter timeStampNanoTZ() - TimeStampNanoTZWriter timeStampNanoTZ(String name, String timezone) - TimeStampNanoTZWriter timeStampNanoTZ(String name) - fixedSizeBinary() methods - FixedSizeBinaryWriter fixedSizeBinary() - FixedSizeBinaryWriter fixedSizeBinary(String name, int byteWidth) - FixedSizeBinaryWriter fixedSizeBinary(String name) - write() methods for Duration - void writeDuration(long value) - void write(DurationHolder holder) - write() methods for TimeStampSecTZ - void writeTimeStampSecTZ(long value) - void write(TimeStampSecTZHolder holder) - write() methods for TimeStampMilliTZ - void writeTimeStampMilliTZ(long value) - void write(TimeStampMilliTZHolder holder) - write() methods for TimeStampMicroTZ - void writeTimeStampMicroTZ(long value) - void write(TimeStampMicroTZHolder holder) - write() methods for TimeStampNanoTZ - void writeTimeStampNanoTZ(long value) - void write(TimeStampNanoTZHolder holder) - write() methods for FixedSizeBinary - void writeFixedSizeBinary(ArrowBuf buffer) - void write(FixedSizeBinaryHolder holder) ``` 2. Add `structName = name;` for 2 existing methods (align them with other similar methods): ``` - DecimalWriter decimal(String name) - Decimal256Writer decimal256(String name) ``` 3. Remove unused assignments from the `UnionListWriter` template + add missing overrides. This fix adds override annotations to the `UnionListWriter` generated class and extend/fix the code of the `UnionFixedSizeListWriter` generated class. So, the change is backward compatible. See the gists for the generated writer classes: - [UnionFixedSizeListWriter](https://gist.github.com/axreldable/9908f98d75ec4c0a62e4ccfa176cbbe1) - [UnionListWriter](https://gist.github.com/axreldable/881f3cf1ed001c513870daf3ea4f3bbd) --- Inspired by this [PR](https://github.com/apache/arrow/pull/35353). --- Closes #552 . --- .../templates/UnionFixedSizeListWriter.java | 151 +++++-------- .../codegen/templates/UnionListWriter.java | 6 +- .../arrow/vector/TestFixedSizeListVector.java | 207 ++++++++++++++++++ .../apache/arrow/vector/TestMapVector.java | 35 ++- .../org/apache/arrow/vector/TestUtils.java | 13 ++ 5 files changed, 293 insertions(+), 119 deletions(-) diff --git a/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java b/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java index f6e3f63caf..484199ab2a 100644 --- a/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java +++ b/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java @@ -35,6 +35,10 @@ <#include "/@includes/vv_imports.ftl" /> +<#function is_timestamp_tz type> + <#return type?starts_with("TimeStamp") && type?ends_with("TZ")> + + /* * This class is generated using freemarker and the ${.template_name} template. */ @@ -96,55 +100,30 @@ public void close() throws Exception { public void setPosition(int index) { super.setPosition(index); } - <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> - <#assign fields = minor.fields!type.fields /> - <#assign uncappedName = name?uncap_first/> - <#if uncappedName == "int" ><#assign uncappedName = "integer" /> - <#if !minor.typeParams?? > + <#list vv.types as type><#list type.minor as minor> + <#assign lowerName = minor.class?uncap_first /> + <#if lowerName == "int" ><#assign lowerName = "integer" /> + <#assign upperName = minor.class?upper_case /> @Override - public ${name}Writer ${uncappedName}() { + public ${minor.class}Writer ${lowerName}() { return this; } + <#if minor.typeParams?? > @Override - public ${name}Writer ${uncappedName}(String name) { - structName = name; - return writer.${uncappedName}(name); + public ${minor.class}Writer ${lowerName}(String name<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}) { + return writer.${lowerName}(name<#list minor.typeParams as typeParam>, ${typeParam.name}); } - - - @Override - public DecimalWriter decimal() { - return this; - } - - @Override - public DecimalWriter decimal(String name, int scale, int precision) { - return writer.decimal(name, scale, precision); - } - - @Override - public DecimalWriter decimal(String name) { - return writer.decimal(name); - } - @Override - public Decimal256Writer decimal256() { - return this; - } - - @Override - public Decimal256Writer decimal256(String name, int scale, int precision) { - return writer.decimal256(name, scale, precision); + public ${minor.class}Writer ${lowerName}(String name) { + structName = name; + return writer.${lowerName}(name); } - @Override - public Decimal256Writer decimal256(String name) { - return writer.decimal256(name); - } + @Override public StructWriter struct() { @@ -215,87 +194,86 @@ public void end() { } @Override - public void write(DecimalHolder holder) { - if (writer.idx() >= (idx() + 1) * listSize) { - throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); - } - writer.write(holder); - writer.setPosition(writer.idx() + 1); - } - - @Override - public void write(Decimal256Holder holder) { + public void writeNull() { if (writer.idx() >= (idx() + 1) * listSize) { throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); } - writer.write(holder); - writer.setPosition(writer.idx() + 1); + writer.writeNull(); } + <#list vv.types as type> + <#list type.minor as minor> + <#assign name = minor.class?cap_first /> + <#assign fields = minor.fields!type.fields /> + <#assign uncappedName = name?uncap_first/> @Override - public void writeNull() { + public void write${name}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, ) { if (writer.idx() >= (idx() + 1) * listSize) { throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); } - writer.writeNull(); + writer.write${name}(<#list fields as field>${field.name}<#if field_has_next>, ); + writer.setPosition(writer.idx()+1); } - public void writeDecimal(long start, ArrowBuf buffer, ArrowType arrowType) { + <#if is_timestamp_tz(minor.class) || minor.class == "Duration" || minor.class == "FixedSizeBinary"> + @Override + public void write(${name}Holder holder) { if (writer.idx() >= (idx() + 1) * listSize) { throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); } - writer.writeDecimal(start, buffer, arrowType); - writer.setPosition(writer.idx() + 1); + writer.write(holder); + writer.setPosition(writer.idx()+1); } - public void writeDecimal(BigDecimal value) { + <#elseif minor.class?starts_with("Decimal")> + @Override + public void write${name}(long start, ArrowBuf buffer, ArrowType arrowType) { if (writer.idx() >= (idx() + 1) * listSize) { throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); } - writer.writeDecimal(value); - writer.setPosition(writer.idx() + 1); + writer.write${name}(start, buffer, arrowType); + writer.setPosition(writer.idx()+1); } - public void writeBigEndianBytesToDecimal(byte[] value, ArrowType arrowType) { + @Override + public void write(${name}Holder holder) { if (writer.idx() >= (idx() + 1) * listSize) { throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); } - writer.writeBigEndianBytesToDecimal(value, arrowType); - writer.setPosition(writer.idx() + 1); + writer.write(holder); + writer.setPosition(writer.idx()+1); } - public void writeDecimal256(long start, ArrowBuf buffer, ArrowType arrowType) { + @Override + public void write${name}(BigDecimal value) { if (writer.idx() >= (idx() + 1) * listSize) { throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); } - writer.writeDecimal256(start, buffer, arrowType); - writer.setPosition(writer.idx() + 1); + writer.write${name}(value); + writer.setPosition(writer.idx()+1); } - public void writeDecimal256(BigDecimal value) { + @Override + public void writeBigEndianBytesTo${name}(byte[] value, ArrowType arrowType){ if (writer.idx() >= (idx() + 1) * listSize) { throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); } - writer.writeDecimal256(value); + writer.writeBigEndianBytesTo${name}(value, arrowType); writer.setPosition(writer.idx() + 1); } - - public void writeBigEndianBytesToDecimal256(byte[] value, ArrowType arrowType) { + <#else> + @Override + public void write(${name}Holder holder) { if (writer.idx() >= (idx() + 1) * listSize) { throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); } - writer.writeBigEndianBytesToDecimal256(value, arrowType); - writer.setPosition(writer.idx() + 1); + writer.write${name}(<#list fields as field>holder.${field.name}<#if field_has_next>, ); + writer.setPosition(writer.idx()+1); } + - - <#list vv.types as type> - <#list type.minor as minor> - <#assign name = minor.class?cap_first /> - <#assign fields = minor.fields!type.fields /> - <#assign uncappedName = name?uncap_first/> - <#if minor.class?ends_with("VarBinary")> + <#if minor.class?ends_with("VarBinary")> @Override public void write${minor.class}(byte[] value) { if (writer.idx() >= (idx() + 1) * listSize) { @@ -349,27 +327,8 @@ public void writeBigEndianBytesToDecimal256(byte[] value, ArrowType arrowType) { writer.write${minor.class}(value); writer.setPosition(writer.idx() + 1); } - - - <#if !minor.typeParams?? > - @Override - public void write${name}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, ) { - if (writer.idx() >= (idx() + 1) * listSize) { - throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); - } - writer.write${name}(<#list fields as field>${field.name}<#if field_has_next>, ); - writer.setPosition(writer.idx() + 1); - } - - public void write(${name}Holder holder) { - if (writer.idx() >= (idx() + 1) * listSize) { - throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); - } - writer.write${name}(<#list fields as field>holder.${field.name}<#if field_has_next>, ); - writer.setPosition(writer.idx() + 1); - } + - } diff --git a/vector/src/main/codegen/templates/UnionListWriter.java b/vector/src/main/codegen/templates/UnionListWriter.java index 4b54739230..394348f029 100644 --- a/vector/src/main/codegen/templates/UnionListWriter.java +++ b/vector/src/main/codegen/templates/UnionListWriter.java @@ -123,8 +123,6 @@ public void setPosition(int index) { <#assign lowerName = minor.class?uncap_first /> <#if lowerName == "int" ><#assign lowerName = "integer" /> <#assign upperName = minor.class?upper_case /> - <#assign capName = minor.class?cap_first /> - <#assign vectName = capName /> @Override public ${minor.class}Writer ${lowerName}() { return this; @@ -370,6 +368,7 @@ public void write(${name}Holder holder) { } <#elseif minor.class?starts_with("Decimal")> + @Override public void write${name}(long start, ArrowBuf buffer, ArrowType arrowType) { writer.write${name}(start, buffer, arrowType); writer.setPosition(writer.idx()+1); @@ -381,11 +380,13 @@ public void write(${name}Holder holder) { writer.setPosition(writer.idx()+1); } + @Override public void write${name}(BigDecimal value) { writer.write${name}(value); writer.setPosition(writer.idx()+1); } + @Override public void writeBigEndianBytesTo${name}(byte[] value, ArrowType arrowType){ writer.writeBigEndianBytesTo${name}(value, arrowType); writer.setPosition(writer.idx() + 1); @@ -429,6 +430,7 @@ public void write(${name}Holder holder) { writer.setPosition(writer.idx() + 1); } + @Override public void write${minor.class}(String value) { writer.write${minor.class}(value); writer.setPosition(writer.idx() + 1); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java index 73a88b3a1e..b3455fe52c 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java @@ -30,14 +30,21 @@ import java.util.Arrays; import java.util.List; import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.BaseRepeatedValueVector; import org.apache.arrow.vector.complex.FixedSizeListVector; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.impl.UnionFixedSizeListReader; import org.apache.arrow.vector.complex.impl.UnionFixedSizeListWriter; import org.apache.arrow.vector.complex.impl.UnionListReader; import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.DurationHolder; +import org.apache.arrow.vector.holders.FixedSizeBinaryHolder; +import org.apache.arrow.vector.holders.TimeStampMilliTZHolder; +import org.apache.arrow.vector.holders.TimeStampNanoTZHolder; +import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.Text; import org.apache.arrow.vector.util.TransferPair; @@ -628,6 +635,206 @@ public void testWriteLargeVarBinaryHelpers() throws Exception { } } + @Test + public void testWriterTimeStampNanoTZField() { + try (final FixedSizeListVector vector = + FixedSizeListVector.empty("vector", /* size= */ 3, allocator)) { + UnionFixedSizeListWriter writer = vector.getWriter(); + writer.allocate(); + + final int valueCount = 10; + + for (int i = 0; i < valueCount; i++) { + writer.startList(); + writer.timeStampNanoTZ().writeTimeStampNanoTZ(i * 1000L); + writer.timeStampNanoTZ().writeTimeStampNanoTZ((i + 1) * 1000L); + writer.timeStampNanoTZ().writeTimeStampNanoTZ((i + 2) * 1000L); + writer.endList(); + } + vector.setValueCount(valueCount); + + UnionFixedSizeListReader reader = vector.getReader(); + for (int i = 0; i < valueCount; i++) { + reader.setPosition(i); + assertTrue(reader.isSet()); + assertTrue(reader.next()); + assertEquals(i * 1000L, reader.reader().readLong().longValue()); + assertTrue(reader.next()); + assertEquals((i + 1) * 1000L, reader.reader().readLong().longValue()); + assertTrue(reader.next()); + assertEquals((i + 2) * 1000L, reader.reader().readLong().longValue()); + assertFalse(reader.next()); + } + } + } + + @Test + public void testWriterUsingHolderTimeStampNanoTZField() { + try (final FixedSizeListVector vector = + FixedSizeListVector.empty("vector", /* size= */ 3, allocator)) { + UnionFixedSizeListWriter writer = vector.getWriter(); + writer.allocate(); + + TimeStampNanoTZHolder holder = new TimeStampNanoTZHolder(); + holder.timezone = "SomeFakeTimeZone"; + writer.startList(); + holder.value = 12341234L; + writer.timeStampNanoTZ().write(holder); + holder.value = 55555L; + writer.timeStampNanoTZ().write(holder); + + // Writing with a different timezone should throw + holder.timezone = "AsdfTimeZone"; + holder.value = 77777; + IllegalArgumentException ex = + assertThrows( + IllegalArgumentException.class, () -> writer.timeStampNanoTZ().write(holder)); + assertEquals( + "holder.timezone: AsdfTimeZone not equal to vector timezone: SomeFakeTimeZone", + ex.getMessage()); + + writer.endList(); + vector.setValueCount(1); + + Field expectedDataField = + new Field( + BaseRepeatedValueVector.DATA_VECTOR_NAME, + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.NANOSECOND, "SomeFakeTimeZone")), + null); + Field expectedField = + new Field( + vector.getName(), + FieldType.nullable(new ArrowType.FixedSizeList(3)), + List.of(expectedDataField)); + + assertEquals(expectedField, writer.getField()); + } + } + + @Test + public void testWriterUsingHolderTimestampMilliTZField() { + try (final FixedSizeListVector vector = + FixedSizeListVector.empty("vector", /* size= */ 3, allocator)) { + UnionFixedSizeListWriter writer = vector.getWriter(); + writer.allocate(); + + TimeStampMilliTZHolder holder = new TimeStampMilliTZHolder(); + holder.timezone = "SomeFakeTimeZone"; + writer.startList(); + holder.value = 12341234L; + writer.timeStampMilliTZ().write(holder); + holder.value = 55555L; + writer.timeStampMilliTZ().write(holder); + + // Writing with a different timezone should throw + holder.timezone = "AsdfTimeZone"; + holder.value = 77777; + IllegalArgumentException ex = + assertThrows( + IllegalArgumentException.class, () -> writer.timeStampMilliTZ().write(holder)); + assertEquals( + "holder.timezone: AsdfTimeZone not equal to vector timezone: SomeFakeTimeZone", + ex.getMessage()); + + writer.endList(); + vector.setValueCount(1); + + Field expectedDataField = + new Field( + BaseRepeatedValueVector.DATA_VECTOR_NAME, + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, "SomeFakeTimeZone")), + null); + Field expectedField = + new Field( + vector.getName(), + FieldType.nullable(new ArrowType.FixedSizeList(3)), + List.of(expectedDataField)); + + assertEquals(expectedField, writer.getField()); + } + } + + @Test + public void testWriterUsingHolderDurationField() { + try (final FixedSizeListVector vector = + FixedSizeListVector.empty("vector", /* size= */ 3, allocator)) { + UnionFixedSizeListWriter writer = vector.getWriter(); + writer.allocate(); + + DurationHolder durationHolder = new DurationHolder(); + durationHolder.unit = TimeUnit.MILLISECOND; + + writer.startList(); + durationHolder.value = 812374L; + writer.duration().write(durationHolder); + durationHolder.value = 143451L; + writer.duration().write(durationHolder); + + // Writing with a different unit should throw + durationHolder.unit = TimeUnit.SECOND; + durationHolder.value = 8888888; + IllegalArgumentException ex = + assertThrows( + IllegalArgumentException.class, () -> writer.duration().write(durationHolder)); + assertEquals("holder.unit: SECOND not equal to vector unit: MILLISECOND", ex.getMessage()); + + writer.endList(); + vector.setValueCount(1); + + Field expectedDataField = + new Field( + BaseRepeatedValueVector.DATA_VECTOR_NAME, + FieldType.nullable(new ArrowType.Duration(TimeUnit.MILLISECOND)), + null); + Field expectedField = + new Field( + vector.getName(), + FieldType.nullable(new ArrowType.FixedSizeList(3)), + List.of(expectedDataField)); + + assertEquals(expectedField, writer.getField()); + } + } + + @Test + public void testWriterUsingHolderFixedSizeBinaryField() { + try (final FixedSizeListVector vector = + FixedSizeListVector.empty("vector", /* size= */ 2, allocator)) { + UnionFixedSizeListWriter writer = vector.getWriter(); + writer.allocate(); + + FixedSizeBinaryHolder holder1 = + TestUtils.fixedSizeBinaryHolder(allocator, new byte[] {11, 22}); + FixedSizeBinaryHolder holder2 = + TestUtils.fixedSizeBinaryHolder(allocator, new byte[] {32, 21}); + + writer.startList(); + writer.fixedSizeBinary().write(holder1); + holder1.buffer.close(); + writer.fixedSizeBinary().write(holder2); + holder2.buffer.close(); + + writer.endList(); + vector.setValueCount(1); + + FieldReader reader = vector.getReader(); + assertTrue(reader.isSet(), "shouldn't be null"); + + Field expectedDataField = + new Field( + BaseRepeatedValueVector.DATA_VECTOR_NAME, + FieldType.nullable(new ArrowType.FixedSizeBinary(2)), + null); + Field expectedField = + new Field( + vector.getName(), + FieldType.nullable(new ArrowType.FixedSizeList(2)), + List.of(expectedDataField)); + + assertEquals(expectedField, writer.getField()); + } + } + private int[] convertListToIntArray(List list) { int[] values = new int[list.size()]; for (int i = 0; i < list.size(); i++) { diff --git a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java index 274d2973bd..2f520f3882 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java @@ -1353,17 +1353,6 @@ public void testCopyFromForExtensionType() throws Exception { } } - private FixedSizeBinaryHolder getFixedSizeBinaryHolder(byte[] array) { - FixedSizeBinaryHolder holder = new FixedSizeBinaryHolder(); - holder.byteWidth = array.length; - holder.buffer = allocator.buffer(array.length); - for (int i = 0; i < array.length; i++) { - holder.buffer.setByte(i, array[i]); - } - - return holder; - } - /** * Regression test for GH-586: UnionMapWriter.fixedSizeBinary() should properly delegate to the * entry writer for both key and value paths. @@ -1382,8 +1371,10 @@ public void testFixedSizeBinaryWriter() { // {[11, 22] -> null} // {null -> [32, 21]} - wrong "for a given entry, the "key" is non-nullable" - todo: it // shouldn't work. Should it? - FixedSizeBinaryHolder holder1 = getFixedSizeBinaryHolder(new byte[] {11, 22}); - FixedSizeBinaryHolder holder2 = getFixedSizeBinaryHolder(new byte[] {32, 21}); + FixedSizeBinaryHolder holder1 = + TestUtils.fixedSizeBinaryHolder(allocator, new byte[] {11, 22}); + FixedSizeBinaryHolder holder2 = + TestUtils.fixedSizeBinaryHolder(allocator, new byte[] {32, 21}); writer.setPosition(0); // optional writer.startMap(); @@ -1399,8 +1390,8 @@ public void testFixedSizeBinaryWriter() { writer.endMap(); // {1 -> [11, 22], 2 -> [32, 21]} - holder1 = getFixedSizeBinaryHolder(new byte[] {11, 22}); - holder2 = getFixedSizeBinaryHolder(new byte[] {32, 21}); + holder1 = TestUtils.fixedSizeBinaryHolder(allocator, new byte[] {11, 22}); + holder2 = TestUtils.fixedSizeBinaryHolder(allocator, new byte[] {32, 21}); writer.setPosition(1); writer.startMap(); writer.startEntry(); @@ -1416,8 +1407,8 @@ public void testFixedSizeBinaryWriter() { holder2.buffer.close(); // {[11, 22] -> 1, [32, 21] -> 2} - holder1 = getFixedSizeBinaryHolder(new byte[] {11, 22}); - holder2 = getFixedSizeBinaryHolder(new byte[] {32, 21}); + holder1 = TestUtils.fixedSizeBinaryHolder(allocator, new byte[] {11, 22}); + holder2 = TestUtils.fixedSizeBinaryHolder(allocator, new byte[] {32, 21}); writer.setPosition(3); writer.startMap(); writer.startEntry(); @@ -1433,7 +1424,7 @@ public void testFixedSizeBinaryWriter() { holder2.buffer.close(); // {[11, 22] -> null} - holder1 = getFixedSizeBinaryHolder(new byte[] {11, 22}); + holder1 = TestUtils.fixedSizeBinaryHolder(allocator, new byte[] {11, 22}); writer.setPosition(4); writer.startMap(); writer.startEntry(); @@ -1443,7 +1434,7 @@ public void testFixedSizeBinaryWriter() { holder1.buffer.close(); // {null -> [32, 21]} - holder2 = getFixedSizeBinaryHolder(new byte[] {32, 21}); + holder2 = TestUtils.fixedSizeBinaryHolder(allocator, new byte[] {32, 21}); writer.setPosition(5); writer.startMap(); writer.startEntry(); @@ -1536,8 +1527,10 @@ public void testFixedSizeBinaryFirstInitialization() { // populate input vector with the following records // {[11, 22] -> [32, 21]} - FixedSizeBinaryHolder holder1 = getFixedSizeBinaryHolder(new byte[] {11, 22}); - FixedSizeBinaryHolder holder2 = getFixedSizeBinaryHolder(new byte[] {32, 21}); + FixedSizeBinaryHolder holder1 = + TestUtils.fixedSizeBinaryHolder(allocator, new byte[] {11, 22}); + FixedSizeBinaryHolder holder2 = + TestUtils.fixedSizeBinaryHolder(allocator, new byte[] {32, 21}); writer.setPosition(0); // optional writer.startMap(); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestUtils.java b/vector/src/test/java/org/apache/arrow/vector/TestUtils.java index c28751aa58..d91b2004c0 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestUtils.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestUtils.java @@ -18,6 +18,7 @@ import java.util.Random; import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.holders.FixedSizeBinaryHolder; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.ExtensionTypeRegistry; @@ -73,4 +74,16 @@ public static void ensureRegistered(ArrowType.ExtensionType type) { ExtensionTypeRegistry.register(type); } } + + public static FixedSizeBinaryHolder fixedSizeBinaryHolder( + BufferAllocator allocator, byte[] array) { + FixedSizeBinaryHolder holder = new FixedSizeBinaryHolder(); + holder.byteWidth = array.length; + holder.buffer = allocator.buffer(array.length); + for (int i = 0; i < array.length; i++) { + holder.buffer.setByte(i, array[i]); + } + + return holder; + } } From 77df3ecb2cf5517fb5d37a4b2806844e3b4700df Mon Sep 17 00:00:00 2001 From: Yicong Huang <17627829+Yicong-Huang@users.noreply.github.com> Date: Thu, 12 Mar 2026 01:39:44 -0700 Subject: [PATCH 315/408] GH-343: Fix BaseVariableWidthVector and BaseLargeVariableWidthVector offset buffer serialization (#989) ## What's Changed Fix `BaseVariableWidthVector`/`BaseLargeVariableWidthVector` IPC serialization when `valueCount` is 0. ### Problem When `valueCount == 0`, `setReaderAndWriterIndex()` was setting `offsetBuffer.writerIndex(0)`, which means `readableBytes() == 0`. IPC serializer uses `readableBytes()` to determine buffer size, so 0 bytes were written to the IPC stream. This crashes IPC readers in other libraries because Arrow spec requires offset buffer to have at least one entry `[0]`. This is a follow-up to #967 which fixed the same issue in `ListVector`/`LargeListVector`. ### Fix Modify `setReaderAndWriterIndex()` to always use `(valueCount + 1) * OFFSET_WIDTH` for the offset buffer's `writerIndex`, moved outside the if/else branch. When the offset buffer capacity is insufficient (e.g., empty buffer from constructor or loaded via `loadFieldBuffers()`), it reallocates a properly sized buffer on demand. ### Testing Added tests for empty `VarCharVector` and `LargeVarCharVector` verifying offset buffer has correct `readableBytes()` after `setValueCount(0)`. Closes #343 --------- Co-authored-by: Yicong Huang --- .../adapter/jdbc/ResultSetUtilityTest.java | 22 ++++++----- .../vector/BaseLargeVariableWidthVector.java | 16 +++++++- .../arrow/vector/BaseVariableWidthVector.java | 16 +++++++- .../apache/arrow/vector/TestValueVector.java | 38 +++++++++++++++++++ 4 files changed, 79 insertions(+), 13 deletions(-) diff --git a/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ResultSetUtilityTest.java b/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ResultSetUtilityTest.java index c7dc9b2791..e5039ccf59 100644 --- a/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ResultSetUtilityTest.java +++ b/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ResultSetUtilityTest.java @@ -43,15 +43,19 @@ public void testZeroRowResultSet() throws Exception { .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) .build(); - ArrowVectorIterator iter = JdbcToArrow.sqlToArrowVectorIterator(rs, config); - assertTrue(iter.hasNext(), "Iterator on zero row ResultSet should haveNext() before use"); - VectorSchemaRoot root = iter.next(); - assertNotNull(root, "VectorSchemaRoot from first next() result should never be null"); - assertEquals( - 0, root.getRowCount(), "VectorSchemaRoot from empty ResultSet should have zero rows"); - assertFalse( - iter.hasNext(), - "hasNext() should return false on empty ResultSets after initial next() call"); + try (ArrowVectorIterator iter = JdbcToArrow.sqlToArrowVectorIterator(rs, config)) { + assertTrue(iter.hasNext(), "Iterator on zero row ResultSet should haveNext() before use"); + VectorSchemaRoot root = iter.next(); + assertNotNull(root, "VectorSchemaRoot from first next() result should never be null"); + assertEquals( + 0, root.getRowCount(), "VectorSchemaRoot from empty ResultSet should have zero rows"); + assertFalse( + iter.hasNext(), + "hasNext() should return false on empty ResultSets after initial next() call"); + if (!reuseVectorSchemaRoot) { + root.close(); + } + } } } } diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java index 6c451f10a7..3fac195786 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java @@ -373,14 +373,26 @@ private void setReaderAndWriterIndex() { valueBuffer.readerIndex(0); if (valueCount == 0) { validityBuffer.writerIndex(0); - offsetBuffer.writerIndex(0); valueBuffer.writerIndex(0); } else { final long lastDataOffset = getStartOffset(valueCount); validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); - offsetBuffer.writerIndex((long) (valueCount + 1) * OFFSET_WIDTH); valueBuffer.writerIndex(lastDataOffset); } + // IPC serializer will determine readable bytes based on `readerIndex` and `writerIndex`. + // Both are set to 0 means 0 bytes are written to the IPC stream which will crash IPC readers + // in other libraries. According to Arrow spec, we should still output the offset buffer which + // is [0]. + final long requiredOffsetBufferSize = (long) (valueCount + 1) * OFFSET_WIDTH; + if (offsetBuffer.capacity() < requiredOffsetBufferSize) { + ArrowBuf newOffsetBuffer = allocateOffsetBuffer(requiredOffsetBufferSize); + if (offsetBuffer.capacity() > 0) { + newOffsetBuffer.setBytes(0, offsetBuffer, 0, offsetBuffer.capacity()); + } + offsetBuffer.getReferenceManager().release(); + offsetBuffer = newOffsetBuffer; + } + offsetBuffer.writerIndex(requiredOffsetBufferSize); } /** Same as {@link #allocateNewSafe()}. */ diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java index 96e2afbd29..d5bd167256 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java @@ -389,14 +389,26 @@ private void setReaderAndWriterIndex() { valueBuffer.readerIndex(0); if (valueCount == 0) { validityBuffer.writerIndex(0); - offsetBuffer.writerIndex(0); valueBuffer.writerIndex(0); } else { final int lastDataOffset = getStartOffset(valueCount); validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); - offsetBuffer.writerIndex((long) (valueCount + 1) * OFFSET_WIDTH); valueBuffer.writerIndex(lastDataOffset); } + // IPC serializer will determine readable bytes based on `readerIndex` and `writerIndex`. + // Both are set to 0 means 0 bytes are written to the IPC stream which will crash IPC readers + // in other libraries. According to Arrow spec, we should still output the offset buffer which + // is [0]. + final long requiredOffsetBufferSize = (long) (valueCount + 1) * OFFSET_WIDTH; + if (offsetBuffer.capacity() < requiredOffsetBufferSize) { + ArrowBuf newOffsetBuffer = allocateOffsetBuffer(requiredOffsetBufferSize); + if (offsetBuffer.capacity() > 0) { + newOffsetBuffer.setBytes(0, offsetBuffer, 0, offsetBuffer.capacity()); + } + offsetBuffer.getReferenceManager().release(); + offsetBuffer = newOffsetBuffer; + } + offsetBuffer.writerIndex(requiredOffsetBufferSize); } /** Same as {@link #allocateNewSafe()}. */ diff --git a/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index df42d04e60..22c93b0cbe 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -3940,4 +3940,42 @@ public void testVectorLoadUnloadOnNonVariadicVectors() { } } } + + @Test + public void testEmptyVarCharOffsetBuffer() { + // Validates that offset buffer has at least OFFSET_WIDTH bytes (for offset[0]=0) + // even when valueCount is 0, per Arrow specification. + try (VarCharVector vector = newVarCharVector("varchar", allocator)) { + vector.allocateNew(); + vector.setValueCount(0); + + List buffers = vector.getFieldBuffers(); + // buffers: [validity, offset, data] + assertTrue( + buffers.get(1).readableBytes() >= BaseVariableWidthVector.OFFSET_WIDTH, + "Offset buffer should have at least " + + BaseVariableWidthVector.OFFSET_WIDTH + + " bytes for offset[0]"); + assertEquals(0, vector.getOffsetBuffer().getInt(0)); + } + } + + @Test + public void testEmptyLargeVarCharOffsetBuffer() { + // Validates that offset buffer has at least OFFSET_WIDTH bytes (for offset[0]=0) + // even when valueCount is 0, per Arrow specification. + try (LargeVarCharVector vector = new LargeVarCharVector("largevarchar", allocator)) { + vector.allocateNew(); + vector.setValueCount(0); + + List buffers = vector.getFieldBuffers(); + // buffers: [validity, offset, data] + assertTrue( + buffers.get(1).readableBytes() >= BaseLargeVariableWidthVector.OFFSET_WIDTH, + "Offset buffer should have at least " + + BaseLargeVariableWidthVector.OFFSET_WIDTH + + " bytes for offset[0]"); + assertEquals(0, vector.getOffsetBuffer().getLong(0)); + } + } } From b410fb26d01c7cefc4c6e3443a53562164001371 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Thu, 12 Mar 2026 11:58:50 +0100 Subject: [PATCH 316/408] MINOR: Bump version to 19.0.0 (#1066) --- adapter/avro/pom.xml | 2 +- adapter/jdbc/pom.xml | 2 +- adapter/orc/pom.xml | 2 +- algorithm/pom.xml | 2 +- arrow-variant/pom.xml | 2 +- bom/pom.xml | 4 ++-- c/pom.xml | 2 +- compression/pom.xml | 2 +- dataset/pom.xml | 2 +- flight/flight-core/pom.xml | 2 +- flight/flight-integration-tests/pom.xml | 2 +- flight/flight-sql-jdbc-core/pom.xml | 2 +- flight/flight-sql-jdbc-driver/pom.xml | 2 +- flight/flight-sql/pom.xml | 2 +- flight/pom.xml | 2 +- format/pom.xml | 2 +- gandiva/pom.xml | 2 +- memory/memory-core/pom.xml | 2 +- memory/memory-netty-buffer-patch/pom.xml | 2 +- memory/memory-netty/pom.xml | 2 +- memory/memory-unsafe/pom.xml | 2 +- memory/pom.xml | 2 +- performance/pom.xml | 2 +- pom.xml | 6 +++--- tools/pom.xml | 2 +- vector/pom.xml | 2 +- 26 files changed, 29 insertions(+), 29 deletions(-) diff --git a/adapter/avro/pom.xml b/adapter/avro/pom.xml index 827d19f2a2..18c48a0e8f 100644 --- a/adapter/avro/pom.xml +++ b/adapter/avro/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 ../../pom.xml diff --git a/adapter/jdbc/pom.xml b/adapter/jdbc/pom.xml index 2f621d7a05..a0819d7aef 100644 --- a/adapter/jdbc/pom.xml +++ b/adapter/jdbc/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 ../../pom.xml diff --git a/adapter/orc/pom.xml b/adapter/orc/pom.xml index c96ab36119..fbb72d6c19 100644 --- a/adapter/orc/pom.xml +++ b/adapter/orc/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 ../../pom.xml diff --git a/algorithm/pom.xml b/algorithm/pom.xml index 898c2605b6..116be9aebf 100644 --- a/algorithm/pom.xml +++ b/algorithm/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 arrow-algorithm Arrow Algorithms diff --git a/arrow-variant/pom.xml b/arrow-variant/pom.xml index 3a842178a4..fea724824b 100644 --- a/arrow-variant/pom.xml +++ b/arrow-variant/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 arrow-variant Arrow Variant diff --git a/bom/pom.xml b/bom/pom.xml index 0de43a1217..6a4f741fca 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -29,7 +29,7 @@ under the License. org.apache.arrow arrow-bom - 19.0.0-SNAPSHOT + 19.0.0 pom Arrow Bill of Materials @@ -68,7 +68,7 @@ under the License. scm:git:https://github.com/apache/arrow-java.git scm:git:https://github.com/apache/arrow-java.git - main + v19.0.0 https://github.com/apache/arrow-java/tree/${project.scm.tag} diff --git a/c/pom.xml b/c/pom.xml index c90b6dc0ef..b0a7ffe41d 100644 --- a/c/pom.xml +++ b/c/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 arrow-c-data diff --git a/compression/pom.xml b/compression/pom.xml index 29f8b41788..92144addc4 100644 --- a/compression/pom.xml +++ b/compression/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 arrow-compression Arrow Compression diff --git a/dataset/pom.xml b/dataset/pom.xml index 1852c6eddc..df5620c641 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 arrow-dataset diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml index f1d58a0cad..fbed544a1b 100644 --- a/flight/flight-core/pom.xml +++ b/flight/flight-core/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 19.0.0-SNAPSHOT + 19.0.0 flight-core diff --git a/flight/flight-integration-tests/pom.xml b/flight/flight-integration-tests/pom.xml index f0f10ada43..5ee7c6fc14 100644 --- a/flight/flight-integration-tests/pom.xml +++ b/flight/flight-integration-tests/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 19.0.0-SNAPSHOT + 19.0.0 flight-integration-tests diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index da00baf32a..d6fa11688d 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 19.0.0-SNAPSHOT + 19.0.0 flight-sql-jdbc-core diff --git a/flight/flight-sql-jdbc-driver/pom.xml b/flight/flight-sql-jdbc-driver/pom.xml index 559c42597d..e6f23bcb08 100644 --- a/flight/flight-sql-jdbc-driver/pom.xml +++ b/flight/flight-sql-jdbc-driver/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 19.0.0-SNAPSHOT + 19.0.0 flight-sql-jdbc-driver diff --git a/flight/flight-sql/pom.xml b/flight/flight-sql/pom.xml index a5954819c3..a58b76acda 100644 --- a/flight/flight-sql/pom.xml +++ b/flight/flight-sql/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 19.0.0-SNAPSHOT + 19.0.0 flight-sql diff --git a/flight/pom.xml b/flight/pom.xml index 2fc3e89ef8..a5a40a834a 100644 --- a/flight/pom.xml +++ b/flight/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 arrow-flight diff --git a/format/pom.xml b/format/pom.xml index d3578b63d2..c09fad32fb 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 arrow-format diff --git a/gandiva/pom.xml b/gandiva/pom.xml index 5367bfdedf..d26edeb9d6 100644 --- a/gandiva/pom.xml +++ b/gandiva/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 org.apache.arrow.gandiva diff --git a/memory/memory-core/pom.xml b/memory/memory-core/pom.xml index 72ee69d60a..586fddae1a 100644 --- a/memory/memory-core/pom.xml +++ b/memory/memory-core/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 19.0.0-SNAPSHOT + 19.0.0 arrow-memory-core diff --git a/memory/memory-netty-buffer-patch/pom.xml b/memory/memory-netty-buffer-patch/pom.xml index 07dc7d2403..cb38efa345 100644 --- a/memory/memory-netty-buffer-patch/pom.xml +++ b/memory/memory-netty-buffer-patch/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 19.0.0-SNAPSHOT + 19.0.0 arrow-memory-netty-buffer-patch diff --git a/memory/memory-netty/pom.xml b/memory/memory-netty/pom.xml index 6d660da117..f33eb95e44 100644 --- a/memory/memory-netty/pom.xml +++ b/memory/memory-netty/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 19.0.0-SNAPSHOT + 19.0.0 arrow-memory-netty diff --git a/memory/memory-unsafe/pom.xml b/memory/memory-unsafe/pom.xml index 92dc0c9fe5..d941fee645 100644 --- a/memory/memory-unsafe/pom.xml +++ b/memory/memory-unsafe/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 19.0.0-SNAPSHOT + 19.0.0 arrow-memory-unsafe diff --git a/memory/pom.xml b/memory/pom.xml index bc34c26050..af953ccd21 100644 --- a/memory/pom.xml +++ b/memory/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 arrow-memory pom diff --git a/performance/pom.xml b/performance/pom.xml index 3f18188e3a..685f433f05 100644 --- a/performance/pom.xml +++ b/performance/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 arrow-performance jar diff --git a/pom.xml b/pom.xml index 19625617b1..0b0aa58232 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 pom Apache Arrow Java Root POM @@ -82,7 +82,7 @@ under the License. scm:git:https://github.com/apache/arrow-java.git scm:git:https://github.com/apache/arrow-java.git - main + v19.0.0 https://github.com/apache/arrow-java/tree/${project.scm.tag} @@ -92,7 +92,7 @@ under the License. - 1695310533 + 1773307790 ${project.build.directory}/generated-sources 1.9.0 5.12.2 diff --git a/tools/pom.xml b/tools/pom.xml index d43adb1fdf..9e557bab76 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 arrow-tools Arrow Tools diff --git a/vector/pom.xml b/vector/pom.xml index f46bd0e7b4..bc64cbc76b 100644 --- a/vector/pom.xml +++ b/vector/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 arrow-vector Arrow Vectors From 77127ef54272b578075c574071f24ee8c1bba22e Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 16 Mar 2026 16:17:40 +0900 Subject: [PATCH 317/408] GH-1077: Add missing `export GH_TOKEN` to release scripts --- dev/release/bump_version.sh | 1 + dev/release/release.sh | 1 + dev/release/release_rc.sh | 1 + 3 files changed, 3 insertions(+) diff --git a/dev/release/bump_version.sh b/dev/release/bump_version.sh index 458e930f98..68cafb99bd 100755 --- a/dev/release/bump_version.sh +++ b/dev/release/bump_version.sh @@ -37,6 +37,7 @@ if [ ! -f "${SOURCE_DIR}/.env" ]; then exit 1 fi . "${SOURCE_DIR}/.env" +export GH_TOKEN cd "${SOURCE_TOP_DIR}" diff --git a/dev/release/release.sh b/dev/release/release.sh index f08a618c4f..d1db7ad05a 100755 --- a/dev/release/release.sh +++ b/dev/release/release.sh @@ -36,6 +36,7 @@ if [ ! -f "${SOURCE_DIR}/.env" ]; then exit 1 fi . "${SOURCE_DIR}/.env" +export GH_TOKEN git_origin_url="$(git remote get-url origin)" repository="${git_origin_url#*github.com?}" diff --git a/dev/release/release_rc.sh b/dev/release/release_rc.sh index ff77718b8d..0920edbe35 100755 --- a/dev/release/release_rc.sh +++ b/dev/release/release_rc.sh @@ -42,6 +42,7 @@ if [ ! -f "${SOURCE_DIR}/.env" ]; then exit 1 fi . "${SOURCE_DIR}/.env" +export GH_TOKEN cd "${SOURCE_TOP_DIR}" From d5e132b96500646ea4021d3b032d00a65233b6b5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 08:45:29 +0100 Subject: [PATCH 318/408] MINOR: [CI] Bump actions/download-artifact from 8.0.0 to 8.0.1 (#1068) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 8.0.0 to 8.0.1.
Release notes

Sourced from actions/download-artifact's releases.

v8.0.1

What's Changed

Full Changelog: https://github.com/actions/download-artifact/compare/v8...v8.0.1

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/download-artifact&package-manager=github_actions&previous-version=8.0.0&new-version=8.0.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index a202777143..4973b1afb5 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -101,7 +101,7 @@ jobs: packages: write steps: - name: Download source archive - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: release-source - name: Extract source archive @@ -168,7 +168,7 @@ jobs: MACOSX_DEPLOYMENT_TARGET: "14.0" steps: - name: Download source archive - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: release-source - name: Extract source archive @@ -296,7 +296,7 @@ jobs: arch: "x86_64" steps: - name: Download source archive - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: release-source - name: Extract source archive @@ -369,7 +369,7 @@ jobs: - jni-windows steps: - name: Download artifacts - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: artifacts - name: Decompress artifacts @@ -450,11 +450,11 @@ jobs: with: cache: 'pip' - name: Download source archive - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: release-source - name: Download Javadocs - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: reference - name: Extract source archive @@ -519,7 +519,7 @@ jobs: cp ../.asf.yaml ./ git add .nojekyll .asf.yaml - name: Download - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: release-html - name: Extract @@ -555,7 +555,7 @@ jobs: - ubuntu-latest steps: - name: Download release artifacts - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: release-* - name: Verify @@ -589,7 +589,7 @@ jobs: contents: write steps: - name: Download release artifacts - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: release-* path: artifacts From fa781dcc3c42ab98214743e3aef1eb8fd5297209 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 08:45:54 +0100 Subject: [PATCH 319/408] MINOR: Bump com.squareup.okio:okio-jvm from 3.16.4 to 3.17.0 (#1075) Bumps [com.squareup.okio:okio-jvm](https://github.com/square/okio) from 3.16.4 to 3.17.0.
Changelog

Sourced from com.squareup.okio:okio-jvm's changelog.

Version 3.17.0

2026-03-11

  • New: Adjust down the Kotlin stdlib dependency to [Kotlin 2.1.21][kotlin_2_1_21]. Okio is built with an up-to-date Kotlin compiler (2.2.21), but depends on an older kotlin-stdlib. We're doing this so you can update Okio and Kotlin independently.

  • Fix: Return the correct timestamp in FileMetadata.createdAtMillis on Kotlin/Native on UNIX platforms. We were incorrectly using the POSIX ctime (change time) instead of the birthtime. With this fix Okio now prefers statx() over stat() on native platforms. This API first appeared in Linux in 4.11 (2017) and Android in API 30 (2020).

Commits
  • 80a5023 Prepare for release 3.17.0.
  • 65c0c26 Switch to FileMetadata to use statx instead of stat on Linux and Apple platfo...
  • b11f17b Remove Kotlin/JS IR default parameter workarounds. (#1786)
  • b35f473 Update Gradle to v9.4.0 (#1785)
  • cbcee31 Update actions/upload-artifact action to v7 (#1783)
  • fc7aecb Update dependency com.android.tools.build:gradle to v9.0.1 (#1781)
  • 79aa267 Drop isWasm() early return workaround for KT-60212. (#1777)
  • 45459dc Fix result of an 'errnoToIOException' call is not thrown. inside `PosixFileSy...
  • 9fbab0f Decode env variables in WASI tests (#1773)
  • 50abe89 Stop using AssertJ (#1771)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.squareup.okio:okio-jvm&package-manager=maven&previous-version=3.16.4&new-version=3.17.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index d6fa11688d..4b8122fa4d 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -141,7 +141,7 @@ under the License. com.squareup.okio okio-jvm - 3.16.4 + 3.17.0 test From 47cd032939f15edbfc7c87fbcb02d585f82e5948 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 08:46:10 +0100 Subject: [PATCH 320/408] MINOR: Bump org.codehaus.mojo:properties-maven-plugin from 1.2.1 to 1.3.0 (#1072) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.codehaus.mojo:properties-maven-plugin](https://github.com/mojohaus/properties-maven-plugin) from 1.2.1 to 1.3.0.
Release notes

Sourced from org.codehaus.mojo:properties-maven-plugin's releases.

1.3.0

🚀 New features and improvements

👻 Maintenance

🔧 Build

📦 Dependency updates

Commits
  • 91a2ade [maven-release-plugin] prepare release properties-maven-plugin-1.3.0
  • bf56d32 Bump org.codehaus.mojo:mojo-parent from 94 to 95
  • 80e20be Bump org.codehaus.mojo:mojo-parent from 93 to 94
  • e8ae7a5 Bump org.yaml:snakeyaml from 2.4 to 2.5
  • bb39c3d Bump org.codehaus.mojo:mojo-parent from 92 to 93
  • b41267b Bump org.codehaus.mojo:mojo-parent from 91 to 92
  • bb548c5 Bump org.codehaus.mojo:mojo-parent from 87 to 91
  • 3ffa9cb Use Sisu plugin
  • 7adbe3b Require Maven 3.6.3
  • 407342f Bump org.yaml:snakeyaml from 2.3 to 2.4
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.codehaus.mojo:properties-maven-plugin&package-manager=maven&previous-version=1.2.1&new-version=1.3.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0b0aa58232..a414dcd229 100644 --- a/pom.xml +++ b/pom.xml @@ -502,7 +502,7 @@ under the License. org.codehaus.mojo properties-maven-plugin - 1.2.1 + 1.3.0 org.codehaus.mojo From 4242d587b25a74e944f2a01e79d52dcd401d5cf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Mon, 16 Mar 2026 09:37:09 +0100 Subject: [PATCH 321/408] MINOR: Bump version to 20.0.0-SNAPSHOT (#1076) --- adapter/avro/pom.xml | 2 +- adapter/jdbc/pom.xml | 2 +- adapter/orc/pom.xml | 2 +- algorithm/pom.xml | 2 +- arrow-variant/pom.xml | 2 +- bom/pom.xml | 4 ++-- c/pom.xml | 2 +- compression/pom.xml | 2 +- dataset/pom.xml | 2 +- flight/flight-core/pom.xml | 2 +- flight/flight-integration-tests/pom.xml | 2 +- flight/flight-sql-jdbc-core/pom.xml | 2 +- flight/flight-sql-jdbc-driver/pom.xml | 2 +- flight/flight-sql/pom.xml | 2 +- flight/pom.xml | 2 +- format/pom.xml | 2 +- gandiva/pom.xml | 2 +- memory/memory-core/pom.xml | 2 +- memory/memory-netty-buffer-patch/pom.xml | 2 +- memory/memory-netty/pom.xml | 2 +- memory/memory-unsafe/pom.xml | 2 +- memory/pom.xml | 2 +- performance/pom.xml | 2 +- pom.xml | 6 +++--- tools/pom.xml | 2 +- vector/pom.xml | 2 +- 26 files changed, 29 insertions(+), 29 deletions(-) diff --git a/adapter/avro/pom.xml b/adapter/avro/pom.xml index 18c48a0e8f..4f7f90d7a9 100644 --- a/adapter/avro/pom.xml +++ b/adapter/avro/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT ../../pom.xml diff --git a/adapter/jdbc/pom.xml b/adapter/jdbc/pom.xml index a0819d7aef..9ff44593ff 100644 --- a/adapter/jdbc/pom.xml +++ b/adapter/jdbc/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT ../../pom.xml diff --git a/adapter/orc/pom.xml b/adapter/orc/pom.xml index fbb72d6c19..50a9b3a603 100644 --- a/adapter/orc/pom.xml +++ b/adapter/orc/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT ../../pom.xml diff --git a/algorithm/pom.xml b/algorithm/pom.xml index 116be9aebf..24adcefa6f 100644 --- a/algorithm/pom.xml +++ b/algorithm/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT arrow-algorithm Arrow Algorithms diff --git a/arrow-variant/pom.xml b/arrow-variant/pom.xml index fea724824b..e578626dd4 100644 --- a/arrow-variant/pom.xml +++ b/arrow-variant/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT arrow-variant Arrow Variant diff --git a/bom/pom.xml b/bom/pom.xml index 6a4f741fca..f9200a7e8d 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -29,7 +29,7 @@ under the License. org.apache.arrow arrow-bom - 19.0.0 + 20.0.0-SNAPSHOT pom Arrow Bill of Materials @@ -68,7 +68,7 @@ under the License. scm:git:https://github.com/apache/arrow-java.git scm:git:https://github.com/apache/arrow-java.git - v19.0.0 + main https://github.com/apache/arrow-java/tree/${project.scm.tag} diff --git a/c/pom.xml b/c/pom.xml index b0a7ffe41d..27b6619c4c 100644 --- a/c/pom.xml +++ b/c/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT arrow-c-data diff --git a/compression/pom.xml b/compression/pom.xml index 92144addc4..945738d2b8 100644 --- a/compression/pom.xml +++ b/compression/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT arrow-compression Arrow Compression diff --git a/dataset/pom.xml b/dataset/pom.xml index df5620c641..7a0210ce95 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT arrow-dataset diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml index fbed544a1b..92490dd67b 100644 --- a/flight/flight-core/pom.xml +++ b/flight/flight-core/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 19.0.0 + 20.0.0-SNAPSHOT flight-core diff --git a/flight/flight-integration-tests/pom.xml b/flight/flight-integration-tests/pom.xml index 5ee7c6fc14..ec81162e59 100644 --- a/flight/flight-integration-tests/pom.xml +++ b/flight/flight-integration-tests/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 19.0.0 + 20.0.0-SNAPSHOT flight-integration-tests diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 4b8122fa4d..53c630ab40 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 19.0.0 + 20.0.0-SNAPSHOT flight-sql-jdbc-core diff --git a/flight/flight-sql-jdbc-driver/pom.xml b/flight/flight-sql-jdbc-driver/pom.xml index e6f23bcb08..55de7221ec 100644 --- a/flight/flight-sql-jdbc-driver/pom.xml +++ b/flight/flight-sql-jdbc-driver/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 19.0.0 + 20.0.0-SNAPSHOT flight-sql-jdbc-driver diff --git a/flight/flight-sql/pom.xml b/flight/flight-sql/pom.xml index a58b76acda..b7c8931391 100644 --- a/flight/flight-sql/pom.xml +++ b/flight/flight-sql/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 19.0.0 + 20.0.0-SNAPSHOT flight-sql diff --git a/flight/pom.xml b/flight/pom.xml index a5a40a834a..30f75fa27e 100644 --- a/flight/pom.xml +++ b/flight/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT arrow-flight diff --git a/format/pom.xml b/format/pom.xml index c09fad32fb..8c2f2d3387 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT arrow-format diff --git a/gandiva/pom.xml b/gandiva/pom.xml index d26edeb9d6..190bf016ce 100644 --- a/gandiva/pom.xml +++ b/gandiva/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT org.apache.arrow.gandiva diff --git a/memory/memory-core/pom.xml b/memory/memory-core/pom.xml index 586fddae1a..1c7b6f8834 100644 --- a/memory/memory-core/pom.xml +++ b/memory/memory-core/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 19.0.0 + 20.0.0-SNAPSHOT arrow-memory-core diff --git a/memory/memory-netty-buffer-patch/pom.xml b/memory/memory-netty-buffer-patch/pom.xml index cb38efa345..039b2aa04a 100644 --- a/memory/memory-netty-buffer-patch/pom.xml +++ b/memory/memory-netty-buffer-patch/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 19.0.0 + 20.0.0-SNAPSHOT arrow-memory-netty-buffer-patch diff --git a/memory/memory-netty/pom.xml b/memory/memory-netty/pom.xml index f33eb95e44..4218910980 100644 --- a/memory/memory-netty/pom.xml +++ b/memory/memory-netty/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 19.0.0 + 20.0.0-SNAPSHOT arrow-memory-netty diff --git a/memory/memory-unsafe/pom.xml b/memory/memory-unsafe/pom.xml index d941fee645..3fafb42802 100644 --- a/memory/memory-unsafe/pom.xml +++ b/memory/memory-unsafe/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 19.0.0 + 20.0.0-SNAPSHOT arrow-memory-unsafe diff --git a/memory/pom.xml b/memory/pom.xml index af953ccd21..4ea3d1f9ca 100644 --- a/memory/pom.xml +++ b/memory/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT arrow-memory pom diff --git a/performance/pom.xml b/performance/pom.xml index 685f433f05..96ea5291ad 100644 --- a/performance/pom.xml +++ b/performance/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT arrow-performance jar diff --git a/pom.xml b/pom.xml index a414dcd229..43149bc957 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT pom Apache Arrow Java Root POM @@ -82,7 +82,7 @@ under the License. scm:git:https://github.com/apache/arrow-java.git scm:git:https://github.com/apache/arrow-java.git - v19.0.0 + main https://github.com/apache/arrow-java/tree/${project.scm.tag} @@ -92,7 +92,7 @@ under the License. - 1773307790 + 1773644827 ${project.build.directory}/generated-sources 1.9.0 5.12.2 diff --git a/tools/pom.xml b/tools/pom.xml index 9e557bab76..64634b9abe 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT arrow-tools Arrow Tools diff --git a/vector/pom.xml b/vector/pom.xml index bc64cbc76b..4d247961c9 100644 --- a/vector/pom.xml +++ b/vector/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT arrow-vector Arrow Vectors From 74a7996d40226c659c3b0ae03c0ae75a4964f282 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 09:55:27 +0100 Subject: [PATCH 322/408] MINOR: Bump com.nimbusds:oauth2-oidc-sdk from 11.20.1 to 11.34 (#1074) Bumps [com.nimbusds:oauth2-oidc-sdk](https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions) from 11.20.1 to 11.34.
Changelog

Sourced from com.nimbusds:oauth2-oidc-sdk's changelog.

version 1.0 (2012-05-29) * First official release with authorisation endpoint, token endpoint, check ID endpoint and UserInfo endpoint support. * JSON Web Tokens (JWTs) support through the Nimbus-JWT library. * Language Tags (RFC 5646) support through the Nimbus-LangTag library. * JSON support through the JSON Smart library.

version 2.0 (2013-05-13) * Intermediary development release with Maven build, published to Maven Central.

version 2.1 (2013-06-06) * Updates the APIs to OpenID Connect Messages draft 20, OpenID Connect Standard draft 21, OpenID Connect Discovery draft 17 and OpenID Connect Registration draft 19. * Major refactoring of the APIs for greater simplicity. * Adds JUnit tests.

version 2.2 (2013-06-18) * Refactors dynamic OpenID Connect client registration. * Adds partial support of the OAuth 2.0 Dynamic Client Registration Protocol (draft-ietf-oauth-dyn-reg-12). * Optimises parsing of request parameters consisting of one or more tokens (scope, response type, etc).

version 2.3 (2013-06-19) * Renames OAuth 2.0 dynamic client registration package. * Adds ClientInformation.getClientMetadata() method. * Adds OIDCClientInformation class.

version 2.4 (2013-06-20) * Adds static OIDCClientInformation.parse(JSONObject) method.

version 2.5 (2013-06-22) * Adds support OAuth 2.0 dynamic client update. * Adds OpenID Connect dynamic client registration classes.

version 2.6 (2013-06-25) * Enforces order of preference of ACR values in OpenID Connect client metadata, as required by the specification. * Documentation and performance improvements.

version 2.7 (2013-06-26) * Switches Identifier generation to java.security.SecureRandom.

version 2.8 (2013-06-30) * Fixes serialisation and assignment bugs in ClientMetadata. * Switches Secret generation to java.security.SecureRandom.

version 2.9 (2013-09-17)

... (truncated)

Commits
  • 668f6d8 The ParseException message thrown by Prompt.Type.parse must not include parse...
  • 75cde87 Updates test sample X.509 cert chain resource
  • a7a9623 [maven-release-plugin] prepare release 11.30.2
  • e03c9bb [maven-release-plugin] prepare for next development iteration
  • 6f11e30 Expands AMR test coverage
  • afba676 Adds static AMR.parseList(Collection<String>) method
  • 4b700b3 [maven-release-plugin] prepare release 11.31
  • b214cfa [maven-release-plugin] prepare for next development iteration
  • 28628f9 The DPoPCommonVerifier must instantiate the DPoPProofClaimsSetVerifier with t...
  • 4df4d53 The DPoPCommonVerifier must instantiate the DPoPProofClaimsSetVerifier with t...
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.nimbusds:oauth2-oidc-sdk&package-manager=maven&previous-version=11.20.1&new-version=11.34)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 53c630ab40..ffeff12462 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -182,7 +182,7 @@ under the License. com.nimbusds oauth2-oidc-sdk - 11.20.1 + 11.34 From 97e491399382215e248b3ba33e7f0d7d9650da5e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 09:59:10 +0100 Subject: [PATCH 323/408] MINOR: Bump com.github.luben:zstd-jni from 1.5.7-6 to 1.5.7-7 (#1073) Bumps [com.github.luben:zstd-jni](https://github.com/luben/zstd-jni) from 1.5.7-6 to 1.5.7-7.
Commits
  • 73bfa27 Bump version to v1.5.7-7
  • 322f6bc Update GH actions
  • d18bc0a Use latest MacOS runners
  • 1051112 Fix typo in ZstdDictCompress.java
  • 2d94b35 address
  • 7c2e3ff Avoid SetLongField call when GetPrimitiveArrayCritical return NULL
  • d82feda fix: ZstdInputStream decompression failure when underlying stream returns 0 t...
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.github.luben:zstd-jni&package-manager=maven&previous-version=1.5.7-6&new-version=1.5.7-7)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- compression/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compression/pom.xml b/compression/pom.xml index 945738d2b8..9014b6913a 100644 --- a/compression/pom.xml +++ b/compression/pom.xml @@ -55,7 +55,7 @@ under the License. com.github.luben zstd-jni - 1.5.7-6 + 1.5.7-7 From 394d12ef455fad83f76ec4b3a79ac6b5c6fe1139 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 10:04:36 +0100 Subject: [PATCH 324/408] MINOR: Bump com.fasterxml.jackson:jackson-bom from 2.21.0 to 2.21.1 (#1071) Bumps [com.fasterxml.jackson:jackson-bom](https://github.com/FasterXML/jackson-bom) from 2.21.0 to 2.21.1.
Commits
  • 08a5a9a [maven-release-plugin] prepare release jackson-bom-2.21.1
  • 5b03376 Prep for 2.21.1 release
  • 1d78778 Merge branch '2.20' into 2.21
  • cd46b24 Post-release dep version bump
  • 17179ff [maven-release-plugin] prepare for next development iteration
  • 2a26844 [maven-release-plugin] prepare release jackson-bom-2.20.2
  • 6adf11b Prep for 2.20.2 release
  • 441df8a Post-release version bump
  • a1b4814 [maven-release-plugin] prepare for next development iteration
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.fasterxml.jackson:jackson-bom&package-manager=maven&previous-version=2.21.0&new-version=2.21.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 43149bc957..eb7768780f 100644 --- a/pom.xml +++ b/pom.xml @@ -101,7 +101,7 @@ under the License. 4.2.9.Final 1.79.0 4.33.4 - 2.21.0 + 2.21.1 3.4.3 25.2.10 1.12.1 From 27382cd92baf29762dba2454d7e9e911c58821e3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 10:07:18 +0100 Subject: [PATCH 325/408] MINOR: Bump checker.framework.version from 3.53.1 to 3.54.0 (#1070) Bumps `checker.framework.version` from 3.53.1 to 3.54.0. Updates `org.checkerframework:checker-qual` from 3.53.1 to 3.54.0
Release notes

Sourced from org.checkerframework:checker-qual's releases.

Checker Framework 3.54.0

Version 3.54.0 (2026-03-02)

User-visible changes

Command-line arguments:

  • Added -AinferOutputDirectory.
  • Removed long-deprecated -Alint=forbidnonnullarraycomponents.

New command-line argument -Aonelinemsg puts error messages on a single line. This is useful when using a tool that only shows the first line of the error.

The command-line argument -Anomsgtext surrounds the error key with brackets instead of parenthesis. This matches Java error messages.

Implementation details

In AnnotatedTypeFactory, canonicalAnnotation() returns a non-null value.

In AnnotationClassLoader:

  • Renamed hasWellDefinedTargetMetaAnnotation() to isTypeQualifierAnnotation(). The method now returns true for annotations bearing @InvisibleQualifier or @SubtypeOf, in addition to the existing @Target(TYPE_USE) check.

In TestDiagnostic:

  • Renamed field message to key.
  • Added new nullable field message for the full message without the key.

Removed classes and methods that have been deprecated for more than two years.

Closed issues

#6874, #7471, #7475, #7486.

Changelog

Sourced from org.checkerframework:checker-qual's changelog.

Version 3.54.0 (2026-03-02)

User-visible changes

Command-line arguments:

  • Added -AinferOutputDirectory.
  • Removed long-deprecated -Alint=forbidnonnullarraycomponents.

New command-line argument -Aonelinemsg puts error messages on a single line. This is useful when using a tool that only shows the first line of the error.

The command-line argument -Anomsgtext surrounds the error key with brackets instead of parenthesis. This matches Java error messages.

Implementation details

In AnnotatedTypeFactory, canonicalAnnotation() returns a non-null value.

In AnnotationClassLoader:

  • Renamed hasWellDefinedTargetMetaAnnotation() to isTypeQualifierAnnotation(). The method now returns true for annotations bearing @InvisibleQualifier or @SubtypeOf, in addition to the existing @Target(TYPE_USE) check.

In TestDiagnostic:

  • Renamed field message to key.
  • Added new nullable field message for the full message without the key.

Removed classes and methods that have been deprecated for more than two years.

Closed issues

#6874, #7471, #7475, #7486.

Commits
  • a6eff70 new release 3.54.0
  • fd34700 Prep for release.
  • edb6e7a Print error key in brackets (#7525)
  • a79b1de Show details of the error message in test failures (#7513)
  • a5ecc22 Clone the JDK using the same fork and branch as CF (#7491)
  • 2770c52 Update cimg/base Docker tag to v2026.03
  • bba6bc9 Update plugin com-gradleup-shadow to v9.3.2
  • 3a6d4d4 Update error-prone monorepo to v2.48.0
  • 70aa5f3 Update plugin net-ltgt-errorprone to v5.1.0
  • 0dbd3e7 Prepare for javac AST changes
  • Additional commits viewable in compare view

Updates `org.checkerframework:checker` from 3.53.1 to 3.54.0
Release notes

Sourced from org.checkerframework:checker's releases.

Checker Framework 3.54.0

Version 3.54.0 (2026-03-02)

User-visible changes

Command-line arguments:

  • Added -AinferOutputDirectory.
  • Removed long-deprecated -Alint=forbidnonnullarraycomponents.

New command-line argument -Aonelinemsg puts error messages on a single line. This is useful when using a tool that only shows the first line of the error.

The command-line argument -Anomsgtext surrounds the error key with brackets instead of parenthesis. This matches Java error messages.

Implementation details

In AnnotatedTypeFactory, canonicalAnnotation() returns a non-null value.

In AnnotationClassLoader:

  • Renamed hasWellDefinedTargetMetaAnnotation() to isTypeQualifierAnnotation(). The method now returns true for annotations bearing @InvisibleQualifier or @SubtypeOf, in addition to the existing @Target(TYPE_USE) check.

In TestDiagnostic:

  • Renamed field message to key.
  • Added new nullable field message for the full message without the key.

Removed classes and methods that have been deprecated for more than two years.

Closed issues

#6874, #7471, #7475, #7486.

Changelog

Sourced from org.checkerframework:checker's changelog.

Version 3.54.0 (2026-03-02)

User-visible changes

Command-line arguments:

  • Added -AinferOutputDirectory.
  • Removed long-deprecated -Alint=forbidnonnullarraycomponents.

New command-line argument -Aonelinemsg puts error messages on a single line. This is useful when using a tool that only shows the first line of the error.

The command-line argument -Anomsgtext surrounds the error key with brackets instead of parenthesis. This matches Java error messages.

Implementation details

In AnnotatedTypeFactory, canonicalAnnotation() returns a non-null value.

In AnnotationClassLoader:

  • Renamed hasWellDefinedTargetMetaAnnotation() to isTypeQualifierAnnotation(). The method now returns true for annotations bearing @InvisibleQualifier or @SubtypeOf, in addition to the existing @Target(TYPE_USE) check.

In TestDiagnostic:

  • Renamed field message to key.
  • Added new nullable field message for the full message without the key.

Removed classes and methods that have been deprecated for more than two years.

Closed issues

#6874, #7471, #7475, #7486.

Commits
  • a6eff70 new release 3.54.0
  • fd34700 Prep for release.
  • edb6e7a Print error key in brackets (#7525)
  • a79b1de Show details of the error message in test failures (#7513)
  • a5ecc22 Clone the JDK using the same fork and branch as CF (#7491)
  • 2770c52 Update cimg/base Docker tag to v2026.03
  • bba6bc9 Update plugin com-gradleup-shadow to v9.3.2
  • 3a6d4d4 Update error-prone monorepo to v2.48.0
  • 70aa5f3 Update plugin net-ltgt-errorprone to v5.1.0
  • 0dbd3e7 Prepare for javac AST changes
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index eb7768780f..27aa503b99 100644 --- a/pom.xml +++ b/pom.xml @@ -112,7 +112,7 @@ under the License. 10.23.0 true 2.42.0 - 3.53.1 + 3.54.0 1.5.32 none -Xdoclint:none From 899e883428ee067da4542393e7ecc958cfb5d90c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 13:14:24 +0100 Subject: [PATCH 326/408] MINOR: Bump com.google.protobuf:protobuf-bom from 4.33.4 to 4.34.1 (#1086) Bumps [com.google.protobuf:protobuf-bom](https://github.com/protocolbuffers/protobuf) from 4.33.4 to 4.34.1.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.protobuf:protobuf-bom&package-manager=maven&previous-version=4.33.4&new-version=4.34.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 27aa503b99..61ecddf867 100644 --- a/pom.xml +++ b/pom.xml @@ -100,7 +100,7 @@ under the License. 33.4.8-jre 4.2.9.Final 1.79.0 - 4.33.4 + 4.34.1 2.21.1 3.4.3 25.2.10 From a8b238061bfdab433d25ee223ce11d697a572efd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 13:15:02 +0100 Subject: [PATCH 327/408] MINOR: Bump org.mockito:mockito-bom from 5.21.0 to 5.23.0 (#1087) Bumps [org.mockito:mockito-bom](https://github.com/mockito/mockito) from 5.21.0 to 5.23.0.
Release notes

Sourced from org.mockito:mockito-bom's releases.

v5.23.0

NOTE: Breaking change for Android

The mockito-android artifact has a breaking change: tests now require a device or emulator based on API 28+ (Android P). This is to enable new support for mocking Kotlin classes. See #3788 for more details.

Changelog generated by Shipkit Changelog Gradle Plugin

5.23.0

v5.22.0

Changelog generated by Shipkit Changelog Gradle Plugin

5.22.0

Commits
  • a231205 Fix StackOverflowError with AbstractList after using mockSingleton (#3790)
  • f6a91a6 Replace mockito-android mock maker implementation with dexmaker-mockito-inlin...
  • aa2298a fix: make spotless happy
  • a6729d6 chore: update BDDMockito with jspecify annotation
  • bb83c92 chore: move jspecify as a compile only dependency
  • 47a4695 chore: add jspecify with minimal change. Fixes #3503
  • 25f1395 Add core API to enable Kotlin singleton mocking (#3762)
  • ef9ee55 Avoids mocking private static methods, as well as package-private static meth...
  • d16fcfc Bump graalvm/setup-graalvm from 1.4.4 to 1.4.5 (#3780)
  • 27eb8a3 Clarify RETURNS_MOCKS behavior with sealed abstract enums (Java 15+) (#3773)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.mockito:mockito-bom&package-manager=maven&previous-version=5.21.0&new-version=5.23.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 61ecddf867..c96656a67d 100644 --- a/pom.xml +++ b/pom.xml @@ -106,7 +106,7 @@ under the License. 25.2.10 1.12.1 1.17.0 - 5.21.0 + 5.23.0 2 10.23.0 From 1348b8dc8302daebe7bb76df9f3c13bdcf2766e9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 13:16:04 +0100 Subject: [PATCH 328/408] MINOR: Bump io.netty:netty-bom from 4.2.9.Final to 4.2.10.Final (#1085) Bumps [io.netty:netty-bom](https://github.com/netty/netty) from 4.2.9.Final to 4.2.10.Final.
Commits
  • 4cc9873 [maven-release-plugin] prepare release netty-4.2.10.Final
  • 54b8663 Remove unnecessary allocations and abstractions in HttpContentCompressor (#16...
  • 961f427 Update to netty-tcnative 2.0.75.Final (#16227)
  • 3007ba9 Use recommanded finalize chain pattern when override finalize() method (#16212)
  • b918042 Update some dependencies (#16198) (#16215)
  • 874c995 Reduce allocations on DefaultHeaders::containsValue (#15843)
  • e0fe794 Remove unnecessary null check in WebSocketServerExtensionHandler (#16201)
  • 1b0636b Move default compression options into static variable in HttpContentCompresso...
  • 85a3a0e codec-http2: move the accessors from Http2Headers to DefaultHttp2Headers (#16...
  • f44a88d Improve chunk picking for the large-size buddy allocator (#16179)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=io.netty:netty-bom&package-manager=maven&previous-version=4.2.9.Final&new-version=4.2.10.Final)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c96656a67d..c0a01c6050 100644 --- a/pom.xml +++ b/pom.xml @@ -98,7 +98,7 @@ under the License. 5.12.2 2.0.17 33.4.8-jre - 4.2.9.Final + 4.2.10.Final 1.79.0 4.34.1 2.21.1 From d9d6112b3e4037d72aa44e978bc72c2f7706c52c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 18:46:51 +0100 Subject: [PATCH 329/408] MINOR: Bump com.google.guava:guava-bom from 33.4.8-jre to 33.5.0-jre (#1083) Bumps [com.google.guava:guava-bom](https://github.com/google/guava) from 33.4.8-jre to 33.5.0-jre.
Release notes

Sourced from com.google.guava:guava-bom's releases.

33.5.0

Maven

<dependency>
  <groupId>com.google.guava</groupId>
  <artifactId>guava</artifactId>
  <version>33.5.0-jre</version>
  <!-- or, for Android: -->
  <version>33.5.0-android</version>
</dependency>

Jar files

Guava requires one runtime dependency, which you can download here:

Javadoc

JDiff

Changelog

  • Restored the Automatic-Module-Name to guava-android. (It, unlike, guava-jre, is not a proper module.) (7a04a8a955)
  • For users of guava-gwt: Google has moved off GWT internally. We plan to continue to release guava-gwt for users of GWT and J2CL, but the artifact is no longer tested for GWT-specific issues, and we have limited resources to fix any unexpected issues that might arise. While we do not anticipate any specific problems, we can't guarantee how long support will continue.
  • Increased our Android minSdkVersion to 23 (Marshmallow). This follows the minimum of Google's foundational Android libraries, and we expect it to have no practical impact on users. (5c23347cc1)
  • Listed the JSpecify annotations as an optional dependency in our OSGi metadata. (2dfd572981)
  • cache: Improved the handling of exceptions from compute functions in Cache.asMap(). (We do still recommend using Caffeine rather than com.google.common.cache.) (087f2c4a80)
  • collect: Improved Iterators.mergeSorted() to preserve stability for equal elements. (4dc93be9a8)
  • math: Added saturatedAbs methods to IntMath and LongMath. (ed0e518f20)
  • net: Added image/avif to MediaType. (53344caba6)
  • testing: Made CollectorTester available to Android users. (294c251079)
  • util.concurrent: Added Striped.custom. (1586eb271d)
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.guava:guava-bom&package-manager=maven&previous-version=33.4.8-jre&new-version=33.5.0-jre)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c0a01c6050..200371e265 100644 --- a/pom.xml +++ b/pom.xml @@ -97,7 +97,7 @@ under the License. 1.9.0 5.12.2 2.0.17 - 33.4.8-jre + 33.5.0-jre 4.2.10.Final 1.79.0 4.34.1 From d5497915c604582e66613c96599b32a812fda4bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 18:47:12 +0100 Subject: [PATCH 330/408] MINOR: Bump com.fasterxml.jackson:jackson-bom from 2.21.1 to 2.21.2 (#1082) Bumps [com.fasterxml.jackson:jackson-bom](https://github.com/FasterXML/jackson-bom) from 2.21.1 to 2.21.2.
Commits
  • 10e12a5 [maven-release-plugin] prepare release jackson-bom-2.21.2
  • d754903 Prep for 2.21.2 release
  • 63e1b3b Post-release dep version bump
  • 716ab0d [maven-release-plugin] prepare for next development iteration
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.fasterxml.jackson:jackson-bom&package-manager=maven&previous-version=2.21.1&new-version=2.21.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 200371e265..fe5716a5f1 100644 --- a/pom.xml +++ b/pom.xml @@ -101,7 +101,7 @@ under the License. 4.2.10.Final 1.79.0 4.34.1 - 2.21.1 + 2.21.2 3.4.3 25.2.10 1.12.1 From 01affd741864c493cb76b9357fb32ead7f977672 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Tue, 24 Mar 2026 07:49:08 +0100 Subject: [PATCH 331/408] GH-1078: Upgrade minimum JDK version from 11 to 17 (#1079) Update compiler source/target/release to 17 across build config (pom.xml, bom), CI workflows, Docker images, Brewfile, and documentation. Replace deprecated boxed-type constructors with valueOf() in HolderReaderImpl codegen template to fix -Werror under release=17. ## What's Changed JDK 11 would not be supported in some cases. **This contains breaking changes.** Closes #1078. --- .env | 2 +- .github/workflows/rc.yml | 4 ++-- .github/workflows/test.yml | 8 ++++---- Brewfile | 2 +- bom/pom.xml | 8 ++++---- ci/docker/conda-jni.dockerfile | 2 +- ci/docker/vcpkg-jni.dockerfile | 2 +- compose.yaml | 4 ++-- docs/source/cdata.rst | 8 ++++---- docs/source/developers/building.rst | 16 ++++++++-------- docs/source/flight_sql_jdbc_driver.rst | 2 +- docs/source/install.rst | 4 ++-- docs/source/jdbc.rst | 2 +- docs/source/memory.rst | 2 +- pom.xml | 10 +++++----- .../main/codegen/templates/HolderReaderImpl.java | 4 ++-- 16 files changed, 40 insertions(+), 40 deletions(-) diff --git a/.env b/.env index a7783537d0..2bd7255476 100644 --- a/.env +++ b/.env @@ -47,7 +47,7 @@ ARROW_REPO=ghcr.io/apache/arrow-dev ULIMIT_CORE=-1 # Default versions for various dependencies -JDK=11 +JDK=17 MAVEN=3.9.9 # Versions for various dependencies used to build artifacts diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 4973b1afb5..9c75b8c807 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -273,7 +273,7 @@ jobs: run: | set -e # make brew Java available to CMake - export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home + export JAVA_HOME=$(brew --prefix openjdk@17)/libexec/openjdk.jdk/Contents/Home ci/scripts/jni_macos_build.sh . arrow build jni - name: Compress into single artifact to keep directory structure run: tar -cvzf jni-macos-${{ matrix.platform.arch }}.tar.gz jni/ @@ -317,7 +317,7 @@ jobs: - name: Set up Java uses: actions/setup-java@v5 with: - java-version: '11' + java-version: '17' distribution: 'temurin' - name: Download Timezone Database shell: bash diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2602592799..8c437a056d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,7 +45,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17, 21, 23] + jdk: [17, 21, 23] maven: [3.9.9] image: [ubuntu, conda-jni-cdata] include: @@ -88,10 +88,10 @@ jobs: matrix: include: - arch: AMD64 - jdk: 11 + jdk: 17 macos: 15-intel - arch: AArch64 - jdk: 11 + jdk: 17 macos: latest steps: - name: Set up Java @@ -123,7 +123,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11] + jdk: [17] steps: - name: Set up Java uses: actions/setup-java@v5 diff --git a/Brewfile b/Brewfile index af6bd65615..2c47a38af5 100644 --- a/Brewfile +++ b/Brewfile @@ -15,5 +15,5 @@ # specific language governing permissions and limitations # under the License. -brew "openjdk@11" +brew "openjdk@17" brew "sccache" diff --git a/bom/pom.xml b/bom/pom.xml index f9200a7e8d..e4ccee02db 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -80,10 +80,10 @@ under the License. - 11 - 11 - 11 - 11 + 17 + 17 + 17 + 17 diff --git a/ci/docker/conda-jni.dockerfile b/ci/docker/conda-jni.dockerfile index e14db73688..3f31b74052 100644 --- a/ci/docker/conda-jni.dockerfile +++ b/ci/docker/conda-jni.dockerfile @@ -17,7 +17,7 @@ FROM ghcr.io/mamba-org/micromamba:ubuntu24.04 -ARG jdk=11 +ARG jdk=17 ARG maven=3.9.9 RUN micromamba install -y \ diff --git a/ci/docker/vcpkg-jni.dockerfile b/ci/docker/vcpkg-jni.dockerfile index d6bd322a39..f2f5d0d45a 100644 --- a/ci/docker/vcpkg-jni.dockerfile +++ b/ci/docker/vcpkg-jni.dockerfile @@ -20,7 +20,7 @@ FROM ${base} # Install Java # We need Java for JNI headers, but we don't invoke Maven in this build. -ARG java=11 +ARG java=17 RUN dnf install -y java-$java-openjdk-devel && dnf clean all # For ci/scripts/{cpp,java}_*.sh diff --git a/compose.yaml b/compose.yaml index f5082a22aa..fb290b22fe 100644 --- a/compose.yaml +++ b/compose.yaml @@ -40,7 +40,7 @@ services: # docker compose run ubuntu # Parameters: # MAVEN: 3.9.9 - # JDK: 11, 17, 21 + # JDK: 17, 21 image: ${ARCH}/maven:${MAVEN}-eclipse-temurin-${JDK} volumes: - .:/arrow-java:delegated @@ -60,7 +60,7 @@ services: # docker compose run conda-jni-cdata # Parameters: # MAVEN: 3.9.9 - # JDK: 11, 17, 21 + # JDK: 17, 21 image: ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration build: context: . diff --git a/docs/source/cdata.rst b/docs/source/cdata.rst index 9643d88df3..7b2924d259 100644 --- a/docs/source/cdata.rst +++ b/docs/source/cdata.rst @@ -101,8 +101,8 @@ without writing JNI bindings ourselves. 1.0-SNAPSHOT - 8 - 8 + 17 + 17 9.0.0 @@ -237,8 +237,8 @@ For this example, we will build a JAR with all dependencies bundled. cpptojava 1.0-SNAPSHOT - 8 - 8 + 17 + 17 9.0.0 diff --git a/docs/source/developers/building.rst b/docs/source/developers/building.rst index f9ef7daea8..b682957714 100644 --- a/docs/source/developers/building.rst +++ b/docs/source/developers/building.rst @@ -32,7 +32,7 @@ Arrow Java uses the `Maven `_ build system. Building requires: -* JDK 11+ +* JDK 17+ * Maven 3+ .. note:: @@ -345,7 +345,7 @@ configuration file usually located under ``${HOME}/.m2`` with the following snip jdk - 21 + 21 temurin @@ -383,11 +383,11 @@ Arrow repository, and update the following settings: right click the directory, and select Mark Directory as > Generated Sources Root. There is no need to mark other generated sources directories, as only the ``vector`` module generates sources. -* For JDK 11, due to an `IntelliJ bug - `__, you must go into +* Due to an `IntelliJ bug + `__, you may need to go into Settings > Build, Execution, Deployment > Compiler > Java Compiler and disable "Use '--release' option for cross-compilation (Java 9 and later)". Otherwise - you will get an error like "package sun.misc does not exist". + you may get an error like "package sun.misc does not exist". * You may want to disable error-prone entirely if it gives spurious warnings (disable both error-prone profiles in the Maven tool window and "Reload All Maven Projects"). @@ -397,7 +397,7 @@ Arrow repository, and update the following settings: * To enable debugging JNI-based modules like ``dataset``, activate specific profiles in the Maven tab under "Profiles". Ensure the profiles ``arrow-c-data``, ``arrow-jni``, ``generate-libs-cdata-all-os``, - ``generate-libs-jni-macos-linux``, and ``jdk11+`` are enabled, so that the + ``generate-libs-jni-macos-linux``, and ``jdk17+`` are enabled, so that the IDE can build them and enable debugging. You may not need to update all of these settings if you build/test with the @@ -478,8 +478,8 @@ Installing Manually .. code-block:: xml - 8 - 8 + 17 + 17 9.0.0.dev501 diff --git a/docs/source/flight_sql_jdbc_driver.rst b/docs/source/flight_sql_jdbc_driver.rst index 4deb726b33..6d40434a22 100644 --- a/docs/source/flight_sql_jdbc_driver.rst +++ b/docs/source/flight_sql_jdbc_driver.rst @@ -27,7 +27,7 @@ Flight SQL. Installation and Requirements ============================= -The driver is compatible with JDK 11+. Note that the following JVM +The driver is compatible with JDK 17+. Note that the following JVM parameter is required: .. code-block:: shell diff --git a/docs/source/install.rst b/docs/source/install.rst index b2b1c7163f..e0b34515ef 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -27,8 +27,8 @@ Java modules are regularly built and tested on macOS and Linux distributions. Java Compatibility ================== -Java modules are compatible with JDK 11 and above. Currently, JDK versions -11, 17, 21, and latest are tested in CI. +Java modules are compatible with JDK 17 and above. Currently, JDK versions +17, 21, and latest are tested in CI. Note that some JDK internals must be exposed by adding ``--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED`` to the ``java`` command: diff --git a/docs/source/jdbc.rst b/docs/source/jdbc.rst index 2f57c34bf8..e054127faa 100644 --- a/docs/source/jdbc.rst +++ b/docs/source/jdbc.rst @@ -276,7 +276,7 @@ mapping, with additional support for the UUID extension type noted below. JDBC value, because a JDBC Timestamp is in UTC, and we have no timezone information. In this case, the default binder will call `setTimestamp(int, Timestamp) - `_, + `_, which will lead to the driver using the "default timezone" (that of the Java VM). * \(3) For the Flight SQL JDBC driver, the Arrow UUID extension type diff --git a/docs/source/memory.rst b/docs/source/memory.rst index 4a71ed846a..5b9148223a 100644 --- a/docs/source/memory.rst +++ b/docs/source/memory.rst @@ -341,7 +341,7 @@ How this works: .. _`newChildAllocator`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/RootAllocator.html#newChildAllocator-java.lang.String-org.apache.arrow.memory.AllocationListener-long-long- .. _`Netty`: https://netty.io/wiki/ .. _`sun.misc.unsafe`: https://web.archive.org/web/20210929024401/http://www.docjar.com/html/api/sun/misc/Unsafe.java.html -.. _`Direct Memory`: https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/nio/ByteBuffer.html +.. _`Direct Memory`: https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/nio/ByteBuffer.html .. _`ReferenceManager`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/ReferenceManager.html .. _`ReferenceManager.release`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/ReferenceManager.html#release-- .. _`ReferenceManager.retain`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/ReferenceManager.html#retain-- diff --git a/pom.xml b/pom.xml index fe5716a5f1..863169b47b 100644 --- a/pom.xml +++ b/pom.xml @@ -119,10 +119,10 @@ under the License. --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED - 11 - 11 - 11 - 11 + 17 + 17 + 17 + 17

... (truncated)

Commits
  • c8eb81b in preparation for a release
  • 7ff5ee5 Merge pull request #4260 from JackPGreen/update-maven-central-url
  • da4c337 Merge pull request #4271 from IrisesD/master
  • d053544 feat: allow CATALOG in CREATE SCHEMA and DROP SCHEMA (#4277)
  • a448d91 Merge pull request #4273 from naive924/feat/compactThreads
  • 6672123 fix: change log
  • 858e74a fix: MVStore.compact: run map copy in parallel by default (¼ cores, override ...
  • bce0ec1 feat: parallel map copy option for MVStore.compact()
  • d8a6cc3 Fix command syntax in help.csv
  • c45413c Merge pull request #4266 from andreitokar/issue-4208-2
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.h2database:h2&package-manager=maven&previous-version=2.3.232&new-version=2.4.240)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- adapter/jdbc/pom.xml | 2 +- performance/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/adapter/jdbc/pom.xml b/adapter/jdbc/pom.xml index 9ff44593ff..a8ac19721d 100644 --- a/adapter/jdbc/pom.xml +++ b/adapter/jdbc/pom.xml @@ -59,7 +59,7 @@ under the License. com.h2database h2 - 2.3.232 + 2.4.240 test diff --git a/performance/pom.xml b/performance/pom.xml index 96ea5291ad..413d5ba1e0 100644 --- a/performance/pom.xml +++ b/performance/pom.xml @@ -75,7 +75,7 @@ under the License. com.h2database h2 - 2.3.232 + 2.4.240 runtime From 168a969147ea77a6fe8f12fa15c6bc25b1e671f2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 10:21:57 +0200 Subject: [PATCH 333/408] MINOR: Bump io.netty:netty-bom from 4.2.10.Final to 4.2.12.Final (#1091) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [io.netty:netty-bom](https://github.com/netty/netty) from 4.2.10.Final to 4.2.12.Final.
Release notes

Sourced from io.netty:netty-bom's releases.

netty-4.2.12.Final

What's Changed

Full Changelog: https://github.com/netty/netty/compare/netty-4.2.11.Final...netty-4.2.12.Final

netty-4.2.11.Final

Security

What's Changed

... (truncated)

Commits
  • 67ce541 [maven-release-plugin] prepare release netty-4.2.12.Final
  • 7074624 Revert "Eliminate redundant bounds checks in CompositeByteBuf accessors" (#16...
  • c3b0a43 [maven-release-plugin] prepare for next development iteration
  • c94a818 [maven-release-plugin] prepare release netty-4.2.11.Final
  • 3b76df1 Merge commit from fork
  • aae944a Auto-port 4.2: Limit the number of Continuation frames per HTTP2 Headers (#16...
  • 6001499 Eliminate redundant bounds checks in CompositeByteBuf accessors (#16525)
  • a7fbb6f JdkZlibDecoder: accumulate decompressed output before firing channelRead (#16...
  • 7937553 Enforce io.netty.maxDirectMemory accounting on all Java versions (#16489)
  • 893ea2e Allocate less in QueryStringDecoder.addParam for typical use case (#16527)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=io.netty:netty-bom&package-manager=maven&previous-version=4.2.10.Final&new-version=4.2.12.Final)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 863169b47b..61b5b3f0b2 100644 --- a/pom.xml +++ b/pom.xml @@ -98,7 +98,7 @@ under the License. 5.12.2 2.0.17 33.5.0-jre - 4.2.10.Final + 4.2.12.Final 1.79.0 4.34.1 2.21.2 From 89fa995eac2aa84e0195341bff601dfce66b942e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 10:22:13 +0200 Subject: [PATCH 334/408] MINOR: Bump com.nimbusds:oauth2-oidc-sdk from 11.34 to 11.37 (#1096) Bumps [com.nimbusds:oauth2-oidc-sdk](https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions) from 11.34 to 11.37.
Changelog

Sourced from com.nimbusds:oauth2-oidc-sdk's changelog.

version 1.0 (2012-05-29) * First official release with authorisation endpoint, token endpoint, check ID endpoint and UserInfo endpoint support. * JSON Web Tokens (JWTs) support through the Nimbus-JWT library. * Language Tags (RFC 5646) support through the Nimbus-LangTag library. * JSON support through the JSON Smart library.

version 2.0 (2013-05-13) * Intermediary development release with Maven build, published to Maven Central.

version 2.1 (2013-06-06) * Updates the APIs to OpenID Connect Messages draft 20, OpenID Connect Standard draft 21, OpenID Connect Discovery draft 17 and OpenID Connect Registration draft 19. * Major refactoring of the APIs for greater simplicity. * Adds JUnit tests.

version 2.2 (2013-06-18) * Refactors dynamic OpenID Connect client registration. * Adds partial support of the OAuth 2.0 Dynamic Client Registration Protocol (draft-ietf-oauth-dyn-reg-12). * Optimises parsing of request parameters consisting of one or more tokens (scope, response type, etc).

version 2.3 (2013-06-19) * Renames OAuth 2.0 dynamic client registration package. * Adds ClientInformation.getClientMetadata() method. * Adds OIDCClientInformation class.

version 2.4 (2013-06-20) * Adds static OIDCClientInformation.parse(JSONObject) method.

version 2.5 (2013-06-22) * Adds support OAuth 2.0 dynamic client update. * Adds OpenID Connect dynamic client registration classes.

version 2.6 (2013-06-25) * Enforces order of preference of ACR values in OpenID Connect client metadata, as required by the specification. * Documentation and performance improvements.

version 2.7 (2013-06-26) * Switches Identifier generation to java.security.SecureRandom.

version 2.8 (2013-06-30) * Fixes serialisation and assignment bugs in ClientMetadata. * Switches Secret generation to java.security.SecureRandom.

version 2.9 (2013-09-17)

... (truncated)

Commits
  • d98de1a [maven-release-plugin] prepare for next development iteration
  • 2ea716f Shortens InvalidClientException messages
  • ed5773c TokenRevocationRequest receives custom form parameters support
  • e133559 Updates tests for shortened InvalidClientException messages
  • fe43e1f [maven-release-plugin] prepare release 11.35
  • 73224c9 [maven-release-plugin] prepare for next development iteration
  • f3f7286 Adds static JSONObjectUtils.getNonNegativeLong methods
  • d6899e0 Cleans up JSONObjectUtils.getEnum(net.minidev.json.JSONObject, java.lang.Stri...
  • 9b05d23 Adds non-negative checks when parsing Date instances from Unix timestamps (is...
  • 592d8f4 Adds "acr" and "auth_time" parameter (RFC 9470) support to TokenIntrospection...
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.nimbusds:oauth2-oidc-sdk&package-manager=maven&previous-version=11.34&new-version=11.37)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index ffeff12462..483c019cdc 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -182,7 +182,7 @@ under the License. com.nimbusds oauth2-oidc-sdk - 11.34 + 11.37 From 4297733dae36ca83234a239b25068db775a09743 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 10:30:50 +0200 Subject: [PATCH 335/408] MINOR: Bump com.gradle:develocity-maven-extension from 2.3.4 to 2.4.0 (#1095) Bumps com.gradle:develocity-maven-extension from 2.3.4 to 2.4.0. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.gradle:develocity-maven-extension&package-manager=maven&previous-version=2.3.4&new-version=2.4.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .mvn/extensions.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index 4585435b49..f52dfafb4a 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -23,7 +23,7 @@ com.gradle develocity-maven-extension - 2.3.4 + 2.4.0 com.gradle From 447372ce4caa5a53387018861e3ea9d1f7be795c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 10:57:29 +0200 Subject: [PATCH 336/408] MINOR: Bump com.google.api.grpc:proto-google-common-protos from 2.66.0 to 2.67.0 (#1092) Bumps [com.google.api.grpc:proto-google-common-protos](https://github.com/googleapis/sdk-platform-java) from 2.66.0 to 2.67.0.
Release notes

Sourced from com.google.api.grpc:proto-google-common-protos's releases.

v2.67.0

2.67.0 (2026-02-18)

Features

  • observability: introduce minimal tracing implementation (#4105) (e4e5e89)

Dependencies

v2.66.1

2.66.1 (2026-02-04)

Documentation

  • [common-protos] update reference documentation for SelectionInput.DROPDOWN to include dynamic data sources and autosuggestion (9960262)
Changelog

Sourced from com.google.api.grpc:proto-google-common-protos's changelog.

2.67.0 (2026-02-18)

Features

  • observability: introduce minimal tracing implementation (#4105) (e4e5e89)

Dependencies

2.66.1 (2026-02-04)

Documentation

  • [common-protos] update reference documentation for SelectionInput.DROPDOWN to include dynamic data sources and autosuggestion (9960262)
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.api.grpc:proto-google-common-protos&package-manager=maven&previous-version=2.66.0&new-version=2.67.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml index 92490dd67b..5c6c3378c1 100644 --- a/flight/flight-core/pom.xml +++ b/flight/flight-core/pom.xml @@ -134,7 +134,7 @@ under the License. com.google.api.grpc proto-google-common-protos - 2.66.0 + 2.67.0 test From 0703021c8315a55651b6f7446da8cc781f039f72 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 14:56:30 +0200 Subject: [PATCH 337/408] MINOR: Bump org.apache:apache from 33 to 37 (#1033) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.apache:apache](https://github.com/apache/maven-apache-parent) from 33 to 37.
Release notes

Sourced from org.apache:apache's releases.

Apache Parent POM version 37

🚀 New features and improvements

Apache Parent POM version 36

:boom: Breaking changes

🚀 New features and improvements

📝 Documentation updates

👻 Maintenance

📦 Dependency updates

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache:apache&package-manager=maven&previous-version=33&new-version=37)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- bom/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index e4ccee02db..267dcef73c 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache apache - 33 + 37 diff --git a/pom.xml b/pom.xml index 61b5b3f0b2..25d49a0704 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache apache - 35 + 37 org.apache.arrow From e54681b80e84573f4cb6e7fdcb77b4247768965f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 16:20:40 +0200 Subject: [PATCH 338/408] MINOR: Bump com.diffplug.spotless:spotless-maven-plugin from 2.44.4 to 3.4.0 (#1088) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [com.diffplug.spotless:spotless-maven-plugin](https://github.com/diffplug/spotless) from 2.44.4 to 3.4.0.
Release notes

Sourced from com.diffplug.spotless:spotless-maven-plugin's releases.

Maven Plugin v3.4.0

Added

  • Add tableTest format type for standalone .table files. (#2880)

Changes

  • Bump default tabletest-formatter version 1.0.1 -> 1.1.1, now works with Java 17+. (#2880)

Lib v3.3.1

Fixed

  • GitPrePushHookInstaller didn't work on windows, now fixed. (#2562)

Lib v3.3.0

Added

  • Allow specifying path to Biome JSON config file directly in biome step. Requires biome 2.x. (#2548)
  • GitPrePushHookInstaller, a reusable library component for installing a Git pre-push hook that runs formatter checks. (#2553)
  • Allow setting Eclipse XML config from a string, not only from files (#2361)

Changed

  • Bump default gson version to latest 2.11.0 -> 2.13.1. (#2414)
  • Bump default jackson version to latest 2.18.1 -> 2.19.2. (#2558)
  • Bump default gherkin-utils version to latest 9.0.0 -> 9.2.0. (#2408)
  • Bump default cleanthat version to latest 2.22 -> 2.23. (#2556)

Maven Plugin v3.3.0

Added

  • Add tabletest-formatter support for Java and Kotlin. (#2860)

Fixed

  • Fix the ability to specify a wildcard version (*) for external formatter executables, which did not work. (#2848)
  • [fix] ConcurrentModificationException in expandWildcardImports (#2830)

Maven Plugin v3.2.1

Fixed

  • removeSemicolons() should not be applied to multiline strings in groovy #2780 (#2792)

Lib v3.2.0

Added

  • Support for idea (#2020, #2535)
  • Add support for removing wildcard imports via removeWildcardImports step. (#2517)
  • scalafmt: enforce version consistency between the version configured in Spotless and the version declared in Scalafmt config file (#2460)

Fixed

  • SortPom disable expandEmptyElements, to avoid empty body warnings. (#2520)
  • Fix biome formatter for new major release 2.x of biome (#2537)
  • Make sure npm-based formatters use the correct node_modules directory when running in parallel. (#2542)

Changed

  • Bump internal dependencies for npm-based formatters (#2542)

Maven Plugin v3.2.0

Added

  • Add the ability to specify a wildcard version (*) for external formatter executables. (#2757)

Changes

  • Dramatic (~100x) performance improvement when using git ratchetFrom. (#2805)

Fixed

... (truncated)

Commits
  • 708a1b0 Published maven/3.4.0
  • 1cc0163 Published gradle/8.4.0
  • a4cd808 Published lib/4.5.0
  • 9066bf6 Add links to the changelog.
  • db8dc1c Fix for illegal mutation issue with predeclareDeps (#2892)
  • 0eb98a9 chore: Updated gradle plugin change
  • 3f7f12e chore: Removes check for predeclare as it's not needed anymore
  • 55c0c5c fix: IsolatedProjectTest.predeclaredIsUnsupported() is now actually supported...
  • 47489af fix: avoid IllegalMutationException when root project uses predeclareDeps() w...
  • 4010e8b test: Introduce a test harnessing predeclared deps
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.diffplug.spotless:spotless-maven-plugin&package-manager=maven&previous-version=2.44.4&new-version=3.4.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: JB Onofré Co-authored-by: JB Onofré --- bom/pom.xml | 8 ++++---- flight/flight-integration-tests/pom.xml | 2 +- flight/flight-sql-jdbc-driver/pom.xml | 2 +- memory/memory-core/pom.xml | 4 ++-- performance/pom.xml | 6 +++--- pom.xml | 14 +++++++------- tools/pom.xml | 2 +- 7 files changed, 19 insertions(+), 19 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index 267dcef73c..d97cc291c2 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -24,7 +24,7 @@ under the License. org.apache apache 37 - + org.apache.arrow @@ -78,7 +78,7 @@ under the License. - + 17 17 @@ -208,7 +208,7 @@ under the License. com.diffplug.spotless spotless-maven-plugin - 2.44.4 + 3.4.0 org.codehaus.mojo @@ -235,7 +235,7 @@ under the License. ${maven.multiModuleProjectDirectory}/dev/license/asf-xml.license (<configuration|<project) - + diff --git a/flight/flight-integration-tests/pom.xml b/flight/flight-integration-tests/pom.xml index ec81162e59..f6ae8e16a5 100644 --- a/flight/flight-integration-tests/pom.xml +++ b/flight/flight-integration-tests/pom.xml @@ -101,7 +101,7 @@ under the License. - + META-INF/LICENSE.txt src/shade/LICENSE.txt diff --git a/flight/flight-sql-jdbc-driver/pom.xml b/flight/flight-sql-jdbc-driver/pom.xml index 55de7221ec..ff5763702b 100644 --- a/flight/flight-sql-jdbc-driver/pom.xml +++ b/flight/flight-sql-jdbc-driver/pom.xml @@ -138,7 +138,7 @@ under the License. - + META-INF/LICENSE.txt src/shade/LICENSE.txt diff --git a/memory/memory-core/pom.xml b/memory/memory-core/pom.xml index 1c7b6f8834..825b3dae4b 100644 --- a/memory/memory-core/pom.xml +++ b/memory/memory-core/pom.xml @@ -100,8 +100,8 @@ under the License. test - - + + **/TestOpens.java diff --git a/performance/pom.xml b/performance/pom.xml index 413d5ba1e0..d994bff45b 100644 --- a/performance/pom.xml +++ b/performance/pom.xml @@ -35,10 +35,10 @@ under the License. true .* 1 - + 5 5 - + jmh-result.json json @@ -143,7 +143,7 @@ under the License. java -classpath - + org.openjdk.jmh.Main ${benchmark.filter} -f diff --git a/pom.xml b/pom.xml index 25d49a0704..06cb916b45 100644 --- a/pom.xml +++ b/pom.xml @@ -107,7 +107,7 @@ under the License. 1.12.1 1.17.0 5.23.0 - + 2 10.23.0 true @@ -374,7 +374,7 @@ under the License. - + @@ -387,7 +387,7 @@ under the License. - + @@ -400,7 +400,7 @@ under the License. - + @@ -413,7 +413,7 @@ under the License. - + @@ -492,7 +492,7 @@ under the License. com.diffplug.spotless spotless-maven-plugin - 2.44.4 + 3.4.0 org.codehaus.mojo @@ -734,7 +734,7 @@ under the License. ${maven.multiModuleProjectDirectory}/dev/license/asf-xml.license (<configuration|<project) - + diff --git a/tools/pom.xml b/tools/pom.xml index 64634b9abe..b4d64fd435 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -126,7 +126,7 @@ under the License. - + META-INF/LICENSE.txt src/shade/LICENSE.txt From d8ad7b6765620c4f3f3078ee87c3d2cfb7423945 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2026 17:41:35 +0900 Subject: [PATCH 339/408] MINOR: [CI] Bump docker/login-action from 4.0.0 to 4.1.0 (#1103) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [docker/login-action](https://github.com/docker/login-action) from 4.0.0 to 4.1.0.
Release notes

Sourced from docker/login-action's releases.

v4.1.0

Full Changelog: https://github.com/docker/login-action/compare/v4.0.0...v4.1.0

Commits
  • 4907a6d Merge pull request #930 from docker/dependabot/npm_and_yarn/aws-sdk-dependenc...
  • 1e233e6 chore: update generated content
  • 6c24ead build(deps): bump the aws-sdk-dependencies group with 2 updates
  • ee034d7 Merge pull request #958 from docker/dependabot/npm_and_yarn/lodash-4.18.1
  • 1527209 Merge pull request #937 from docker/dependabot/npm_and_yarn/proxy-agent-depen...
  • d39362a build(deps): bump lodash from 4.17.23 to 4.18.1
  • a6f092b chore: update generated content
  • 60953f0 build(deps): bump the proxy-agent-dependencies group with 2 updates
  • 62c6885 Merge pull request #936 from docker/dependabot/npm_and_yarn/docker/actions-to...
  • 102c0e6 chore: update generated content
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=docker/login-action&package-manager=github_actions&previous-version=4.0.0&new-version=4.1.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 9c75b8c807..8f52d54bd6 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -127,7 +127,7 @@ jobs: with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing - - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: registry: ghcr.io username: ${{ github.actor }} From 13c8b9353b80f553e720ff50fce0cb103de87c55 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2026 14:23:37 +0200 Subject: [PATCH 340/408] MINOR: Bump checker.framework.version from 3.54.0 to 3.55.1 (#1105) Bumps `checker.framework.version` from 3.54.0 to 3.55.1. Updates `org.checkerframework:checker-qual` from 3.54.0 to 3.55.1
Release notes

Sourced from org.checkerframework:checker-qual's releases.

Checker Framework 3.55.1

Version 3.55.1 (2026-04-03)

No user-visible changes.

Checker Framework 3.54.1

Version 3.55.0 (2026-04-02)

User-visible changes

The Checker Framework runs under JDK 26 -- that is, it runs on a version 26 JVM.

Removed deprecated command-line option -AskipDirs; use -AskipFiles.

Implementation details

In AnnotatedTypeMirror:

  • Renamed getEffectiveAnnotation*() to getAnnotation*().
  • Renamed hasEffectiveAnnotation*() to hasAnnotation*().

Removed deprecated method ObjectCreationNode.getConstructor(); use getTypeToInstantiate().

Closed issues

#7079, #7489, #7539.

Changelog

Sourced from org.checkerframework:checker-qual's changelog.

Version 3.55.1 (2026-04-03)

No user-visible changes.

Version 3.55.0 (2026-04-02)

User-visible changes

The Checker Framework runs under JDK 26 -- that is, it runs on a version 26 JVM.

Removed deprecated command-line option -AskipDirs; use -AskipFiles.

Implementation details

In AnnotatedTypeMirror:

  • Renamed getEffectiveAnnotation*() to getAnnotation*().
  • Renamed hasEffectiveAnnotation*() to hasAnnotation*().

Removed deprecated method ObjectCreationNode.getConstructor(); use getTypeToInstantiate().

Closed issues

#7079, #7489, #7539.

Commits

Updates `org.checkerframework:checker` from 3.54.0 to 3.55.1
Release notes

Sourced from org.checkerframework:checker's releases.

Checker Framework 3.55.1

Version 3.55.1 (2026-04-03)

No user-visible changes.

Checker Framework 3.54.1

Version 3.55.0 (2026-04-02)

User-visible changes

The Checker Framework runs under JDK 26 -- that is, it runs on a version 26 JVM.

Removed deprecated command-line option -AskipDirs; use -AskipFiles.

Implementation details

In AnnotatedTypeMirror:

  • Renamed getEffectiveAnnotation*() to getAnnotation*().
  • Renamed hasEffectiveAnnotation*() to hasAnnotation*().

Removed deprecated method ObjectCreationNode.getConstructor(); use getTypeToInstantiate().

Closed issues

#7079, #7489, #7539.

Changelog

Sourced from org.checkerframework:checker's changelog.

Version 3.55.1 (2026-04-03)

No user-visible changes.

Version 3.55.0 (2026-04-02)

User-visible changes

The Checker Framework runs under JDK 26 -- that is, it runs on a version 26 JVM.

Removed deprecated command-line option -AskipDirs; use -AskipFiles.

Implementation details

In AnnotatedTypeMirror:

  • Renamed getEffectiveAnnotation*() to getAnnotation*().
  • Renamed hasEffectiveAnnotation*() to hasAnnotation*().

Removed deprecated method ObjectCreationNode.getConstructor(); use getTypeToInstantiate().

Closed issues

#7079, #7489, #7539.

Commits

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 06cb916b45..363ad7bd7d 100644 --- a/pom.xml +++ b/pom.xml @@ -112,7 +112,7 @@ under the License. 10.23.0 true 2.42.0 - 3.54.0 + 3.55.1 1.5.32 none -Xdoclint:none From d952ed48ef257b90a2085bca574ddd9213b67c91 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2026 14:24:24 +0200 Subject: [PATCH 341/408] MINOR: Bump dep.hadoop.version from 3.4.3 to 3.5.0 (#1104) Bumps `dep.hadoop.version` from 3.4.3 to 3.5.0. Updates `org.apache.hadoop:hadoop-client-runtime` from 3.4.3 to 3.5.0 Updates `org.apache.hadoop:hadoop-client-api` from 3.4.3 to 3.5.0 Updates `org.apache.hadoop:hadoop-common` from 3.4.3 to 3.5.0 Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 363ad7bd7d..5ef8364176 100644 --- a/pom.xml +++ b/pom.xml @@ -102,7 +102,7 @@ under the License. 1.79.0 4.34.1 2.21.2 - 3.4.3 + 3.5.0 25.2.10 1.12.1 1.17.0 From e4f8c9251229846190716cce5a9772292952c761 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 10 Apr 2026 14:31:54 +0900 Subject: [PATCH 342/408] GH-1107: Increase publish job timeout (#1108) ## What's Changed We have many artifacts and need `sleep 1` per upload. So 5min timeout is too short. Closes #1107. --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5ca9cf9b73..a2c5a55544 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,7 +32,7 @@ jobs: publish: name: Publish runs-on: ubuntu-latest - timeout-minutes: 5 + timeout-minutes: 30 steps: - name: Download RC contents run: | From 4ca6017bdd8590a6ad0a0a4154d993ec0c416076 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2026 17:37:40 +0900 Subject: [PATCH 343/408] MINOR: [CI] Bump actions/github-script from 8 to 9 (#1110) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/github-script](https://github.com/actions/github-script) from 8 to 9.
Release notes

Sourced from actions/github-script's releases.

v9.0.0

New features:

  • getOctokit factory function — Available directly in the script context. Create additional authenticated Octokit clients with different tokens for multi-token workflows, GitHub App tokens, and cross-org access. See Creating additional clients with getOctokit for details and examples.
  • Orchestration ID in user-agent — The ACTIONS_ORCHESTRATION_ID environment variable is automatically appended to the user-agent string for request tracing.

Breaking changes:

  • require('@actions/github') no longer works in scripts. The upgrade to @actions/github v9 (ESM-only) means require('@actions/github') will fail at runtime. If you previously used patterns like const { getOctokit } = require('@actions/github') to create secondary clients, use the new injected getOctokit function instead — it's available directly in the script context with no imports needed.
  • getOctokit is now an injected function parameter. Scripts that declare const getOctokit = ... or let getOctokit = ... will get a SyntaxError because JavaScript does not allow const/let redeclaration of function parameters. Use the injected getOctokit directly, or use var getOctokit = ... if you need to redeclare it.
  • If your script accesses other @actions/github internals beyond the standard github/octokit client, you may need to update those references for v9 compatibility.

What's Changed

New Contributors

Full Changelog: https://github.com/actions/github-script/compare/v8.0.0...v9.0.0

Commits
  • 3a2844b Merge pull request #700 from actions/salmanmkc/expose-getoctokit + prepare re...
  • ca10bbd fix: use @​octokit/core/types import for v7 compatibility
  • 86e48e2 merge: incorporate main branch changes
  • c108472 chore: rebuild dist for v9 upgrade and getOctokit factory
  • afff112 Merge pull request #712 from actions/salmanmkc/deployment-false + fix user-ag...
  • ff8117e ci: fix user-agent test to handle orchestration ID
  • 81c6b78 ci: use deployment: false to suppress deployment noise from integration tests
  • 3953caf docs: update README examples from @​v8 to @​v9, add getOctokit docs and v9 brea...
  • c17d55b ci: add getOctokit integration test job
  • a047196 test: add getOctokit integration tests via callAsyncFunction
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/github-script&package-manager=github_actions&previous-version=8&new-version=9)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/comment_bot.yml | 2 +- .github/workflows/dev_pr.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml index b4dbc92dfb..507d6a969c 100644 --- a/.github/workflows/comment_bot.yml +++ b/.github/workflows/comment_bot.yml @@ -30,7 +30,7 @@ jobs: if: github.event.comment.body == 'take' runs-on: ubuntu-latest steps: - - uses: actions/github-script@v8 + - uses: actions/github-script@v9 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: |- diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml index 84740628ee..20946c8185 100644 --- a/.github/workflows/dev_pr.yml +++ b/.github/workflows/dev_pr.yml @@ -50,28 +50,28 @@ jobs: - name: Ensure PR title format id: title-format - uses: actions/github-script@v8 + uses: actions/github-script@v9 with: script: | const scripts = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr.js`); return scripts.check_title_format({core, github, context}); - name: Label PR - uses: actions/github-script@v8 + uses: actions/github-script@v9 with: script: | const scripts = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr.js`); await scripts.apply_labels({core, github, context}); - name: Ensure PR is labeled - uses: actions/github-script@v8 + uses: actions/github-script@v9 with: script: | const scripts = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr.js`); await scripts.check_labels({core, github, context}); - name: Ensure PR is linked to an issue - uses: actions/github-script@v8 + uses: actions/github-script@v9 with: script: | const scripts = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr.js`); From 0d55ba78aeed3e6b28e3d65ced37c360f5e0ed4e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2026 17:37:53 +0900 Subject: [PATCH 344/408] MINOR: [CI] Bump actions/upload-artifact from 7.0.0 to 7.0.1 (#1111) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 7.0.0 to 7.0.1.
Release notes

Sourced from actions/upload-artifact's releases.

v7.0.1

What's Changed

Full Changelog: https://github.com/actions/upload-artifact/compare/v7...v7.0.1

Commits
  • 043fb46 Merge pull request #797 from actions/yacaovsnc/update-dependency
  • 634250c Include changes in typespec/ts-http-runtime 0.3.5
  • e454baa Readme: bump all the example versions to v7 (#796)
  • 74fad66 Update the readme with direct upload details (#795)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/upload-artifact&package-manager=github_actions&previous-version=7.0.0&new-version=7.0.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 8f52d54bd6..2658f1da00 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -71,7 +71,7 @@ jobs: run: | dev/release/run_rat.sh "${TAR_GZ}" - name: Upload source archive - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: release-source path: | @@ -148,7 +148,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf jni-linux-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: jni-linux-${{ matrix.platform.arch }} path: jni-linux-${{ matrix.platform.arch }}.tar.gz @@ -278,7 +278,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf jni-macos-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: jni-macos-${{ matrix.platform.arch }} path: jni-macos-${{ matrix.platform.arch }}.tar.gz @@ -356,7 +356,7 @@ jobs: shell: bash run: tar -cvzf jni-windows-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: jni-windows-${{ matrix.platform.arch }} path: jni-windows-${{ matrix.platform.arch }}.tar.gz @@ -428,12 +428,12 @@ jobs: cp -a target/site/apidocs reference tar -cvzf reference.tar.gz reference - name: Upload binaries - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: release-binaries path: binaries/* - name: Upload docs - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: reference path: reference.tar.gz @@ -471,7 +471,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf html.tar.gz -C docs/build html - name: Upload artifacts - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: release-html path: html.tar.gz From 8d3e8dd2f01f7ddc22cda5aa04cde30eb5f6d308 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Apr 2026 15:18:55 +0200 Subject: [PATCH 345/408] MINOR: Bump com.google.api.grpc:proto-google-common-protos from 2.67.0 to 2.70.0 (#1124) Bumps [com.google.api.grpc:proto-google-common-protos](https://github.com/googleapis/sdk-platform-java) from 2.67.0 to 2.70.0.
Release notes

Sourced from com.google.api.grpc:proto-google-common-protos's releases.

v2.68.0

2.68.0 (2026-03-17)

Features

  • Add client request duration metric. (#4132) (6a76397)
  • Add more attributes to golden signals metrics. (#4135) (59d0624)
  • gax-httpjson: add HttpJsonErrorParser utility (#4137) (a1b7565)
  • generator: add extra allowed modules that will not be removed from the monorepo if they are present (#4124) (774fe6e)
  • o11y: introduce gcp.client.repo and gcp.client.artifact attributes (#4120) (105f644)
  • o11y: Introduce rpc.system.name and rpc.method in gRPC (#4121) (7ab6d2e)
  • o11y: introduce server.port attribute (#4128) (56aa343)

Bug Fixes

  • add null checks for ApiTracerFactory in ClientContext (#4122) (4b3dbe2)
  • Decrease log level for directpath warnings outside GCE (#4139) (c9651e7)
  • gax-grpc: add pick_first fallback to direct path service config (#4143) (b150fe9)
  • Populate method level attributes in metrics recording (#4149) (7b7e6c9)
  • suppress warnings in generated projects for non-idiomatic durations (#4119) (4206e6e)
  • Use ServiceName + MethodName as the regex for Otel (#2543) (b9ae73f)

Documentation

  • hermetic_build: fix config field name in readme (#4130) (a0c8f67)
Changelog

Sourced from com.google.api.grpc:proto-google-common-protos's changelog.

Changelog

2.68.0 (2026-03-17)

Features

  • Add client request duration metric. (#4132) (6a76397)
  • Add more attributes to golden signals metrics. (#4135) (59d0624)
  • gax-httpjson: add HttpJsonErrorParser utility (#4137) (a1b7565)
  • generator: add extra allowed modules that will not be removed from the monorepo if they are present (#4124) (774fe6e)
  • o11y: introduce gcp.client.repo and gcp.client.artifact attributes (#4120) (105f644)
  • o11y: Introduce rpc.system.name and rpc.method in gRPC (#4121) (7ab6d2e)
  • o11y: introduce server.port attribute (#4128) (56aa343)

Bug Fixes

  • add null checks for ApiTracerFactory in ClientContext (#4122) (4b3dbe2)
  • Decrease log level for directpath warnings outside GCE (#4139) (c9651e7)
  • gax-grpc: add pick_first fallback to direct path service config (#4143) (b150fe9)
  • Populate method level attributes in metrics recording (#4149) (7b7e6c9)
  • suppress warnings in generated projects for non-idiomatic durations (#4119) (4206e6e)
  • Use ServiceName + MethodName as the regex for Otel (#2543) (b9ae73f)

Documentation

  • hermetic_build: fix config field name in readme (#4130) (a0c8f67)
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.api.grpc:proto-google-common-protos&package-manager=maven&previous-version=2.67.0&new-version=2.70.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml index 5c6c3378c1..d199f70da4 100644 --- a/flight/flight-core/pom.xml +++ b/flight/flight-core/pom.xml @@ -134,7 +134,7 @@ under the License. com.google.api.grpc proto-google-common-protos - 2.67.0 + 2.70.0 test From 3dca92baf2a806802e437f84bfe06947d4433343 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Apr 2026 15:46:36 +0200 Subject: [PATCH 346/408] MINOR: Bump com.google.guava:guava-bom from 33.5.0-jre to 33.6.0-jre (#1123) Bumps [com.google.guava:guava-bom](https://github.com/google/guava) from 33.5.0-jre to 33.6.0-jre.
Release notes

Sourced from com.google.guava:guava-bom's releases.

33.6.0

Maven

<dependency>
  <groupId>com.google.guava</groupId>
  <artifactId>guava</artifactId>
  <version>33.6.0-jre</version>
  <!-- or, for Android: -->
  <version>33.6.0-android</version>
</dependency>

Jar files

Guava requires one runtime dependency, which you can download here:

Javadoc

JDiff

Changelog

  • Migrated some classes from finalize() to PhantomReference in preparation for the removal of finalization. (786b619dd6, 7c6b17c, aeef90988d)
  • cache: Deprecated CacheBuilder APIs that use TimeUnit in favor of those that use Duration. (73f8b0bb84)
  • collect: Added toImmutableSortedMap collectors that use the natural comparator. (64d70b9f94)
  • collect: Changed ConcurrentHashMultiset, ImmutableMap and TreeMultiset deserialization to avoid mutating final fields. In extremely unlikely scenarios in which an instance of that type contains an object that refers back to that instance, this could lead to a broken instance that throws NullPointerException when used. (8240c7e596, 046468055f)
  • graph: Removed @Beta from all APIs in the package. (dae9566b73)
  • graph: Added support to Graphs.transitiveClosure() for different strategies for adding self-loops. (2e13df25b2)
  • graph: Added an asNetwork() view to Graph and ValueGraph. (909c593c61)
  • hash: Added BloomFilter.serializedSize(). (df9bcc251a)
  • net: Added HttpHeaders.CDN_CACHE_CONTROL. (75331b5030)
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.guava:guava-bom&package-manager=maven&previous-version=33.5.0-jre&new-version=33.6.0-jre)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5ef8364176..a388c90fee 100644 --- a/pom.xml +++ b/pom.xml @@ -97,7 +97,7 @@ under the License. 1.9.0 5.12.2 2.0.17 - 33.5.0-jre + 33.6.0-jre 4.2.12.Final 1.79.0 4.34.1 From 980b5146b5c659e5a06c4592b59cef0f2514ea72 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Apr 2026 15:57:26 +0200 Subject: [PATCH 347/408] MINOR: Bump org.bouncycastle:bcpkix-jdk18on from 1.83 to 1.84 (#1122) Bumps [org.bouncycastle:bcpkix-jdk18on](https://github.com/bcgit/bc-java) from 1.83 to 1.84.
Changelog

Sourced from org.bouncycastle:bcpkix-jdk18on's changelog.

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.bouncycastle:bcpkix-jdk18on&package-manager=maven&previous-version=1.83&new-version=1.84)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 483c019cdc..1f9bc3e00e 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -165,7 +165,7 @@ under the License. org.bouncycastle bcpkix-jdk18on - 1.83 + 1.84 From 09038fbb578d38e98878359557b1e13dcb96b04f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 06:15:04 +0200 Subject: [PATCH 348/408] MINOR: Bump checker.framework.version from 3.55.1 to 4.0.0 (#1113) Bumps `checker.framework.version` from 3.55.1 to 4.0.0. Updates `org.checkerframework:checker-qual` from 3.55.1 to 4.0.0
Release notes

Sourced from org.checkerframework:checker-qual's releases.

Release 4.0.0 of the Checker Framework

Version 4.0.0 (2026-04-07)

User-visible changes

To run the Checker Framework, you need to use a JDK 17 or later version of javac. That is, you need to use JDK 17 or later when compiling your code.

The Checker Framework can type-check any Java project, including projects that compile to Java 8 or 11 bytecodes and run on JRE versions 8 or 11. That is, your code can run under any release of Java, from Java 8 onward.

The type qualifiers and utility libraries in checker-qual.jar and checker-util.jar still use Java 11 bytecode. Thus, they may be used in projects that run under Java 11 or later.

Changes since version 3.0.0

Since version 3.0.0, 91 authors have made over 4500 commits and closed over 600 issues. Thanks to everyone who contributed!

New checkers include:

  • The Index Checker warns about out-of-bounds accesses to arrays and strings.
  • The Initialized Fields Checker warns if a constructor does not initialize a field.
  • The Resource Leak Checker guarantees that every resource is closed rather than leaked. Examples of resources are a channel, executor, ExecutionControl, file, FileLock, Formatter, reader, Scanner, socket, stream, writer, etc.
  • The SQL Quotes Checker helps prevent SQL injection vulnerabilities.

New command-line arguments include:

  • -AskipFiles, -AonlyFiles
  • -AassumeSideEffectFree, -AassumeDeterministic, -AassumePure, -AassumePureGetters
  • -AuseConservativeDefaultsForUncheckedCode
  • -AignoreRawTypeArguments
  • -AwarnRedundantAnnotations
  • -Ainfer=ajava, -AinferOutputDirectory, -AinferOutputOriginal, -AshowWpiFailedInferences
  • -AshowSuppressWarningsStrings, -AwarnUnneededSuppressionsExceptions
  • -AshowPrefixInWarningMessages
  • -AstubNoWarnIfNotFound, -AstubWarnNote, -AmergeStubsWithSource
  • -Aonelinemsg, -AdumpOnErrors, -AexceptionLineSeparator
  • -ApermitMissingJdk, -AparseAllJdk
  • -AslowTypecheckingSeconds
  • -Aversion, -AprintGitProperties
  • You can pass an option to only a particular checker (not all checkers) by using an underscore prefix.

Other improvements include thousands of enhancements and bug fixes -- too many to list here.

Implementation details

All previously-deprecated methods and classes have been removed. If your project builds upon the Checker Framework, we suggest that you upgrade to version 3.55.1, resolve all the deprecation warnings, then upgrade to version 4.0.0.

Changelog

Sourced from org.checkerframework:checker-qual's changelog.

Version 4.0.0 (2026-04-07)

User-visible changes

To run the Checker Framework, you need to use a JDK 17 or later version of javac. That is, you need to use JDK 17 or later when compiling your code.

The Checker Framework can type-check any Java project, including projects that compile to Java 8 or 11 bytecodes and run on JRE versions 8 or 11. That is, your code can run under any release of Java, from Java 8 onward.

The type qualifiers and utility libraries in checker-qual.jar and checker-util.jar still use Java 11 bytecode. Thus, they may be used in projects that run under Java 11 or later.

Changes since version 3.0.0

Since version 3.0.0, 91 authors have made over 4500 commits and closed over 600 issues. Thanks to everyone who contributed!

New checkers include:

  • The Index Checker warns about out-of-bounds accesses to arrays and strings.
  • The Initialized Fields Checker warns if a constructor does not initialize a field.
  • The Resource Leak Checker guarantees that every resource is closed rather than leaked. Examples of resources are a channel, executor, ExecutionControl, file, FileLock, Formatter, reader, Scanner, socket, stream, writer, etc.
  • The SQL Quotes Checker helps prevent SQL injection vulnerabilities.

New command-line arguments include:

  • -AskipFiles, -AonlyFiles
  • -AassumeSideEffectFree, -AassumeDeterministic, -AassumePure, -AassumePureGetters
  • -AuseConservativeDefaultsForUncheckedCode
  • -AignoreRawTypeArguments
  • -AwarnRedundantAnnotations
  • -Ainfer=ajava, -AinferOutputDirectory, -AinferOutputOriginal, -AshowWpiFailedInferences
  • -AshowSuppressWarningsStrings, -AwarnUnneededSuppressionsExceptions
  • -AshowPrefixInWarningMessages
  • -AstubNoWarnIfNotFound, -AstubWarnNote, -AmergeStubsWithSource
  • -Aonelinemsg, -AdumpOnErrors, -AexceptionLineSeparator
  • -ApermitMissingJdk, -AparseAllJdk
  • -AslowTypecheckingSeconds

... (truncated)

Commits
  • 479d087 new release 4.0.0
  • bfff757 Put the manual in the right place.
  • c532f6d Put a copy of manual.pdf at top level of website as expected.
  • 5e53e6c No closed issues.
  • e67ae85 Prep for release.
  • 4192d0d Remove file SKIP-REQUIRE-JAVADOC
  • 7d6d856 Remove or update references to JDK 8-16
  • b1e3761 Remove all deprecated methods
  • a1b3064 Directly use Java 17 and below Javac APIs. (#7582)
  • 4efdbdb Remove support for Java 8 from scripts and build scripts. (#7575)
  • Additional commits viewable in compare view

Updates `org.checkerframework:checker` from 3.55.1 to 4.0.0
Release notes

Sourced from org.checkerframework:checker's releases.

Release 4.0.0 of the Checker Framework

Version 4.0.0 (2026-04-07)

User-visible changes

To run the Checker Framework, you need to use a JDK 17 or later version of javac. That is, you need to use JDK 17 or later when compiling your code.

The Checker Framework can type-check any Java project, including projects that compile to Java 8 or 11 bytecodes and run on JRE versions 8 or 11. That is, your code can run under any release of Java, from Java 8 onward.

The type qualifiers and utility libraries in checker-qual.jar and checker-util.jar still use Java 11 bytecode. Thus, they may be used in projects that run under Java 11 or later.

Changes since version 3.0.0

Since version 3.0.0, 91 authors have made over 4500 commits and closed over 600 issues. Thanks to everyone who contributed!

New checkers include:

  • The Index Checker warns about out-of-bounds accesses to arrays and strings.
  • The Initialized Fields Checker warns if a constructor does not initialize a field.
  • The Resource Leak Checker guarantees that every resource is closed rather than leaked. Examples of resources are a channel, executor, ExecutionControl, file, FileLock, Formatter, reader, Scanner, socket, stream, writer, etc.
  • The SQL Quotes Checker helps prevent SQL injection vulnerabilities.

New command-line arguments include:

  • -AskipFiles, -AonlyFiles
  • -AassumeSideEffectFree, -AassumeDeterministic, -AassumePure, -AassumePureGetters
  • -AuseConservativeDefaultsForUncheckedCode
  • -AignoreRawTypeArguments
  • -AwarnRedundantAnnotations
  • -Ainfer=ajava, -AinferOutputDirectory, -AinferOutputOriginal, -AshowWpiFailedInferences
  • -AshowSuppressWarningsStrings, -AwarnUnneededSuppressionsExceptions
  • -AshowPrefixInWarningMessages
  • -AstubNoWarnIfNotFound, -AstubWarnNote, -AmergeStubsWithSource
  • -Aonelinemsg, -AdumpOnErrors, -AexceptionLineSeparator
  • -ApermitMissingJdk, -AparseAllJdk
  • -AslowTypecheckingSeconds
  • -Aversion, -AprintGitProperties
  • You can pass an option to only a particular checker (not all checkers) by using an underscore prefix.

Other improvements include thousands of enhancements and bug fixes -- too many to list here.

Implementation details

All previously-deprecated methods and classes have been removed. If your project builds upon the Checker Framework, we suggest that you upgrade to version 3.55.1, resolve all the deprecation warnings, then upgrade to version 4.0.0.

Changelog

Sourced from org.checkerframework:checker's changelog.

Version 4.0.0 (2026-04-07)

User-visible changes

To run the Checker Framework, you need to use a JDK 17 or later version of javac. That is, you need to use JDK 17 or later when compiling your code.

The Checker Framework can type-check any Java project, including projects that compile to Java 8 or 11 bytecodes and run on JRE versions 8 or 11. That is, your code can run under any release of Java, from Java 8 onward.

The type qualifiers and utility libraries in checker-qual.jar and checker-util.jar still use Java 11 bytecode. Thus, they may be used in projects that run under Java 11 or later.

Changes since version 3.0.0

Since version 3.0.0, 91 authors have made over 4500 commits and closed over 600 issues. Thanks to everyone who contributed!

New checkers include:

  • The Index Checker warns about out-of-bounds accesses to arrays and strings.
  • The Initialized Fields Checker warns if a constructor does not initialize a field.
  • The Resource Leak Checker guarantees that every resource is closed rather than leaked. Examples of resources are a channel, executor, ExecutionControl, file, FileLock, Formatter, reader, Scanner, socket, stream, writer, etc.
  • The SQL Quotes Checker helps prevent SQL injection vulnerabilities.

New command-line arguments include:

  • -AskipFiles, -AonlyFiles
  • -AassumeSideEffectFree, -AassumeDeterministic, -AassumePure, -AassumePureGetters
  • -AuseConservativeDefaultsForUncheckedCode
  • -AignoreRawTypeArguments
  • -AwarnRedundantAnnotations
  • -Ainfer=ajava, -AinferOutputDirectory, -AinferOutputOriginal, -AshowWpiFailedInferences
  • -AshowSuppressWarningsStrings, -AwarnUnneededSuppressionsExceptions
  • -AshowPrefixInWarningMessages
  • -AstubNoWarnIfNotFound, -AstubWarnNote, -AmergeStubsWithSource
  • -Aonelinemsg, -AdumpOnErrors, -AexceptionLineSeparator
  • -ApermitMissingJdk, -AparseAllJdk
  • -AslowTypecheckingSeconds

... (truncated)

Commits
  • 479d087 new release 4.0.0
  • bfff757 Put the manual in the right place.
  • c532f6d Put a copy of manual.pdf at top level of website as expected.
  • 5e53e6c No closed issues.
  • e67ae85 Prep for release.
  • 4192d0d Remove file SKIP-REQUIRE-JAVADOC
  • 7d6d856 Remove or update references to JDK 8-16
  • b1e3761 Remove all deprecated methods
  • a1b3064 Directly use Java 17 and below Javac APIs. (#7582)
  • 4efdbdb Remove support for Java 8 from scripts and build scripts. (#7575)
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a388c90fee..0c5a00ded4 100644 --- a/pom.xml +++ b/pom.xml @@ -112,7 +112,7 @@ under the License. 10.23.0 true 2.42.0 - 3.55.1 + 4.0.0 1.5.32 none -Xdoclint:none From 0f0a58433b46004ff9869a2512beded177340bca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 06:17:22 +0200 Subject: [PATCH 349/408] MINOR: Bump io.grpc:grpc-bom from 1.79.0 to 1.80.0 (#1093) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [io.grpc:grpc-bom](https://github.com/grpc/grpc-java) from 1.79.0 to 1.80.0.
Release notes

Sourced from io.grpc:grpc-bom's releases.

V1.80.0

API Changes

  • core: Added PickResult.copyWithSubchannel() and PickResult.copyWithStreamTracerFactory() to simplify updating PickResult while preserving metadata. Load balancing policies should now ensure ForwardingSubchannel decorators are unwrapped before being returned in a pick result. (#12658) (eae16b251)

Bug Fixes

  • core: Fixed the retry backoff jitter range to [0.8, 1.2] to align with the gRPC A6 specification. Retries will now occur more consistently around the calculated backoff interval. (#12639) (024fdd0ea) core: Fixed a race condition in RetriableStream where inFlightSubStreams counting could become inconsistent during concurrent retry and deadline events. This ensures that client calls (such as blockingUnaryCall) do not hang indefinitely and correctly receive a close signal. (#12649) (73abb4854)

Improvements

  • api: Trigger R8's ServiceLoader optimization to reduce necessary configuration when using R8 Full Mode (470219f9c). This allows gRPC to avoid reflection, and the need to specify -keeps for various class’s constructors. Upgrade to protobuf 33.4 (#12615) (50c18f183)
  • cronet: Introduced CRONET_READ_BUFFER_SIZE_KEY to allow customizing the read buffer size per-stream via CallOptions. Increasing the buffer size from the 4KB default can significantly improve performance for large messages by reducing JNI and context-switching overhead. (31fdb6c22)
  • api: Moved FlagResetRule to api/testFixtures and updated ManagedChannelRegistry to honor the GRPC_ENABLE_RFC3986_URIS feature flag. This ensures that target parsing is consistent across the library when the new URI parser is enabled. (#12608)
  • api: Updated NameResolverRegistry to natively support io.grpc.Uri. This is a foundational change that allows gRPC's name resolution system to handle URIs parsed with the new RFC 3986-compliant parser, ensuring more robust target handling. (#12609) (990348876)
  • xds: Removed the GRPC_EXPERIMENTAL_XDS_SNI feature flag. SNI determination via xDS is now always enabled and follows gRFC A101, where SNI is derived from xDS configurations like auto_host_sni or UpstreamTlsContext.sni. This ensures that no SNI is sent if not explicitly configured, unless the legacy channel authority fallback is enabled. (#12625) (ac44e9681)

New Features

  • core: pick_first shuffling now a weighted shuffle and observes weights from EDS (34dd29042). This finishes the gRFC A113 pick_first: Weighted Random Shuffling support
  • netty: Added RFC 3986 support to the unix: name resolver. This enables proper parsing of Unix domain socket URIs, including correct handling of query and fragment components in both hierarchical (e.g., unix:///path) and opaque (e.g., unix:/path) formats. (#12659)

Thanks to

Commits
  • 6c231b4 Bump version to 1.80.0
  • daf7a6c Update README etc to reference 1.80.0
  • b7f9074 Revert "fix(xds): Allow and normalize trailing dot (FQDN) in matchHostName (#...
  • 09a6e2e Revert "netty: Preserve early server handshake failure cause in logs"
  • 31fdb6c Add CRONET_READ_BUFFER_SIZE_KEY API to CronetClientStream
  • 470219f Trigger R8's ServiceLoader optimization
  • 50ead96 netty: Preserve early server handshake failure cause in logs
  • eae16b2 unwrap ForwardingSubchannel during Picks (#12658)
  • d9320ee netty: Add RFC 3986 support to the 'unix:' name resolver.
  • d5536b3 netty: factor out some duplicated code into a helper method
  • Additional commits viewable in compare view

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0c5a00ded4..738aea03b4 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,7 @@ under the License. 2.0.17 33.6.0-jre 4.2.12.Final - 1.79.0 + 1.80.0 4.34.1 2.21.2 3.5.0 From e5482cd63d2043c0e418f1adc7bcd1e5192c5078 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Sun, 10 May 2026 08:15:56 +0200 Subject: [PATCH 350/408] MINOR: Fix Gandiva JNI build against Arrow C++ 24.0.0 (#1140) Arrow C++ 24.0.0 introduced two breaking changes for the JNI build: 1. `arrow::decimal()` was removed; replaced with `arrow::decimal128()` in the Gandiva JNI source. 2. xsimd >= 14.0.0 is now a required dependency. The Docker image's vcpkg registry only ships xsimd 13.2.0, so the Linux JNI CMake configuration was failing. Fixed by passing `xsimd_SOURCE=BUNDLED` to the Arrow C++ CMake configure step so Arrow downloads and uses xsimd 14.0.0 directly, and by passing the vcpkg toolchain file to the Arrow C++ configure step so other vcpkg-managed dependencies are still resolved correctly. --- .env | 2 +- compose.yaml | 3 ++- gandiva/src/main/cpp/jni_common.cc | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.env b/.env index 2bd7255476..51daa0406c 100644 --- a/.env +++ b/.env @@ -53,4 +53,4 @@ MAVEN=3.9.9 # Versions for various dependencies used to build artifacts # Keep in sync with apache/arrow ARROW_REPO_ROOT=./arrow -VCPKG="4334d8b4c8916018600212ab4dd4bbdc343065d1" # 2025.09.17 Release +VCPKG="66c0373dc7fca549e5803087b9487edfe3aca0a1" # 2026.01.16 Release diff --git a/compose.yaml b/compose.yaml index fb290b22fe..4fd825e5a5 100644 --- a/compose.yaml +++ b/compose.yaml @@ -109,5 +109,6 @@ services: ARROW_JAVA_CDATA: "ON" CCACHE_DIR: "/ccache" command: - ["git config --global --add safe.directory /arrow-java && \ + ["/bin/bash", "-c", + "git config --global --add safe.directory /arrow-java && /arrow-java/ci/scripts/jni_manylinux_build.sh /arrow-java /arrow /build/java /arrow-java/jni"] diff --git a/gandiva/src/main/cpp/jni_common.cc b/gandiva/src/main/cpp/jni_common.cc index 2851250072..ec4888a512 100644 --- a/gandiva/src/main/cpp/jni_common.cc +++ b/gandiva/src/main/cpp/jni_common.cc @@ -221,7 +221,7 @@ DataTypePtr ProtoTypeToDataType(const gandiva::types::ExtGandivaType& ext_type) return arrow::date64(); case gandiva::types::DECIMAL: // TODO: error handling - return arrow::decimal(ext_type.precision(), ext_type.scale()); + return arrow::decimal128(ext_type.precision(), ext_type.scale()); case gandiva::types::TIME32: return ProtoTypeToTime32(ext_type); case gandiva::types::TIME64: From 138e8521b17a2b83814b8714c7d8e0f94fe13656 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 May 2026 05:20:41 +0200 Subject: [PATCH 351/408] MINOR: Bump com.fasterxml.jackson:jackson-bom from 2.21.2 to 2.21.3 (#1133) Bumps [com.fasterxml.jackson:jackson-bom](https://github.com/FasterXML/jackson-bom) from 2.21.2 to 2.21.3.
Commits
  • 374fbd0 [maven-release-plugin] prepare release jackson-bom-2.21.3
  • 7059df7 Prep for 2.21.3 release
  • 2fd60bd Merge branch '2.20' into 2.21
  • b82a364 Merge branch '2.19' into 2.20
  • ef4e013 Merge branch '2.18' into 2.19
  • 536ae51 Post-release dep version bump
  • 536c533 [maven-release-plugin] prepare for next development iteration
  • 426b778 [maven-release-plugin] prepare release jackson-bom-2.18.7
  • a73cda9 Prep for 2.18.7 release
  • 76b4a05 Post-release dep version bump
  • Additional commits viewable in compare view

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 738aea03b4..a5058a6336 100644 --- a/pom.xml +++ b/pom.xml @@ -101,7 +101,7 @@ under the License. 4.2.12.Final 1.80.0 4.34.1 - 2.21.2 + 2.21.3 3.5.0 25.2.10 1.12.1 From 5dc3ea699a4b7f8e4799c8ee318b96c9a9cc4f31 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 May 2026 07:58:24 +0200 Subject: [PATCH 352/408] MINOR: Bump com.github.luben:zstd-jni from 1.5.7-7 to 1.5.7-8 (#1132) Bumps [com.github.luben:zstd-jni](https://github.com/luben/zstd-jni) from 1.5.7-7 to 1.5.7-8.
Commits

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- compression/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compression/pom.xml b/compression/pom.xml index 9014b6913a..f3de6fc248 100644 --- a/compression/pom.xml +++ b/compression/pom.xml @@ -55,7 +55,7 @@ under the License. com.github.luben zstd-jni - 1.5.7-7 + 1.5.7-8 From 5e2fb636514c708cf27bd367c8f452d72ab5fdef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 May 2026 15:45:59 +0200 Subject: [PATCH 353/408] MINOR: Bump com.google.api.grpc:proto-google-common-protos from 2.70.0 to 2.71.0 (#1144) Bumps [com.google.api.grpc:proto-google-common-protos](https://github.com/googleapis/sdk-platform-java) from 2.70.0 to 2.71.0.
Commits
  • 6473668 chore(main): release 2.63.0 (#3927)
  • 8015e7e chore: update googleapis commit at Fri Oct 3 02:28:22 UTC 2025 (#3923)
  • 48075a8 chore: Upper bound file deps change has chore type (#3949)
  • 1d74663 deps: update google auth library dependencies to v1.40.0 (#3945)
  • 7fb4f15 deps: Upgrade Google Http Java Client to v2.0.2 (#3946)
  • feabef3 feat(librariangen): add bazel package (#3940)
  • 8d6c1f9 deps: Bump Guava to v33.5.0 (#3943)
  • 180b9a0 build(deps): update dependency com.google.cloud:google-cloud-shared-config to...
  • 3f548fb deps: update upper bound dependencies file (#3947)
  • a1b5ba3 chore: Manage errorprone and j2objc versions in pom-parent (#3948)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.api.grpc:proto-google-common-protos&package-manager=maven&previous-version=2.70.0&new-version=2.71.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml index d199f70da4..15b870905d 100644 --- a/flight/flight-core/pom.xml +++ b/flight/flight-core/pom.xml @@ -134,7 +134,7 @@ under the License. com.google.api.grpc proto-google-common-protos - 2.70.0 + 2.71.0 test From 30d528fa9992c9ed16a01853a2ed285f0efcbd21 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 May 2026 15:52:35 +0200 Subject: [PATCH 354/408] MINOR: Bump checker.framework.version from 4.0.0 to 4.1.0 (#1131) Bumps `checker.framework.version` from 4.0.0 to 4.1.0. Updates `org.checkerframework:checker-qual` from 4.0.0 to 4.1.0
Release notes

Sourced from org.checkerframework:checker-qual's releases.

Checker Framework 4.1.0

Version 4.1.0 (2026-05-01)

User-visible changes

Removed deprecated script checker/bin-devel/build.sh; use ./gradlew assemble instead.

Removed deprecated names "builder", "object.construction", and "objectconstruction" for the Called Methods Checker.

Implementation details

New method annotation @DoesNotUnrefineReceiver.

In AnnotatedTypeFactory:

  • new method hasDoesNotUnrefineReceiver().
  • isAliasedTypeAnnotation() is now protected rather than public.

Closed issues

#6890, #7364, #7488.

Changelog

Sourced from org.checkerframework:checker-qual's changelog.

Version 4.1.0 (2026-05-01)

User-visible changes

Removed deprecated script checker/bin-devel/build.sh; use ./gradlew assemble instead.

Removed deprecated names "builder", "object.construction", and "objectconstruction" for the Called Methods Checker.

Implementation details

New method annotation @DoesNotUnrefineReceiver.

In AnnotatedTypeFactory:

  • new method hasDoesNotUnrefineReceiver().
  • isAliasedTypeAnnotation() is now protected rather than public.

Closed issues

#6890, #7364, #7488.

Commits

Updates `org.checkerframework:checker` from 4.0.0 to 4.1.0
Release notes

Sourced from org.checkerframework:checker's releases.

Checker Framework 4.1.0

Version 4.1.0 (2026-05-01)

User-visible changes

Removed deprecated script checker/bin-devel/build.sh; use ./gradlew assemble instead.

Removed deprecated names "builder", "object.construction", and "objectconstruction" for the Called Methods Checker.

Implementation details

New method annotation @DoesNotUnrefineReceiver.

In AnnotatedTypeFactory:

  • new method hasDoesNotUnrefineReceiver().
  • isAliasedTypeAnnotation() is now protected rather than public.

Closed issues

#6890, #7364, #7488.

Changelog

Sourced from org.checkerframework:checker's changelog.

Version 4.1.0 (2026-05-01)

User-visible changes

Removed deprecated script checker/bin-devel/build.sh; use ./gradlew assemble instead.

Removed deprecated names "builder", "object.construction", and "objectconstruction" for the Called Methods Checker.

Implementation details

New method annotation @DoesNotUnrefineReceiver.

In AnnotatedTypeFactory:

  • new method hasDoesNotUnrefineReceiver().
  • isAliasedTypeAnnotation() is now protected rather than public.

Closed issues

#6890, #7364, #7488.

Commits

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a5058a6336..e0754f7789 100644 --- a/pom.xml +++ b/pom.xml @@ -112,7 +112,7 @@ under the License. 10.23.0 true 2.42.0 - 4.0.0 + 4.1.0 1.5.32 none -Xdoclint:none From b068e28343d54604b1966e916a376ea4c1491f48 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 May 2026 15:52:55 +0200 Subject: [PATCH 355/408] MINOR: Bump com.github.ben-manes.caffeine:caffeine from 3.2.3 to 3.2.4 (#1130) Bumps [com.github.ben-manes.caffeine:caffeine](https://github.com/ben-manes/caffeine) from 3.2.3 to 3.2.4.
Release notes

Sourced from com.github.ben-manes.caffeine:caffeine's releases.

3.2.4

  • Improved access expiration's read performance by avoiding false sharing effects caused by the timestamp update
  • Fixed head-of-line blocking of expiration queues caused by in-flight async entries (#1954)
  • Fixed various minor issues found using AI audits
  • Added ObjectInputFilter support to JCache
Commits
  • 836b65c use a consistent expiration tolerance calculation
  • 0dc7daf resurrect in-flight async entries on expiration
  • 0bac8b5 handle head-of-line blocking of expiration queues (fixes #1954)
  • ff25836 test polish
  • f3a6176 Fix JCache close/createCache races and recursive teardown
  • 622fbe7 Fix removal in identity views and widen hill-climber counters
  • 8da5a7a defer weighing the entry until after the putIfAbsent hit fast-path
  • 94ad0ff Record eviction stats before notifying the removal listener consistently
  • f94c011 Auto-assert eviction stats alongside notifications.withCause.exclusively
  • 2e945e0 Skip timestamp writes within tolerance on the read path.
  • Additional commits viewable in compare view

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 1f9bc3e00e..a813b2e168 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -176,7 +176,7 @@ under the License. com.github.ben-manes.caffeine caffeine - 3.2.3 + 3.2.4 From e6d9248447ef90d2d9a1412bfdee7019377e0603 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 May 2026 15:53:10 +0200 Subject: [PATCH 356/408] MINOR: Bump com.gradle:common-custom-user-data-maven-extension from 2.1.0 to 2.2.0 (#1128) Bumps [com.gradle:common-custom-user-data-maven-extension](https://github.com/gradle/common-custom-user-data-maven-extension) from 2.1.0 to 2.2.0.
Release notes

Sourced from com.gradle:common-custom-user-data-maven-extension's releases.

2.2.0

  • [NEW] Add AI tag to the Build Scan when invoked by an AI Agent
  • [NEW] Add custom value to the Build Scan indicating which AI Agent invoked the build
  • [NEW] Add link in Build Scan to GitHub PR
  • [NEW] For GitHub PRs, capture GITHUB_BASE_REF as the value PR base branch
Commits
  • d594c60 [maven-release-plugin] prepare release v2.2.0
  • 0a48e7a Merge pull request #375 from gradle/cj/github-pr-base-branch
  • 02f6001 Capture GITHUB_BASE_REF as 'PR base branch' for GitHub PR builds
  • fc03fa9 Add recent feature additions to changes.md
  • 1501555 Merge pull request #370 from gradle/add-ai-agent-metadata
  • de773ae [Renovate Bot] Update dependency org.apache.maven:maven-core to v3.9.15 (#374)
  • 074832a [Renovate Bot] Update dependency maven to v3.9.15 (#373)
  • 9de6cf1 Merge pull request #372 from gradle/renovate/github-actions
  • ea1e3fc [Renovate Bot] Update actions/upload-artifact digest to 043fb46
  • 4384849 Match AI tags/values to CCUD Gradle
  • Additional commits viewable in compare view

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .mvn/extensions.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index f52dfafb4a..e909f05105 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -28,6 +28,6 @@ com.gradle common-custom-user-data-maven-extension - 2.1.0 + 2.2.0 From 331e9ee0968ab92499ed362ed276fa85e60b870d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 May 2026 15:53:29 +0200 Subject: [PATCH 357/408] MINOR: Bump commons-codec:commons-codec from 1.21.0 to 1.22.0 (#1127) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [commons-codec:commons-codec](https://github.com/apache/commons-codec) from 1.21.0 to 1.22.0.
Changelog

Sourced from commons-codec:commons-codec's changelog.

Apache Commons Codec 1.22.0 Release Notes

The Apache Commons Codec team is pleased to announce the release of Apache Commons Codec 1.22.0.

The Apache Commons Codec component contains encoders and decoders for formats such as Base16, Base32, Base64, digest, and Hexadecimal. In addition to these widely used encoders and decoders, the codec package also maintains a collection of phonetic encoding utilities.

This is a feature and maintenance release. Java 8 or later is required.

New features

  • CODEC-326: Add Base58 support. Thanks to Inkeet, Gary Gregory, Wolff Bock von Wuelfingen.
  •  Add
BaseNCodecInputStream.AbstracBuilder.setByteArray(byte[]). Thanks to
Gary Gregory.
  • CODEC-335: Add GitIdentifiers to compute Git blob and tree object identifiers. Thanks to Piotr P. Karwasz, Gary Gregory.

Fixed Bugs

  • CODEC-249: Fix Incorrect transform of CH digraph according Metaphone basic rules #423. Thanks to Shalu Jha, Andrey, Gary Gregory.
  • CODEC-317: ColognePhonetic can create duplicate consecutive codes in some cases. Thanks to DRUser123, Shalu Jha, Gary Gregory.
  •  Add boundary tests for BinaryCodec.fromAscii partial-bit
inputs [#425](https://github.com/apache/commons-codec/issues/425).
Thanks to fancying, Gary Gregory.
  • CODEC-336: Base64.Builder.setUrlSafe(boolean) Javadoc incorrectly states null is accepted for primitive boolean parameter. Thanks to Partha Paul, Gary Gregory.

Changes

  •  Bump org.apache.commons:commons-parent from 96 to 98. Thanks
to Gary Gregory.

For complete information on Apache Commons Codec, including instructions on how to submit bug reports, patches, or suggestions for improvement, see the Apache Commons Codec website:

https://commons.apache.org/proper/commons-codec/

Download page: https://commons.apache.org/proper/commons-codec/download_codec.cgi

Commits

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- vector/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vector/pom.xml b/vector/pom.xml index 4d247961c9..9b40e8820c 100644 --- a/vector/pom.xml +++ b/vector/pom.xml @@ -60,7 +60,7 @@ under the License. commons-codec commons-codec - 1.21.0 + 1.22.0 org.apache.arrow From 0f7665f2b78a5f6bcfb37924fbee05a45adfb5f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 May 2026 09:31:15 +0200 Subject: [PATCH 358/408] MINOR: Bump com.nimbusds:oauth2-oidc-sdk from 11.37 to 11.37.1 (#1143) Bumps [com.nimbusds:oauth2-oidc-sdk](https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions) from 11.37 to 11.37.1.
Changelog

Sourced from com.nimbusds:oauth2-oidc-sdk's changelog.

version 1.0 (2012-05-29) * First official release with authorisation endpoint, token endpoint, check ID endpoint and UserInfo endpoint support. * JSON Web Tokens (JWTs) support through the Nimbus-JWT library. * Language Tags (RFC 5646) support through the Nimbus-LangTag library. * JSON support through the JSON Smart library.

version 2.0 (2013-05-13) * Intermediary development release with Maven build, published to Maven Central.

version 2.1 (2013-06-06) * Updates the APIs to OpenID Connect Messages draft 20, OpenID Connect Standard draft 21, OpenID Connect Discovery draft 17 and OpenID Connect Registration draft 19. * Major refactoring of the APIs for greater simplicity. * Adds JUnit tests.

version 2.2 (2013-06-18) * Refactors dynamic OpenID Connect client registration. * Adds partial support of the OAuth 2.0 Dynamic Client Registration Protocol (draft-ietf-oauth-dyn-reg-12). * Optimises parsing of request parameters consisting of one or more tokens (scope, response type, etc).

version 2.3 (2013-06-19) * Renames OAuth 2.0 dynamic client registration package. * Adds ClientInformation.getClientMetadata() method. * Adds OIDCClientInformation class.

version 2.4 (2013-06-20) * Adds static OIDCClientInformation.parse(JSONObject) method.

version 2.5 (2013-06-22) * Adds support OAuth 2.0 dynamic client update. * Adds OpenID Connect dynamic client registration classes.

version 2.6 (2013-06-25) * Enforces order of preference of ACR values in OpenID Connect client metadata, as required by the specification. * Documentation and performance improvements.

version 2.7 (2013-06-26) * Switches Identifier generation to java.security.SecureRandom.

version 2.8 (2013-06-30) * Fixes serialisation and assignment bugs in ClientMetadata. * Switches Secret generation to java.security.SecureRandom.

version 2.9 (2013-09-17)

... (truncated)

Commits
  • 2a0f271 [maven-release-plugin] prepare for next development iteration
  • fac7277 Bumps Nimbus JOSE+JWT, BouncyCastle
  • 517deb7 [maven-release-plugin] prepare release 11.37.1
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.nimbusds:oauth2-oidc-sdk&package-manager=maven&previous-version=11.37&new-version=11.37.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index a813b2e168..ea33091101 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -182,7 +182,7 @@ under the License. com.nimbusds oauth2-oidc-sdk - 11.37 + 11.37.1 From 3bc34b041761081ac32a7cd3b167f9ab8b628677 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 May 2026 13:51:48 +0200 Subject: [PATCH 359/408] MINOR: Bump commons-io:commons-io from 2.21.0 to 2.22.0 (#1126) Bumps commons-io:commons-io from 2.21.0 to 2.22.0. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=commons-io:commons-io&package-manager=maven&previous-version=2.21.0&new-version=2.22.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dataset/pom.xml | 2 +- flight/flight-sql-jdbc-core/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dataset/pom.xml b/dataset/pom.xml index 7a0210ce95..cb889ecd7d 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -156,7 +156,7 @@ under the License. commons-io commons-io - 2.21.0 + 2.22.0 test diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index ea33091101..237b25d0e5 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -105,7 +105,7 @@ under the License. commons-io commons-io - 2.21.0 + 2.22.0 test From af86cd3a7c17237368ea8411ed32e21b262c5b7b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 May 2026 15:22:37 +0200 Subject: [PATCH 360/408] MINOR: Bump org.apache.calcite.avatica:avatica from 1.26.0 to 1.27.0 (#986) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.apache.calcite.avatica:avatica](https://github.com/apache/calcite-avatica) from 1.26.0 to 1.27.0.
Commits
  • 7754d94 [CALCITE-7200] Release Avatica 1.27.0
  • 1e05be2 [CALCITE-7171] Update Jackson from 2.15.4 to 2.18.4.1 and switch to using jac...
  • 9698a96 [CALCITE-7177] Update Guava from 33.4.0-jre to 33.4.8-jre in Avatica
  • 0aec625 Bump rexml from 3.4.1 to 3.4.2 in /site
  • 5954d1a [CALCITE-7165] Update OWASP plugin version to 12.1.3 for JDKs >= 11
  • 7b6f14c [CALCITE-7172] Update chekstyle version from 10.19.0 to 10.26.1 in Avatica
  • 3ee1fd7 [CALCITE-7169] Update protobuf from 3.25.5 to 3.25.8 in Avatica
  • 927dc10 [CALCITE-7168] Update httpcore5 from 5.3.1 to 5.3.5 in Avatica
  • 458189f [CALCITE-7167] Upgrade Jetty from 9.4.56.v20240826 to 9.4.58.v20250814 in Ava...
  • 592a39e [CALCITE-7166] Update Gradle from 8.7 to 8.14.3 in Avatica
  • Additional commits viewable in compare view

> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: JB Onofré --- flight/flight-sql-jdbc-core/pom.xml | 2 +- ...owFlightJdbcVectorSchemaRootResultSet.java | 2 +- .../accessor/ArrowFlightJdbcAccessor.java | 24 +++++++++++++++++++ flight/flight-sql-jdbc-driver/pom.xml | 1 + 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 237b25d0e5..96a11f2b9a 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -159,7 +159,7 @@ under the License. org.apache.calcite.avatica avatica - 1.26.0 + 1.27.0 diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java index 49334951de..5d02d6e843 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java @@ -128,7 +128,7 @@ public Object getObject(int columnIndex) throws SQLException { if (metaData.type.id == Types.TIMESTAMP_WITH_TIMEZONE) { return accessor.getTimestamp(localCalendar); } else { - return AvaticaSite.get(accessor, metaData.type.id, localCalendar); + return AvaticaSite.get(accessor, metaData.type.id, true, localCalendar); } } diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessor.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessor.java index f0fa55fa82..cd762fb1ac 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessor.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessor.java @@ -36,6 +36,10 @@ import java.util.Calendar; import java.util.Map; import java.util.function.IntSupplier; +import org.joou.UByte; +import org.joou.UInteger; +import org.joou.ULong; +import org.joou.UShort; /** Base Jdbc Accessor. */ public abstract class ArrowFlightJdbcAccessor implements Accessor { @@ -99,6 +103,26 @@ public long getLong() throws SQLException { throw getOperationNotSupported(this.getClass()); } + @Override + public UByte getUByte() throws SQLException { + throw getOperationNotSupported(this.getClass()); + } + + @Override + public UShort getUShort() throws SQLException { + throw getOperationNotSupported(this.getClass()); + } + + @Override + public UInteger getUInt() throws SQLException { + throw getOperationNotSupported(this.getClass()); + } + + @Override + public ULong getULong() throws SQLException { + throw getOperationNotSupported(this.getClass()); + } + @Override public float getFloat() throws SQLException { throw getOperationNotSupported(this.getClass()); diff --git a/flight/flight-sql-jdbc-driver/pom.xml b/flight/flight-sql-jdbc-driver/pom.xml index ff5763702b..801089d090 100644 --- a/flight/flight-sql-jdbc-driver/pom.xml +++ b/flight/flight-sql-jdbc-driver/pom.xml @@ -159,6 +159,7 @@ under the License. org.apache.calcite.avatica:* META-INF/services/java.sql.Driver + META-INF/README.txt From 8c36d02205974c0cb912b11383e26f141ee2971e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 May 2026 17:45:09 +0200 Subject: [PATCH 361/408] MINOR: Bump org.apache:apache from 37 to 38 (#1156) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.apache:apache](https://github.com/apache/maven-apache-parent) from 37 to 38.
Release notes

Sourced from org.apache:apache's releases.

Apache Parent POM version 38

👻 Maintenance

📦 Dependency updates

  • Bump org.apache.maven.plugins:maven-invoker-plugin from 3.10.0 to 3.10.1 (#578) @dependabot[bot]
  • Bump org.apache.maven.plugins:maven-invoker-plugin from 3.9.1 to 3.10.0 (#575) @dependabot[bot]
  • Bump org.apache.maven.plugins:maven-resources-plugin from 3.4.0 to 3.5.0 (#572) @dependabot[bot]
  • Bump org.apache.maven.plugins:maven-shade-plugin from 3.6.1 to 3.6.2 (#573) @dependabot[bot]
  • Bump org.apache.apache.resources:apache-source-release-assembly-descriptor from 1.7 to 1.8 (#571) @dependabot[bot]
  • Bump version.maven-surefire from 3.5.4 to 3.5.5 (#570) @dependabot[bot]
  • Bump org.apache.maven.plugins:maven-dependency-plugin from 3.9.0 to 3.10.0 (#568) @dependabot[bot]
  • Bump org.apache.maven.plugins:maven-compiler-plugin from 3.14.1 to 3.15.0 (#567) @dependabot[bot]
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache:apache&package-manager=maven&previous-version=37&new-version=38)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- bom/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index d97cc291c2..083ee1e0de 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache apache - 37 + 38 diff --git a/pom.xml b/pom.xml index e0754f7789..d6d36e5607 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache apache - 37 + 38 org.apache.arrow From c49d97625524a9ba3cb72380f88a2ee12f435695 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 May 2026 17:45:26 +0200 Subject: [PATCH 362/408] MINOR: Bump org.immutables:value-annotations from 2.12.1 to 2.12.2 (#1157) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.immutables:value-annotations](https://github.com/immutables/immutables) from 2.12.1 to 2.12.2.
Release notes

Sourced from org.immutables:value-annotations's releases.

2.12.2

Maintenance release

What's Changed

New Contributors

Full Changelog: https://github.com/immutables/immutables/compare/2.12.1...2.12.2

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.immutables:value-annotations&package-manager=maven&previous-version=2.12.1&new-version=2.12.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d6d36e5607..dabb02594a 100644 --- a/pom.xml +++ b/pom.xml @@ -183,7 +183,7 @@ under the License. org.immutables value-annotations - 2.12.1 + 2.12.2 provided From d88adb33b00e8a7c3b743312b11c05ed36d2bd37 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 May 2026 18:42:53 +0200 Subject: [PATCH 363/408] MINOR: Bump io.netty:netty-bom from 4.2.12.Final to 4.2.13.Final (#1155) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [io.netty:netty-bom](https://github.com/netty/netty) from 4.2.12.Final to 4.2.13.Final.
Release notes

Sourced from io.netty:netty-bom's releases.

netty-4.2.13.Final

CVEs Fixed

What's Changed

... (truncated)

Commits
  • b3844c8 [maven-release-plugin] prepare release netty-4.2.13.Final
  • 82f47fa Merge commit from fork
  • ada0999 Merge commit from fork
  • b4051e2 Fix BrotliDecoder not forwarding all decompressed chunks
  • 67207c1 Merge commit from fork
  • 541ca7c Merge commit from fork
  • 943edb3 Fix codec-dns tests
  • 6459a28 Merge commit from fork
  • b4ba61b Fix checkstyle in HttpObjectDecoder
  • 977661f Merge commit from fork
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=io.netty:netty-bom&package-manager=maven&previous-version=4.2.12.Final&new-version=4.2.13.Final)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index dabb02594a..df387a067d 100644 --- a/pom.xml +++ b/pom.xml @@ -98,7 +98,7 @@ under the License. 5.12.2 2.0.17 33.6.0-jre - 4.2.12.Final + 4.2.13.Final 1.80.0 4.34.1 2.21.3 From 28367478a6ae695e409368b6f68b9ea22730fd9c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 May 2026 15:47:49 +0200 Subject: [PATCH 364/408] MINOR: Bump io.grpc:grpc-bom from 1.80.0 to 1.81.0 (#1154) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [io.grpc:grpc-bom](https://github.com/grpc/grpc-java) from 1.80.0 to 1.81.0.
Release notes

Sourced from io.grpc:grpc-bom's releases.

V1.81.0

In this release we drop support for Android API level 22 or lower (Lollipop or earlier), following Google Play Service’s discontinued updates for Lollipop (API levels 21 & 22) and now requires a minimum of API level 23 (Android 6.0 Marshmallow).

API Changes

  • api: Deprecate LoadBalancer.handleResolvedAddresses(). Developers maintaining custom LoadBalancer implementations should transition to using LoadBalancer.acceptResolvedAddresses(). Unlike the deprecated method, acceptResolvedAddresses() returns a Status object, allowing the load balancer to explicitly report success or reject the update if the provided addresses or configuration are invalid. (#11623)

Behavior Changes

  • core: Enable dns "caching" on Android for 30 seconds to reduce CPU impact of a refresh loop with an LB policy (0675f70af). DnsNameResolver ignores re-resolution requests on OpenJDK-like platforms if it has been too soon since the last DNS query because InetAddress.getAllByName() has a cache with a fixed entry lifetime, but this logic was disabled for Android which does not have that style of cache. Android’s cache uses the result TTL, which will rarely be less than 30 seconds. This change would probably be most noticeable when 1) changing to a different network (e.g., from wifi to mobile), 2) the server has different addresses for different networks, and 3) the app is not using AndroidChannelBuilder with an android.context.Context. For reference, it seems Chrome caches for 1 minute

Bug Fixes

  • opentelemetry: Fix baggage propagation, the baggage propagation for opentelemetry introduced in #12389 was broken. The context is decided once and used for all recording for the call, thus guaranteeing all record()s have consistent information.
  • core: Address a race condition where ManagedChannelOrphanWrapper could incorrectly log a "not shutdown properly" warning during garbage collection when using directExecutor(). (#12705) (d459338d9)
  • xds: Fix xDS HTTP CONNECT's transport socket name bug which is now corrected to use typeUrl. (#12740) (eac9fe961)
  • xds: Fix an issue where subchannel metrics were dropping their association with the backend_service. This ensures xDS load balancing metrics are reported accurately. (#12735)

New Features

  • netty: Add tcp metrics, by implementing a few of the metrics defined in A80.
  • api: Add a CallOption for a custom label on per-RPC metrics (0e39b2967). This CallOption is copied by grpc-opentelemetry to the grpc.client.call.custom label as defined by gRFC A108. See also the gRPC OpenTelemetry Metrics guide (update in-progress)
  • xds: Add support for Weighted Round Robin (WRR) load balancing driven by custom backend metrics, implementing the behavior defined in gRFC A114. (#12645)
  • utils: Update AdvancedTlsX509KeyManager so that developers can now preserve and use key aliases when dynamically reloading TLS certificates. (#12686)

Documentation

  • Update the "Outgoing Flow Control" section in the Manual Flow Control example to say onNext() does not block, but rather queues the messages in memory and advises developers to use CallStreamObserver.isReady() to prevent this memory exhaustion (#12700) (a3a9ffcbe) (#12726) (65ae2efda)
  • examples: Clean up Health example, and document need for grpc-services (3ed732fc0)

Dependencies

  • Upgrade Dependencies (#12719) (16e17abba). Google-auth-library: 1.42.1, animal-sniffer: 1.27, assertj-core:3.27.7, error_prone_annotations:2.48.0, proto-google-common-protos:2.64.1, google-cloud-logging:3.23.10, jetty-http2-server:12.1.7, jetty-ee10-servlet:12.1.7, lincheck:3.4, opentelemetry-api:1.60.1, opentelemetry-exporter-prometheus:1.60.1-alpha, opentelemetry-gcp-resources:1.54.0-alpha, opentelemetry-sdk-extension-autoconfigure:1.60.1, opentelemetry-sdk-testing:1.60.1, robolectric:4.16.1, tomcat-embed-core:10.1.52, tomcat-embed-core9: 9.0.115,
  • Upgrade Netty to 4.1.132 and netty-tcnative to 2.0.75 (1528f809c)

Thanks to

Commits
  • 6951542 Bump version to 1.81.0
  • e94188e Update README etc to reference 1.81.0
  • 4813c6d core,xds: Fix backend_service plumbing for subchannel metrics (#12735)
  • 6737eb5 Revert "Replace javax ThreadSafe annotation with errorprone ThreadSafe (#1274...
  • ef35313 Replace javax ThreadSafe annotation with errorprone ThreadSafe (#12742)
  • 3ed732f examples: Clean up Health, and document need for grpc-services
  • eac9fe9 xds: fix xDS HTTP CONNECT's transport socket name bug (#12740)
  • 1528f80 Upgrade Netty to 4.1.132 and netty-tcnative to 2.0.75
  • d057a7e [xds] Implement A114: WRR support for custom backend metrics (#12645)
  • 842636f xds: Add configuration objects for ExtAuthz, GrpcService and Bootstrap change...
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=io.grpc:grpc-bom&package-manager=maven&previous-version=1.80.0&new-version=1.81.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: JB Onofré --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index df387a067d..a28d7167e1 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,7 @@ under the License. 2.0.17 33.6.0-jre 4.2.13.Final - 1.80.0 + 1.81.0 4.34.1 2.21.3 3.5.0 From 4899492a26f941d955f5567797aa697262e747dd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 May 2026 15:48:10 +0200 Subject: [PATCH 365/408] MINOR: Bump parquet.version from 1.17.0 to 1.17.1 (#1152) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps `parquet.version` from 1.17.0 to 1.17.1. Updates `org.apache.parquet:parquet-avro` from 1.17.0 to 1.17.1
Release notes

Sourced from org.apache.parquet:parquet-avro's releases.

Apache Parquet Java 1.17.1

What's Changed

Full Changelog: https://github.com/apache/parquet-java/compare/apache-parquet-1.17.0...apache-parquet-1.17.1

Apache Parquet Java 1.17.1 RC0

What's Changed

Full Changelog: https://github.com/apache/parquet-java/compare/apache-parquet-1.17.0...apache-parquet-1.17.1-rc0

Commits

Updates `org.apache.parquet:parquet-hadoop` from 1.17.0 to 1.17.1
Release notes

Sourced from org.apache.parquet:parquet-hadoop's releases.

Apache Parquet Java 1.17.1

What's Changed

Full Changelog: https://github.com/apache/parquet-java/compare/apache-parquet-1.17.0...apache-parquet-1.17.1

Apache Parquet Java 1.17.1 RC0

What's Changed

Full Changelog: https://github.com/apache/parquet-java/compare/apache-parquet-1.17.0...apache-parquet-1.17.1-rc0

Commits

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dataset/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataset/pom.xml b/dataset/pom.xml index cb889ecd7d..5acc837860 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -32,7 +32,7 @@ under the License. ../../../cpp/release-build/ - 1.17.0 + 1.17.1 1.12.1 From ef359de2cf33f3020eb7c41cd4a5eee564b99bf6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 May 2026 08:27:25 +0200 Subject: [PATCH 366/408] MINOR: Bump dep.slf4j.version from 2.0.17 to 2.0.18 (#1151) Bumps `dep.slf4j.version` from 2.0.17 to 2.0.18. Updates `org.slf4j:slf4j-api` from 2.0.17 to 2.0.18 Updates `org.slf4j:slf4j-jdk14` from 2.0.17 to 2.0.18 Updates `org.slf4j:jul-to-slf4j` from 2.0.17 to 2.0.18 Updates `org.slf4j:jcl-over-slf4j` from 2.0.17 to 2.0.18 Updates `org.slf4j:log4j-over-slf4j` from 2.0.17 to 2.0.18 Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a28d7167e1..2f55d591fd 100644 --- a/pom.xml +++ b/pom.xml @@ -96,7 +96,7 @@ under the License. ${project.build.directory}/generated-sources 1.9.0 5.12.2 - 2.0.17 + 2.0.18 33.6.0-jre 4.2.13.Final 1.81.0 From 4ecc92f817fc71fcf94e0df87d2c6c1bc9877847 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 May 2026 11:32:03 +0200 Subject: [PATCH 367/408] MINOR: Bump org.apache.parquet:parquet-variant from 1.17.0 to 1.17.1 (#1150) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.apache.parquet:parquet-variant](https://github.com/apache/parquet-mr) from 1.17.0 to 1.17.1.
Release notes

Sourced from org.apache.parquet:parquet-variant's releases.

Apache Parquet Java 1.17.1

What's Changed

Full Changelog: https://github.com/apache/parquet-java/compare/apache-parquet-1.17.0...apache-parquet-1.17.1

Apache Parquet Java 1.17.1 RC0

What's Changed

Full Changelog: https://github.com/apache/parquet-java/compare/apache-parquet-1.17.0...apache-parquet-1.17.1-rc0

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.parquet:parquet-variant&package-manager=maven&previous-version=1.17.0&new-version=1.17.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2f55d591fd..0ce71f74cc 100644 --- a/pom.xml +++ b/pom.xml @@ -105,7 +105,7 @@ under the License. 3.5.0 25.2.10 1.12.1 - 1.17.0 + 1.17.1 5.23.0 2 From 755ce550c4dfc1771c2b350a0159b5e7f903debb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 May 2026 10:32:04 +0200 Subject: [PATCH 368/408] MINOR: Bump org.immutables:value from 2.12.1 to 2.12.2 (#1168) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.immutables:value](https://github.com/immutables/immutables) from 2.12.1 to 2.12.2.
Release notes

Sourced from org.immutables:value's releases.

2.12.2

Maintenance release

What's Changed

New Contributors

Full Changelog: https://github.com/immutables/immutables/compare/2.12.1...2.12.2

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.immutables:value&package-manager=maven&previous-version=2.12.1&new-version=2.12.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0ce71f74cc..e69357c6af 100644 --- a/pom.xml +++ b/pom.xml @@ -314,7 +314,7 @@ under the License. org.immutables value - 2.12.1 + 2.12.2 From b58ce11516afd0aa2959673dac74ca0258ca2166 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 May 2026 10:54:46 +0200 Subject: [PATCH 369/408] MINOR: Bump com.google.protobuf:protobuf-bom from 4.34.1 to 4.35.0 (#1167) Bumps [com.google.protobuf:protobuf-bom](https://github.com/protocolbuffers/protobuf) from 4.34.1 to 4.35.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.protobuf:protobuf-bom&package-manager=maven&previous-version=4.34.1&new-version=4.35.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e69357c6af..2b629673ef 100644 --- a/pom.xml +++ b/pom.xml @@ -100,7 +100,7 @@ under the License. 33.6.0-jre 4.2.13.Final 1.81.0 - 4.34.1 + 4.35.0 2.21.3 3.5.0 25.2.10 From 7fc14a532492f00e0744d8316fa8fb2f6970d31e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 May 2026 11:15:13 +0200 Subject: [PATCH 370/408] MINOR: Bump io.netty:netty-bom from 4.2.13.Final to 4.2.14.Final (#1166) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [io.netty:netty-bom](https://github.com/netty/netty) from 4.2.13.Final to 4.2.14.Final.
Release notes

Sourced from io.netty:netty-bom's releases.

netty-4.2.14.Final

What's Changed

New Contributors

Full Changelog: https://github.com/netty/netty/compare/netty-4.2.13.Final...netty-4.2.14.Final

Commits
  • 0a60b75 [maven-release-plugin] prepare release netty-4.2.14.Final
  • 72df658 Fix MQTT decoder size check after variable header replay (#16787)
  • 7125dba MQTT: Allow MQTT 5 CONNECT with password only (#16833)
  • 9e19320 IoUring: Stop generic FileRegion drain loop when transferred() reaches count(...
  • 4ce9f17 Route synchronous onLookupComplete exceptions via fireExceptionCaught (#16794)
  • f7b1b7d Fix memoryAddress() for direct ByteBuffers wrapped by Unpooled without Unsafe...
  • 0ccb265 IpFilter: Fix ClassCastException caused by IpSubnetFilter if only ipv6 rules ...
  • a6aeb6d Resolve all localhost addresses without querying DNS servers (#16749)
  • c328ba2 Fix ResumptionController wrapping (#16815)
  • bc5862b HTTP2: Use 100 as default max concurrent streams setting (#16804)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=io.netty:netty-bom&package-manager=maven&previous-version=4.2.13.Final&new-version=4.2.14.Final)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2b629673ef..322e35f8b4 100644 --- a/pom.xml +++ b/pom.xml @@ -98,7 +98,7 @@ under the License. 5.12.2 2.0.18 33.6.0-jre - 4.2.13.Final + 4.2.14.Final 1.81.0 4.35.0 2.21.3 From 96339f6c2f35c3e6676e62fded200c0934f9d8d3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 May 2026 11:22:01 +0200 Subject: [PATCH 371/408] MINOR: Bump com.nimbusds:oauth2-oidc-sdk from 11.37.1 to 11.37.2 (#1164) Bumps [com.nimbusds:oauth2-oidc-sdk](https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions) from 11.37.1 to 11.37.2.
Changelog

Sourced from com.nimbusds:oauth2-oidc-sdk's changelog.

version 1.0 (2012-05-29) * First official release with authorisation endpoint, token endpoint, check ID endpoint and UserInfo endpoint support. * JSON Web Tokens (JWTs) support through the Nimbus-JWT library. * Language Tags (RFC 5646) support through the Nimbus-LangTag library. * JSON support through the JSON Smart library.

version 2.0 (2013-05-13) * Intermediary development release with Maven build, published to Maven Central.

version 2.1 (2013-06-06) * Updates the APIs to OpenID Connect Messages draft 20, OpenID Connect Standard draft 21, OpenID Connect Discovery draft 17 and OpenID Connect Registration draft 19. * Major refactoring of the APIs for greater simplicity. * Adds JUnit tests.

version 2.2 (2013-06-18) * Refactors dynamic OpenID Connect client registration. * Adds partial support of the OAuth 2.0 Dynamic Client Registration Protocol (draft-ietf-oauth-dyn-reg-12). * Optimises parsing of request parameters consisting of one or more tokens (scope, response type, etc).

version 2.3 (2013-06-19) * Renames OAuth 2.0 dynamic client registration package. * Adds ClientInformation.getClientMetadata() method. * Adds OIDCClientInformation class.

version 2.4 (2013-06-20) * Adds static OIDCClientInformation.parse(JSONObject) method.

version 2.5 (2013-06-22) * Adds support OAuth 2.0 dynamic client update. * Adds OpenID Connect dynamic client registration classes.

version 2.6 (2013-06-25) * Enforces order of preference of ACR values in OpenID Connect client metadata, as required by the specification. * Documentation and performance improvements.

version 2.7 (2013-06-26) * Switches Identifier generation to java.security.SecureRandom.

version 2.8 (2013-06-30) * Fixes serialisation and assignment bugs in ClientMetadata. * Switches Secret generation to java.security.SecureRandom.

version 2.9 (2013-09-17)

... (truncated)

Commits
  • fedf633 [maven-release-plugin] prepare for next development iteration
  • 29b77a0 Updates to JSON Smart 2.6.0
  • 6e53206 [maven-release-plugin] prepare release 11.37.2
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.nimbusds:oauth2-oidc-sdk&package-manager=maven&previous-version=11.37.1&new-version=11.37.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 96a11f2b9a..3741ee083c 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -182,7 +182,7 @@ under the License. com.nimbusds oauth2-oidc-sdk - 11.37.1 + 11.37.2 From 6a6b8c1c8f3364da125df743c4b00a212d91c449 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 May 2026 13:15:12 +0200 Subject: [PATCH 372/408] MINOR: Bump com.gradle:develocity-maven-extension from 2.4.0 to 2.4.1 (#1161) Bumps com.gradle:develocity-maven-extension from 2.4.0 to 2.4.1. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.gradle:develocity-maven-extension&package-manager=maven&previous-version=2.4.0&new-version=2.4.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .mvn/extensions.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index e909f05105..38b3c807b7 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -23,7 +23,7 @@ com.gradle develocity-maven-extension - 2.4.0 + 2.4.1 com.gradle From 315fd710cdf2a418e21cb567bedc8d7473e17ec7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 May 2026 11:20:28 +0200 Subject: [PATCH 373/408] MINOR: [CI] Bump docker/login-action from 4.1.0 to 4.2.0 (#1160) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [docker/login-action](https://github.com/docker/login-action) from 4.1.0 to 4.2.0.
Release notes

Sourced from docker/login-action's releases.

v4.2.0

Full Changelog: https://github.com/docker/login-action/compare/v4.1.0...v4.2.0

Commits
  • 650006c Merge pull request #960 from docker/dependabot/npm_and_yarn/aws-sdk-dependenc...
  • 99df1a3 chore: update generated content
  • 3ab375f build(deps): bump the aws-sdk-dependencies group across 1 directory with 2 up...
  • 39d8580 Merge pull request #970 from docker/dependabot/npm_and_yarn/docker/actions-to...
  • 4eefcd3 chore: update generated content
  • 56d092c build(deps): bump @​docker/actions-toolkit from 0.86.0 to 0.90.0
  • e2e31ca Merge pull request #976 from docker/dependabot/npm_and_yarn/actions/core-3.0.1
  • 0bced94 chore: update generated content
  • 3e75a0f build(deps): bump @​actions/core from 3.0.0 to 3.0.1
  • 365bebd Merge pull request #984 from docker/dependabot/github_actions/aws-actions/con...
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=docker/login-action&package-manager=github_actions&previous-version=4.1.0&new-version=4.2.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 2658f1da00..f0e99f1219 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -127,7 +127,7 @@ jobs: with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing - - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: registry: ghcr.io username: ${{ github.actor }} From da97828bfe2818e1233ef8c421e89e238fff8340 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Jun 2026 15:43:15 +0200 Subject: [PATCH 374/408] MINOR: Bump checker.framework.version from 4.1.0 to 4.2.0 (#1170) Bumps `checker.framework.version` from 4.1.0 to 4.2.0. Updates `org.checkerframework:checker-qual` from 4.1.0 to 4.2.0
Release notes

Sourced from org.checkerframework:checker-qual's releases.

Checker Framework 4.2.0

Version 4.2.0 (2026-06-01)

User-visible changes

Renamed error message key "createsmustcallfor.target.unparseable" to "createsmustcallfor.target.unparsable".

Implementation details

In AnnotatedTypeFactory:

  • new overload canonicalAnnotation(AnnotationMirror, TypeMirror).

In TypeHierarchy:

  • new methods equalsShallowEffective().

Closed issues

#7676, #7679, #7680, #7695, #7697, #7699, #7700, #7727.

Changelog

Sourced from org.checkerframework:checker-qual's changelog.

Version 4.2.0 (2026-06-01)

User-visible changes

Renamed error message key "createsmustcallfor.target.unparseable" to "createsmustcallfor.target.unparsable".

Implementation details

In AnnotatedTypeFactory:

  • new overload canonicalAnnotation(AnnotationMirror, TypeMirror).

In TypeHierarchy:

  • new methods equalsShallowEffective().

Closed issues

#7676, #7679, #7680, #7695, #7697, #7699, #7700, #7727.

Commits

Updates `org.checkerframework:checker` from 4.1.0 to 4.2.0
Release notes

Sourced from org.checkerframework:checker's releases.

Checker Framework 4.2.0

Version 4.2.0 (2026-06-01)

User-visible changes

Renamed error message key "createsmustcallfor.target.unparseable" to "createsmustcallfor.target.unparsable".

Implementation details

In AnnotatedTypeFactory:

  • new overload canonicalAnnotation(AnnotationMirror, TypeMirror).

In TypeHierarchy:

  • new methods equalsShallowEffective().

Closed issues

#7676, #7679, #7680, #7695, #7697, #7699, #7700, #7727.

Changelog

Sourced from org.checkerframework:checker's changelog.

Version 4.2.0 (2026-06-01)

User-visible changes

Renamed error message key "createsmustcallfor.target.unparseable" to "createsmustcallfor.target.unparsable".

Implementation details

In AnnotatedTypeFactory:

  • new overload canonicalAnnotation(AnnotationMirror, TypeMirror).

In TypeHierarchy:

  • new methods equalsShallowEffective().

Closed issues

#7676, #7679, #7680, #7695, #7697, #7699, #7700, #7727.

Commits

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 322e35f8b4..81da2a2866 100644 --- a/pom.xml +++ b/pom.xml @@ -112,7 +112,7 @@ under the License. 10.23.0 true 2.42.0 - 4.1.0 + 4.2.0 1.5.32 none -Xdoclint:none From 035d96b256cc5eb687add6af2f5326ac5523f272 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Jun 2026 15:57:55 +0200 Subject: [PATCH 375/408] MINOR: Bump dep.junit.jupiter.version from 5.12.2 to 6.1.0 (#1162) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps `dep.junit.jupiter.version` from 5.12.2 to 6.1.0. Updates `org.junit.jupiter:junit-jupiter-engine` from 5.12.2 to 6.1.0
Release notes

Sourced from org.junit.jupiter:junit-jupiter-engine's releases.

JUnit 6.1.0 = Platform 6.1.0 + Jupiter 6.1.0 + Vintage 6.1.0

See Release Notes.

New Contributors

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.0.3...r6.1.0

JUnit 6.1.0-RC1 = Platform 6.1.0-RC1 + Jupiter 6.1.0-RC1 + Vintage 6.1.0-RC1

See Release Notes.

New Contributors

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.1.0-M1...r6.1.0-RC1

JUnit 6.1.0-M1 = Platform 6.1.0-M1 + Jupiter 6.1.0-M1 + Vintage 6.1.0-M1

See Release Notes.

New Contributors

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.0.0...r6.1.0-M1

JUnit 6.0.3 = Platform 6.0.3 + Jupiter 6.0.3 + Vintage 6.0.3

See Release Notes.

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.0.2...r6.0.3

JUnit 6.0.2 = Platform 6.0.2 + Jupiter 6.0.2 + Vintage 6.0.2

See Release Notes.

... (truncated)

Commits

Updates `org.junit.jupiter:junit-jupiter-api` from 5.12.2 to 6.1.0
Release notes

Sourced from org.junit.jupiter:junit-jupiter-api's releases.

JUnit 6.1.0 = Platform 6.1.0 + Jupiter 6.1.0 + Vintage 6.1.0

See Release Notes.

New Contributors

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.0.3...r6.1.0

JUnit 6.1.0-RC1 = Platform 6.1.0-RC1 + Jupiter 6.1.0-RC1 + Vintage 6.1.0-RC1

See Release Notes.

New Contributors

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.1.0-M1...r6.1.0-RC1

JUnit 6.1.0-M1 = Platform 6.1.0-M1 + Jupiter 6.1.0-M1 + Vintage 6.1.0-M1

See Release Notes.

New Contributors

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.0.0...r6.1.0-M1

JUnit 6.0.3 = Platform 6.0.3 + Jupiter 6.0.3 + Vintage 6.0.3

See Release Notes.

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.0.2...r6.0.3

JUnit 6.0.2 = Platform 6.0.2 + Jupiter 6.0.2 + Vintage 6.0.2

See Release Notes.

... (truncated)

Commits

Updates `org.junit.jupiter:junit-jupiter-params` from 5.12.2 to 6.1.0
Release notes

Sourced from org.junit.jupiter:junit-jupiter-params's releases.

JUnit 6.1.0 = Platform 6.1.0 + Jupiter 6.1.0 + Vintage 6.1.0

See Release Notes.

New Contributors

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.0.3...r6.1.0

JUnit 6.1.0-RC1 = Platform 6.1.0-RC1 + Jupiter 6.1.0-RC1 + Vintage 6.1.0-RC1

See Release Notes.

New Contributors

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.1.0-M1...r6.1.0-RC1

JUnit 6.1.0-M1 = Platform 6.1.0-M1 + Jupiter 6.1.0-M1 + Vintage 6.1.0-M1

See Release Notes.

New Contributors

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.0.0...r6.1.0-M1

JUnit 6.0.3 = Platform 6.0.3 + Jupiter 6.0.3 + Vintage 6.0.3

See Release Notes.

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.0.2...r6.0.3

JUnit 6.0.2 = Platform 6.0.2 + Jupiter 6.0.2 + Vintage 6.0.2

See Release Notes.

... (truncated)

Commits

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 81da2a2866..2e9b793562 100644 --- a/pom.xml +++ b/pom.xml @@ -95,7 +95,7 @@ under the License. 1773644827 ${project.build.directory}/generated-sources 1.9.0 - 5.12.2 + 6.1.0 2.0.18 33.6.0-jre 4.2.14.Final From a993bf1727098cd21cd4973d6bcff2b20e5ebfd0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Jun 2026 09:19:57 +0200 Subject: [PATCH 376/408] MINOR: Bump com.google.api.grpc:proto-google-common-protos from 2.71.0 to 2.72.0 (#1178) Bumps [com.google.api.grpc:proto-google-common-protos](https://github.com/googleapis/sdk-platform-java) from 2.71.0 to 2.72.0.
Commits
  • 6e1c179 chore(main): release 2.64.0 (#3954)
  • 7a2f0b0 chore: update upper bound dependencies file (#3966)
  • 1e4a7e5 chore: update googleapis commit at Fri Oct 17 02:31:11 UTC 2025 (#3951)
  • ffb557c deps: Bump grpc-java to v1.76.0 (#3942)
  • 9ad8a4d chore: remove internal/librariangen following migration to librarian repo (#3...
  • 0a1bbea chore(librariangen): Generate to use languagecontainer.Run (#3968)
  • 452d703 feat(librariangen): generate grpc stubs and resource helpers (#3967)
  • 85057e8 ci: remove librarian skipping on matrix builds (#3969)
  • a26a6d9 chore(librariangen): languagecontainer package to parse release-init request ...
  • c86b4ea ci: exclude internal/librariangen/** using dorny/paths-filter (#3961)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.api.grpc:proto-google-common-protos&package-manager=maven&previous-version=2.71.0&new-version=2.72.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml index 15b870905d..9ae402cdc4 100644 --- a/flight/flight-core/pom.xml +++ b/flight/flight-core/pom.xml @@ -134,7 +134,7 @@ under the License. com.google.api.grpc proto-google-common-protos - 2.71.0 + 2.72.0 test From 485f813d0cec4987f5b597f6cd448d40f074ccce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Jun 2026 13:14:21 +0200 Subject: [PATCH 377/408] MINOR: Bump com.github.luben:zstd-jni from 1.5.7-8 to 1.5.7-10 (#1177) Bumps [com.github.luben:zstd-jni](https://github.com/luben/zstd-jni) from 1.5.7-8 to 1.5.7-10.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.github.luben:zstd-jni&package-manager=maven&previous-version=1.5.7-8&new-version=1.5.7-10)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- compression/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compression/pom.xml b/compression/pom.xml index f3de6fc248..41cdb03796 100644 --- a/compression/pom.xml +++ b/compression/pom.xml @@ -55,7 +55,7 @@ under the License. com.github.luben zstd-jni - 1.5.7-8 + 1.5.7-10 From c836fad712c8a2ca14ac3a4b5f117a3e9fbefdc7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Jun 2026 09:43:35 +0200 Subject: [PATCH 378/408] MINOR: Bump com.fasterxml.jackson:jackson-bom from 2.21.3 to 2.22.0 (#1173) Bumps [com.fasterxml.jackson:jackson-bom](https://github.com/FasterXML/jackson-bom) from 2.21.3 to 2.22.0.
Commits
  • 112e859 [maven-release-plugin] prepare release jackson-bom-2.22.0
  • 2cae2ce Prep for 2.22.0 release
  • 7955d21 Merge branch '2.21' into 2.x
  • 8922a05 Post-release dep version bump
  • 1fa9943 [maven-release-plugin] prepare for next development iteration
  • d1abd31 [maven-release-plugin] prepare release jackson-bom-2.21.4
  • 2aaea43 Prep for 2.21.4 release
  • 902ec69 Update Woodstox/stax2-api (to 7.2.0/4.3.0)
  • 2570647 Merge branch '2.21' into 2.x
  • 9d3a9d5 Post-release dep version bump
  • Additional commits viewable in compare view

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2e9b793562..a5ed136e95 100644 --- a/pom.xml +++ b/pom.xml @@ -101,7 +101,7 @@ under the License. 4.2.14.Final 1.81.0 4.35.0 - 2.21.3 + 2.22.0 3.5.0 25.2.10 1.12.1 From e525fbfe7f94301ea6b9725472a91206be0c7bd2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Jun 2026 14:31:56 +0200 Subject: [PATCH 379/408] MINOR: Bump com.diffplug.spotless:spotless-maven-plugin from 3.4.0 to 3.6.0 (#1172) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [com.diffplug.spotless:spotless-maven-plugin](https://github.com/diffplug/spotless) from 3.4.0 to 3.6.0.
Release notes

Sourced from com.diffplug.spotless:spotless-maven-plugin's releases.

Maven Plugin v3.6.0

Added

  • Add <cacheDirectory> to <eclipse>, <greclipse>, and <eclipseCdt> for the Equo/Solstice P2 cache. (#2944)
  • EclipseJdtFormtterStep now can conditionally set compiler source/compliance options. Allows for better parsing of AST Node for newer language features and more correct sorting; e.g. records or seal classes. (#2942)

Fixed

  • <versionCatalog> no longer splits long inline tables across multiple lines — Gradle's TOML 1.0 parser cannot read multi-line inline tables. The maxLineLength option has been removed. (#2948)
  • spotless:apply no longer aborts on the first file with lints; it now formats all files and reports a single aggregated lint failure across every file, matching the Gradle plugin's behavior. (#2937)
  • <greclipse> and <eclipseCdt> now default P2 data to the Maven local repository. (#2944)
  • forbidWildcardImports and forbidModuleImports now detect imports that have leading whitespace (indentation/tabs). (#2939)

Changes

  • Improved formatting performance by eliminating redundant per-step line-ending normalization in the core formatter loop. (#2934)

Maven Plugin v3.5.1

Fixed

  • <licenseHeader> with <yearMode>SET_FROM_GIT</yearMode> no longer runs git log through a shell, eliminating a shell-injection vector when formatting files whose names contain shell metacharacters.
  • Bump transitive plexus-utils 4.0.2 -> 4.0.3 to address CVE-2025-67030. (#2919)

Maven Plugin v3.5.0

Added

  • <scalafmt> now reads the version from the version field in the scalafmt config file when no <version> is explicitly set, falling back to the built-in default only if neither is available. (#2922)
  • Add <toml> format type with <versionCatalog> step for formatting and sorting Gradle version catalog files. (#2916)
  • Add <javaparserVersion> option to <cleanthat>, allowing users to override the JavaParser version pulled in transitively by Cleanthat. (#2903)
  • Add a expandWildcardImports API for java (#2829)

Fixed

  • Preserve case of JDBI named bind params that collide with SQL keywords (e.g. :limit, :offset) in the DBeaver SQL formatter. (#2899)
  • The -Dspotless.ratchetFrom=... user property now takes priority over <ratchetFrom> configured in the plugin or in individual formatters, instead of being overridden by them. (#2896, fixes #2842)
  • Fix non-idempotent formatting when importOrder() is combined with greclipse(): a single catch-all group no longer strips blank lines that greclipse() independently inserted between import groups. (#2914)

Changes

  • Fix expandWildcardImports failing on JDK XML types such as org.xml.sax.InputSource. (#2921)
  • Use Eclipse JDT's collator-based comparison when sorting Java members to better match Eclipse save actions. (#2920)
  • Bump default cleanthat version 2.24 -> 2.25. (#2903)
  • Bump default eclipse-jdt version from 4.35 to 4.39. (#2912)
Commits
  • 71a433c Published maven/3.6.0
  • 3a0f101 Published gradle/8.6.0
  • 007e9d8 Published lib/4.6.2
  • a074d53 Allow setting the local P2 cache dir in the Spotless Gradle plugin (#2944)
  • a266fc2 Merge branch 'main' into add-cache-directory-dsl
  • e0d466e Fix: sort members treats record declarations as types (#2942)
  • 3936b6f Merge branch 'main' into main
  • 278765f fix: expandWildcardImports support pom type dependency, fix #2839 (#2935)
  • a18ddec Remove maxLineLength from versionCatalog step (#2949)
  • b91ad87 Add changelog entries for versionCatalog maxLineLength removal
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.diffplug.spotless:spotless-maven-plugin&package-manager=maven&previous-version=3.4.0&new-version=3.6.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- bom/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index 083ee1e0de..3201761e81 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -208,7 +208,7 @@ under the License. com.diffplug.spotless spotless-maven-plugin - 3.4.0 + 3.6.0 org.codehaus.mojo diff --git a/pom.xml b/pom.xml index a5ed136e95..63128589a9 100644 --- a/pom.xml +++ b/pom.xml @@ -492,7 +492,7 @@ under the License. com.diffplug.spotless spotless-maven-plugin - 3.4.0 + 3.6.0 org.codehaus.mojo From cb24576e895c0bf8e1d062e2fc82f2b53bfa2eb8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Jun 2026 10:07:43 +0200 Subject: [PATCH 380/408] MINOR: Bump io.netty:netty-bom from 4.2.14.Final to 4.2.15.Final (#1175) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [io.netty:netty-bom](https://github.com/netty/netty) from 4.2.14.Final to 4.2.15.Final.
Release notes

Sourced from io.netty:netty-bom's releases.

netty-4.2.15.Final

Security fixes

  • CVE-2026-48059: memory exhaustion in io.netty:netty-codec-haproxy (high).
  • CVE-2026-47691: DNS cache poisoning in io.netty:netty-resolver-dns (high).
  • CVE-2026-50560: DDoS in io.netty:netty-codec-http2.
  • CVE-2026-50011: memory exhaustion in io.netty:netty-codec-redis (high).
  • CVE-2026-44250: memory exhaustion in io.netty:netty-codec-redis (high).
  • CVE-2026-44890: memory exhaustion in io.netty:netty-codec-redis (high).
  • CVE-2026-50009: information disclosure and denial of service in io.netty:netty-codec-classes-quic.
  • CVE-2026-44249: IPv6 subnet filter bypass in io.netty:netty-handler (high).
  • CVE-2026-50020: request smuggling in io.netty:netty-codec-http.
  • CVE-2026-44892: memory exhaustion in io.netty:netty-codec-http3 (high).
  • CVE-2026-44893: memory leak in io.netty:netty-codec-haproxy (high).
  • CVE-2026-44894: traffic amplification in io.netty:netty-codec-classes-quic (high).
  • CVE-2026-50010: TLS hostname verification accidentally disabled in io.netty:netty-handler (high).
  • CVE-2026-45673: DNS cache poisoning in io.netty:netty-resolver-dns.
  • CVE-2026-45416: excessive memory usage from SNIHandler in io.netty:netty-handler (high).
  • CVE-2026-45536: file descriptor leak in io.netty:netty-transport-native-epoll and io.netty:netty-transport-native-kqueue.
  • CVE-2026-45674: DNS cache poisoning in io.netty:netty-resolver-dns (high).
  • CVE-2026-46340: memory exhaustion in io.netty:netty-transport-sctp (high).
  • CVE-2026-47244: denial of service in io.netty:netty-codec-http2.
  • CVE-2026-48006: memory exhaustion in io.netty:netty-codec-redis (high).
  • CVE-2026-48748: memory exhaustion in io.netty:netty-codec-http3 (high).
  • CVE-2026-48043: memory exhaustion in io.netty:netty-codec-http2.

What's Changed

New Contributors

Full Changelog: https://github.com/netty/netty/compare/netty-4.2.14.Final...netty-4.2.15.Final

Commits
  • a41f7b2 [maven-release-plugin] prepare release netty-4.2.15.Final
  • 2394530 Auto-port 4.2: MQTT: Reject malformed no-payload packets with non-zero Remain...
  • 0bd1657 Add maxWindowLog parameter to ZstdDecoder to bound memory allocation (#16850)
  • 76291f5 Fix SCTP and Redis tests (#16893)
  • e067b6e Fix revapi warnings (#16885)
  • 5a52600 Pass maxAllocation to Brotli and Zstd decoders (#16844)
  • 541add0 Merge commit from fork
  • 270800e Merge commit from fork
  • 3d45a1e Merge commit from fork
  • 75127ca Merge commit from fork
  • Additional commits viewable in compare view

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 63128589a9..877a355860 100644 --- a/pom.xml +++ b/pom.xml @@ -98,7 +98,7 @@ under the License. 6.1.0 2.0.18 33.6.0-jre - 4.2.14.Final + 4.2.15.Final 1.81.0 4.35.0 2.22.0 From 5540c4d6358cd653fcc7e463e60aa3f21f3288ab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 26 Jun 2026 18:01:08 +0200 Subject: [PATCH 381/408] MINOR: Bump com.squareup.okhttp3:mockwebserver3-junit5 from 5.3.2 to 5.4.0 (#1185) Bumps [com.squareup.okhttp3:mockwebserver3-junit5](https://github.com/square/okhttp) from 5.3.2 to 5.4.0.
Changelog

Sourced from com.squareup.okhttp3:mockwebserver3-junit5's changelog.

Version 5.4.0

2026-06-08

  • New: Add superpowers to interceptors. Interceptors can now override anything settable on OkHttpClient.Builder, such as the cache, connection pool, socket factory, and DNS. We expect this will allow most users to use interceptors everywhere, insted of mixing and matching interceptors with custom Call.Factory wrappers.
  • Fix: Limit each HTTP/2 response to 256 KiB of total headers.
  • Upgrade: [kotlinx.coroutines 1.11.0][coroutines_1_11_0]. This is used by the optional okhttp-coroutines artifact.
  • Upgrade: [GraalVM 25.0.3][graalvm_25].
  • Upgrade: [Okio 3.17.0][okio_3_17_0].
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.squareup.okhttp3:mockwebserver3-junit5&package-manager=maven&previous-version=5.3.2&new-version=5.4.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 3741ee083c..9a8406cf31 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -129,7 +129,7 @@ under the License. com.squareup.okhttp3 mockwebserver3-junit5 - 5.3.2 + 5.4.0 test From 6e0b5f7905c436ef847eaba4a703564ee6faae7e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 26 Jun 2026 18:01:26 +0200 Subject: [PATCH 382/408] MINOR: Bump org.jacoco:jacoco-maven-plugin from 0.8.14 to 0.8.15 (#1176) Bumps [org.jacoco:jacoco-maven-plugin](https://github.com/jacoco/jacoco) from 0.8.14 to 0.8.15.
Release notes

Sourced from org.jacoco:jacoco-maven-plugin's releases.

0.8.15

New Features

  • JaCoCo now officially supports Java 26 (GitHub #2076).
  • Experimental support for Java 27 class files (GitHub #2004).
  • Compatibility methods generated by Kotlin compiler for functions defined in interfaces are filtered out during generation of report (GitHub #1905).
  • Compatibility methods generated by Kotlin compiler for exposed boxed inline value classes (JvmExposeBoxed annotation) are filtered out during generation of report (GitHub #1944).
  • Methods generated by the Kotlin compiler for functions with JvmStatic annotation are filtered out during generation of report (GitHub #2097).
  • Improved filtering of bytecode generated by Kotlin compiler for when expressions and statements with kotlin.String subject where first branch condition contains string with largest hash (GitHub #2098).
  • Part of bytecode that javac versions from 24 to 26 generate for switch statements and expressions with selector expression of type java.lang.String inside lambdas is filtered out during generation of report (GitHub #2023).
  • Improved performance of Kotlin files analysis by parsing SMAPs only once per class (GitHub #2114).
  • For better performance agent output methods tcpclient and tcpserver use BufferedOutputStream to write execution data to socket. Maven plugin, Ant tasks, CLI, API usage examples, and ExecDumpClient API use BufferedInputStream to read execution data from socket. Third-party integrations should do the same to benefit from this change in agent (GitHub #2089).

Fixed bugs

  • Fixed processing of Kotlin SMAP in synthetic classes (GitHub #1985).
  • Multiple JaCoCo runtimes within one JVM writing to the same output file should not cause data corruption when running on JDK versions from 6 to 10 affected by JDK-8166253 (GitHub #2065, #2074).
  • For better performance agent writes to output file via BufferedOutputStream, this fixes regression introduced in version 0.6.2 (GitHub #2073).
  • Fixed NullPointerException when JaCoCo agent is loaded by non system class loader, for example when loaded by JBoss Modules (GitHub #1651).

Non-functional Changes

  • JaCoCo now depends on ASM 9.10.1 (GitHub #2134).
Commits
  • 6c5260a Prepare release v0.8.15
  • 5c05141 Transfer of execution data through socket should use buffered stream (#2089)
  • ab5efa9 Remove from Azure Pipelines all builds except with JDK 5 and JDK EA (#2148)
  • 5f6ea38 Use Windows 2025 image in GitHub Actions (#2130)
  • 35a8af2 Use Renovate instead of Dependabot for updates of ASM (#2137)
  • 85b8ddf Upgrade ASM to 9.10.1 (#2134)
  • 2988647 AgentModule should use ClassLoader of agent instead of SystemClassLoader (#1651)
  • 75a4e31 Add filter for Kotlin @JvmExposeBoxed (#1944)
  • 691fa1d Use Renovate instead of Dependabot for updates of GitHub Actions (#2132)
  • 3e18f17 Require at least JDK 21 for build (#2128)
  • Additional commits viewable in compare view

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 877a355860..326b26ee97 100644 --- a/pom.xml +++ b/pom.xml @@ -352,7 +352,7 @@ under the License. org.jacoco jacoco-maven-plugin - 0.8.14 + 0.8.15

... (truncated)

Commits
  • 7b5e9ff Bump version to 1.82.1
  • 20768f1 Update README etc to reference 1.82.1
  • 5ab5eba kokoro: Remove extra / in architecture replacement
  • 6726caf buildscripts: add regional td config for psm-interop (v1.82.x backport) (#12864)
  • 022256f Bump version to 1.82.1-SNAPSHOT
  • 78fb519 Bump version to 1.82.0
  • b62b0fc Update README etc to reference 1.82.0
  • 8802dc3 build: downgrade multiarch to Ubuntu 20.04 and consolidate images (#12830)
  • be300bd kokoro: Avoid brew on Mac OS
  • 4111f6f core: throw IOException when ProxySelector returns null or empty list (#12793)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=io.grpc:grpc-bom&package-manager=maven&previous-version=1.81.0&new-version=1.82.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 326b26ee97..3aa3867b90 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,7 @@ under the License. 2.0.18 33.6.0-jre 4.2.15.Final - 1.81.0 + 1.82.1 4.35.0 2.22.0 3.5.0 From 2812cfaa4d455f202c9e11a1facdd83acb82530b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jun 2026 09:54:40 +0200 Subject: [PATCH 384/408] MINOR: Bump dep.junit.jupiter.version from 6.1.0 to 6.1.1 (#1200) Bumps `dep.junit.jupiter.version` from 6.1.0 to 6.1.1. Updates `org.junit.jupiter:junit-jupiter-engine` from 6.1.0 to 6.1.1
Release notes

Sourced from org.junit.jupiter:junit-jupiter-engine's releases.

JUnit 6.1.1 = Platform 6.1.1 + Jupiter 6.1.1 + Vintage 6.1.1

See Release Notes.

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.1.0...r6.1.1

Commits
  • 0d85889 Release 6.1.1
  • 0363eee Finalize 6.1.1 release notes
  • a6d540a Move entry to 6.1.1 release notes
  • 69339d5 Only pass timeout when publishing to avoid failure in nmcp plugin
  • dec2eb9 Allow excluding engines from memory cleanup mode (#5786)
  • a5f4270 Publish sha256/sha512 checksums again but filter out signature ones (#5796)
  • 8213012 Update plugin nmcp-settings to v1.6.0 (#5787)
  • d1bf847 Generate Javadoc for aggregator modules
  • d721de5 Pass --no-fonts to javadoc convention
  • d289ec6 Restore original SetSystemProperty values in a ParameterizedTest (#5720)
  • Additional commits viewable in compare view

Updates `org.junit.jupiter:junit-jupiter-api` from 6.1.0 to 6.1.1
Release notes

Sourced from org.junit.jupiter:junit-jupiter-api's releases.

JUnit 6.1.1 = Platform 6.1.1 + Jupiter 6.1.1 + Vintage 6.1.1

See Release Notes.

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.1.0...r6.1.1

Commits
  • 0d85889 Release 6.1.1
  • 0363eee Finalize 6.1.1 release notes
  • a6d540a Move entry to 6.1.1 release notes
  • 69339d5 Only pass timeout when publishing to avoid failure in nmcp plugin
  • dec2eb9 Allow excluding engines from memory cleanup mode (#5786)
  • a5f4270 Publish sha256/sha512 checksums again but filter out signature ones (#5796)
  • 8213012 Update plugin nmcp-settings to v1.6.0 (#5787)
  • d1bf847 Generate Javadoc for aggregator modules
  • d721de5 Pass --no-fonts to javadoc convention
  • d289ec6 Restore original SetSystemProperty values in a ParameterizedTest (#5720)
  • Additional commits viewable in compare view

Updates `org.junit.jupiter:junit-jupiter-params` from 6.1.0 to 6.1.1
Release notes

Sourced from org.junit.jupiter:junit-jupiter-params's releases.

JUnit 6.1.1 = Platform 6.1.1 + Jupiter 6.1.1 + Vintage 6.1.1

See Release Notes.

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.1.0...r6.1.1

Commits
  • 0d85889 Release 6.1.1
  • 0363eee Finalize 6.1.1 release notes
  • a6d540a Move entry to 6.1.1 release notes
  • 69339d5 Only pass timeout when publishing to avoid failure in nmcp plugin
  • dec2eb9 Allow excluding engines from memory cleanup mode (#5786)
  • a5f4270 Publish sha256/sha512 checksums again but filter out signature ones (#5796)
  • 8213012 Update plugin nmcp-settings to v1.6.0 (#5787)
  • d1bf847 Generate Javadoc for aggregator modules
  • d721de5 Pass --no-fonts to javadoc convention
  • d289ec6 Restore original SetSystemProperty values in a ParameterizedTest (#5720)
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3aa3867b90..fa493e066d 100644 --- a/pom.xml +++ b/pom.xml @@ -95,7 +95,7 @@ under the License. 1773644827 ${project.build.directory}/generated-sources 1.9.0 - 6.1.0 + 6.1.1 2.0.18 33.6.0-jre 4.2.15.Final From bf0b3445c5cac4f964665a5f28b4a80c4ad8d640 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jun 2026 16:57:05 +0900 Subject: [PATCH 385/408] MINOR: [CI] Bump actions/cache from 5 to 6 (#1197) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/cache](https://github.com/actions/cache) from 5 to 6.
Release notes

Sourced from actions/cache's releases.

v6.0.0

What's Changed

Full Changelog: https://github.com/actions/cache/compare/v5...v6.0.0

v5.1.0

What's Changed

Full Changelog: https://github.com/actions/cache/compare/v5...v5.1.0

v5.0.5

What's Changed

Full Changelog: https://github.com/actions/cache/compare/v5...v5.0.5

v5.0.4

What's Changed

New Contributors

Full Changelog: https://github.com/actions/cache/compare/v5...v5.0.4

v5.0.3

What's Changed

Full Changelog: https://github.com/actions/cache/compare/v5...v5.0.3

v.5.0.2

v5.0.2

What's Changed

... (truncated)

Commits
  • 55cc834 Merge pull request #1768 from jasongin/readonly-cache
  • d8cd72f Bump @​actions/cache to v6.1.0 - handle cache write error due to RO token
  • 2c8a9bd Merge pull request #1760 from actions/samirat/esm_migration_and_package_update
  • e9b91fd Prettier fixes
  • e4884b8 Rebuild dist
  • 10baf01 Fixed licenses
  • e39b386 Fix test mock return order
  • b692820 PR feedback
  • 6074912 Rebuild dist bundles as ESM to match type:module
  • 5a912e8 Fix lint and jest issues
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/cache&package-manager=github_actions&previous-version=5&new-version=6)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/dev.yml | 2 +- .github/workflows/rc.yml | 8 ++++---- .github/workflows/test.yml | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 2111f47254..f23c4e5e2f 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -42,7 +42,7 @@ jobs: with: python-version: '3.x' - name: pre-commit (cache) - uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cache/pre-commit key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index f0e99f1219..79c4476345 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -133,7 +133,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Cache - uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: .docker key: jni-linux-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} @@ -264,7 +264,7 @@ jobs: run: | echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} - name: Cache ccache - uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ccache key: jni-macos-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} @@ -340,7 +340,7 @@ jobs: run: | echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} - name: Cache ccache - uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ccache key: jni-windows-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} @@ -414,7 +414,7 @@ jobs: repository: apache/arrow-testing path: testing - name: Cache ~/.m2 - uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.m2 key: binaries-build-${{ hashFiles('**/*.java', '**/pom.xml') }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8c437a056d..1e4664237b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,7 +63,7 @@ jobs: fetch-depth: 0 submodules: recursive - name: Cache Docker Volumes - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: .docker key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('compose.yaml', '**/pom.xml', '**/*.java') }} @@ -190,7 +190,7 @@ jobs: run: | ci/scripts/util_free_space.sh - name: Cache Docker Volumes - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: .docker key: integration-conda-${{ hashFiles('cpp/**') }} From 365a5f46d1fe437d2337d705dd6a0043f2ac1703 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jun 2026 18:45:32 +0200 Subject: [PATCH 386/408] MINOR: [CI] Bump actions/checkout from 6 to 7 (#1193) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/checkout](https://github.com/actions/checkout) from 6 to 7.
Release notes

Sourced from actions/checkout's releases.

v7.0.0

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v6.0.3...v7.0.0

v6.0.3

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v6...v6.0.3

v6.0.2

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v6.0.1...v6.0.2

v6.0.1

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v6...v6.0.1

Changelog

Sourced from actions/checkout's changelog.

Changelog

v7.0.0

v6.0.3

v6.0.2

v6.0.1

v6.0.0

v5.0.1

v5.0.0

v4.3.1

v4.3.0

v4.2.2

v4.2.1

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/checkout&package-manager=github_actions&previous-version=6&new-version=7)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/dev.yml | 2 +- .github/workflows/dev_pr.yml | 2 +- .github/workflows/rc.yml | 20 ++++++++++---------- .github/workflows/release.yml | 2 +- .github/workflows/test.yml | 20 ++++++++++---------- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index f23c4e5e2f..25b08700fd 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -33,7 +33,7 @@ jobs: name: "pre-commit" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml index 20946c8185..ad000df88e 100644 --- a/.github/workflows/dev_pr.yml +++ b/.github/workflows/dev_pr.yml @@ -43,7 +43,7 @@ jobs: name: "Ensure PR format" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 79c4476345..fdfcd1cce3 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -38,7 +38,7 @@ jobs: timeout-minutes: 5 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: submodules: recursive - name: Prepare for tag @@ -113,17 +113,17 @@ jobs: ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ if: github.event_name == 'schedule' - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: apache/arrow path: arrow - name: Checkout apache/arrow-testing - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: apache/arrow-testing path: arrow/testing - name: Checkout apache/parquet-testing - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing @@ -180,17 +180,17 @@ jobs: ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ if: github.event_name == 'schedule' - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: apache/arrow path: arrow - name: Checkout apache/arrow-testing - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: apache/arrow-testing path: arrow/testing - name: Checkout apache/parquet-testing - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing @@ -310,7 +310,7 @@ jobs: ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ if: github.event_name == 'schedule' - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: apache/arrow path: arrow @@ -409,7 +409,7 @@ jobs: test -f jni/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll test -f jni/arrow_orc_jni/x86_64/arrow_orc_jni.dll - name: Checkout apache/arrow-testing - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: apache/arrow-testing path: testing @@ -497,7 +497,7 @@ jobs: contents: write steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: path: site - name: Prepare branch diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a2c5a55544..7692eb6cbe 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -65,7 +65,7 @@ jobs: $artifact done - name: Checkout for publishing docs - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: path: site - name: Publish docs diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1e4664237b..ac8080d075 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,7 +58,7 @@ jobs: MAVEN: ${{ matrix.maven }} steps: - name: Checkout Arrow - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: fetch-depth: 0 submodules: recursive @@ -100,7 +100,7 @@ jobs: distribution: 'temurin' java-version: ${{ matrix.jdk }} - name: Checkout Arrow - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: fetch-depth: 0 submodules: recursive @@ -131,7 +131,7 @@ jobs: java-version: ${{ matrix.jdk }} distribution: 'temurin' - name: Checkout Arrow - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: fetch-depth: 0 submodules: recursive @@ -152,37 +152,37 @@ jobs: timeout-minutes: 60 steps: - name: Checkout Arrow - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: fetch-depth: 0 repository: apache/arrow submodules: recursive - name: Checkout Arrow Rust - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: apache/arrow-rs path: rust - name: Checkout Arrow nanoarrow - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: apache/arrow-nanoarrow path: nanoarrow - name: Checkout Arrow .NET - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: apache/arrow-dotnet path: dotnet - name: Checkout Arrow Go - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: apache/arrow-go path: go - name: Checkout Arrow Java - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: path: java - name: Checkout Arrow JavaScript - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: apache/arrow-js path: js From 9122af2ad87c6f51d8ed9082ba2f0a122980ea2a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jun 2026 18:45:52 +0200 Subject: [PATCH 387/408] MINOR: Bump com.google.protobuf:protobuf-bom from 4.35.0 to 4.35.1 (#1186) Bumps [com.google.protobuf:protobuf-bom](https://github.com/protocolbuffers/protobuf) from 4.35.0 to 4.35.1.
Commits

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fa493e066d..c39eafc209 100644 --- a/pom.xml +++ b/pom.xml @@ -100,7 +100,7 @@ under the License. 33.6.0-jre 4.2.15.Final 1.82.1 - 4.35.0 + 4.35.1 2.22.0 3.5.0 25.2.10 From b0e51af855ef5befac903b1a6de3f535258ccd1b Mon Sep 17 00:00:00 2001 From: YangJie Date: Tue, 30 Jun 2026 12:40:01 +0800 Subject: [PATCH 388/408] GH-1116: [Java] Fix compressed buffer prefix write and ZSTD dstCapacity (#1119) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Two fixes in the compression codec: 1. **`AbstractCompressionCodec.compress()`: capture `writerIndex()` once** The previous code read `uncompressedBuffer.writerIndex()` at multiple sites — for the size comparison and again after `doCompress()` to populate the 8-byte uncompressed-length prefix. Capture the value once at the top of `compress()` and reuse it for the empty-buffer check, the size comparison, and the prefix, so all three consumers see the same value. 2. **`ZstdCompressionCodec.doCompress()`: `dstCapacity` overstated by 8 bytes** `Zstd.compressUnsafe(dst, dstSize, ...)` expects `dstSize` to be the available space from `dst`. The code offsets `dst` by 8 bytes past the prefix but passed `8 + maxSize` instead of `maxSize`. The `compressBound()` headroom hides this in practice, but the parameter was semantically wrong. Pass `maxSize`. ## Tests Covered by the existing round-trip tests (`testEmptyBuffer`, `testReadWriteStream`, `testReadWriteFile`, etc.). I was not able to construct a minimal reproducer for the original `declaredUncompressed=0` symptom on the unfixed code, so both fixes are conservative correctness improvements derived from code inspection rather than failing-then-green regression tests. --- .../org/apache/arrow/compression/ZstdCompressionCodec.java | 2 +- .../arrow/vector/compression/AbstractCompressionCodec.java | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/compression/src/main/java/org/apache/arrow/compression/ZstdCompressionCodec.java b/compression/src/main/java/org/apache/arrow/compression/ZstdCompressionCodec.java index 290723608d..ed46fe81b4 100644 --- a/compression/src/main/java/org/apache/arrow/compression/ZstdCompressionCodec.java +++ b/compression/src/main/java/org/apache/arrow/compression/ZstdCompressionCodec.java @@ -44,7 +44,7 @@ protected ArrowBuf doCompress(BufferAllocator allocator, ArrowBuf uncompressedBu long bytesWritten = Zstd.compressUnsafe( compressedBuffer.memoryAddress() + CompressionUtil.SIZE_OF_UNCOMPRESSED_LENGTH, - dstSize, + maxSize, /*src*/ uncompressedBuffer.memoryAddress(), /* srcSize= */ uncompressedBuffer.writerIndex(), /* level= */ this.compressionLevel); diff --git a/vector/src/main/java/org/apache/arrow/vector/compression/AbstractCompressionCodec.java b/vector/src/main/java/org/apache/arrow/vector/compression/AbstractCompressionCodec.java index 58d9e4db9b..b108173c82 100644 --- a/vector/src/main/java/org/apache/arrow/vector/compression/AbstractCompressionCodec.java +++ b/vector/src/main/java/org/apache/arrow/vector/compression/AbstractCompressionCodec.java @@ -29,7 +29,11 @@ public abstract class AbstractCompressionCodec implements CompressionCodec { @Override public ArrowBuf compress(BufferAllocator allocator, ArrowBuf uncompressedBuffer) { - if (uncompressedBuffer.writerIndex() == 0L) { + // GH-1116: capture writerIndex() once so the empty-buffer check, size + // comparison, and uncompressed-length prefix all see the same value. + long uncompressedLength = uncompressedBuffer.writerIndex(); + + if (uncompressedLength == 0L) { // shortcut for empty buffer ArrowBuf compressedBuffer = allocator.buffer(CompressionUtil.SIZE_OF_UNCOMPRESSED_LENGTH); compressedBuffer.setLong(0, 0); @@ -41,7 +45,6 @@ public ArrowBuf compress(BufferAllocator allocator, ArrowBuf uncompressedBuffer) ArrowBuf compressedBuffer = doCompress(allocator, uncompressedBuffer); long compressedLength = compressedBuffer.writerIndex() - CompressionUtil.SIZE_OF_UNCOMPRESSED_LENGTH; - long uncompressedLength = uncompressedBuffer.writerIndex(); if (compressedLength > uncompressedLength) { // compressed buffer is larger, send the raw buffer From 301d3d8f889d3e4d1f45fc6e1707460ae5baaeb4 Mon Sep 17 00:00:00 2001 From: Pedro Matias Date: Tue, 30 Jun 2026 05:41:30 +0100 Subject: [PATCH 389/408] GH-1063: Add is_update field to ActionCreatePreparedStatementResult (#1064) ## What's Changed A new field, `optional bool is_update = 4;`, was added to `message ActionCreatePreparedStatementResult`. When this field is sent by the server, its value indicates whether the proper network flow to execute the query that the driver should follow uses `CommandPreparedStatementQuery` or `CommandPreparedStatementUpdate`. For outdated servers that don't send the field, the driver maintains its current behavior of using `CommandPreparedStatementQuery` when the `dataset_schema` is not empty, thus ensuring the backward compatibility of the new driver with old servers. This change was created with AI assistance (Augment Code and Claude code). All lines were manually reviewed by a human. The output is not copyrightable subject matter. - Closes #1063 --------- Co-authored-by: David Li --- arrow-format/FlightSql.proto | 5 ++ .../ArrowFlightJdbcFlightStreamResultSet.java | 2 +- ...owFlightJdbcVectorSchemaRootResultSet.java | 2 +- .../driver/jdbc/ArrowFlightMetaImpl.java | 23 +++++-- .../client/ArrowFlightSqlClientHandler.java | 19 ++++++ .../ArrowFlightPreparedStatementTest.java | 48 ++++++++++++++ .../jdbc/ArrowFlightStatementExecuteTest.java | 66 +++++++++++++++++++ .../jdbc/utils/MockFlightSqlProducer.java | 42 ++++++++++++ .../arrow/flight/sql/FlightSqlClient.java | 13 ++++ 9 files changed, 212 insertions(+), 8 deletions(-) diff --git a/arrow-format/FlightSql.proto b/arrow-format/FlightSql.proto index 566230c2a6..b1dc57b33b 100644 --- a/arrow-format/FlightSql.proto +++ b/arrow-format/FlightSql.proto @@ -1550,6 +1550,11 @@ message ActionCreatePreparedStatementResult { // If the query provided contained parameters, parameter_schema contains the // schema of the expected parameters. It should be an IPC-encapsulated Schema, as described in Schema.fbs. bytes parameter_schema = 3; + + // When set to true, the query should be executed with CommandPreparedStatementUpdate, + // when set to false, the query should be executed with CommandPreparedStatementQuery. + // If not set, the client can choose how to execute the query. + optional bool is_update = 4; } /* diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcFlightStreamResultSet.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcFlightStreamResultSet.java index 2885f7895b..376e5b11e7 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcFlightStreamResultSet.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcFlightStreamResultSet.java @@ -106,7 +106,7 @@ static ArrowFlightJdbcFlightStreamResultSet fromFlightInfo( final TimeZone timeZone = TimeZone.getDefault(); final QueryState state = new QueryState(); - final Meta.Signature signature = ArrowFlightMetaImpl.newSignature(null, null, null); + final Meta.Signature signature = ArrowFlightMetaImpl.newSignature(null, null, null, null); final AvaticaResultSetMetaData resultSetMetaData = new AvaticaResultSetMetaData(null, null, signature); diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java index 5d02d6e843..ad6670a001 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java @@ -73,7 +73,7 @@ public static ArrowFlightJdbcVectorSchemaRootResultSet fromVectorSchemaRoot( final TimeZone timeZone = TimeZone.getDefault(); final QueryState state = new QueryState(); - final Meta.Signature signature = ArrowFlightMetaImpl.newSignature(null, null, null); + final Meta.Signature signature = ArrowFlightMetaImpl.newSignature(null, null, null, null); final AvaticaResultSetMetaData resultSetMetaData = new AvaticaResultSetMetaData(null, null, signature); diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java index 64529b50c8..0d85b5eddb 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java @@ -53,7 +53,8 @@ public ArrowFlightMetaImpl(final AvaticaConnection connection) { } /** Construct a signature. */ - static Signature newSignature(final String sql, Schema resultSetSchema, Schema parameterSchema) { + static Signature newSignature( + final String sql, Schema resultSetSchema, Schema parameterSchema, Boolean isUpdate) { List columnMetaData = resultSetSchema == null ? new ArrayList<>() @@ -62,10 +63,17 @@ static Signature newSignature(final String sql, Schema resultSetSchema, Schema p parameterSchema == null ? new ArrayList<>() : ConvertUtils.convertArrowFieldsToAvaticaParameters(parameterSchema.getFields()); - StatementType statementType = - resultSetSchema == null || resultSetSchema.getFields().isEmpty() - ? StatementType.IS_DML - : StatementType.SELECT; + // If the server provided the is_update field, use it to determine the statement type + StatementType statementType; + if (isUpdate != null) { + statementType = isUpdate ? StatementType.IS_DML : StatementType.SELECT; + } else { + // Fall back to the legacy logic: check if the result set schema is empty + statementType = + resultSetSchema == null || resultSetSchema.getFields().isEmpty() + ? StatementType.IS_DML + : StatementType.SELECT; + } return new Signature( columnMetaData, sql, @@ -178,7 +186,10 @@ private PreparedStatement prepareForHandle(final String query, StatementHandle h ((ArrowFlightConnection) connection).getClientHandler().prepare(query); handle.signature = newSignature( - query, preparedStatement.getDataSetSchema(), preparedStatement.getParameterSchema()); + query, + preparedStatement.getDataSetSchema(), + preparedStatement.getParameterSchema(), + preparedStatement.isUpdate()); statementHandlePreparedStatementMap.put(new StatementHandleKey(handle), preparedStatement); return preparedStatement; } diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java index f0ea284239..719cc38a2b 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java @@ -388,6 +388,14 @@ public interface PreparedStatement extends AutoCloseable { */ Schema getParameterSchema(); + /** + * Gets whether this {@link PreparedStatement} is an update statement. + * + * @return {@code true} if this is an update statement, {@code false} if it's a query, or {@code + * null} if the server did not provide this information. + */ + @Nullable Boolean isUpdate(); + void setParameters(VectorSchemaRoot parameters); @Override @@ -456,6 +464,12 @@ public long executeUpdate() { @Override public StatementType getType() { + // If the server provided the is_update field, use it to determine the statement type + final Boolean isUpdate = preparedStatement.isUpdate(); + if (isUpdate != null) { + return isUpdate ? StatementType.UPDATE : StatementType.SELECT; + } + // Fall back to the legacy logic: check if the result set schema is empty final Schema schema = preparedStatement.getResultSetSchema(); return schema.getFields().isEmpty() ? StatementType.UPDATE : StatementType.SELECT; } @@ -475,6 +489,11 @@ public void setParameters(VectorSchemaRoot parameters) { preparedStatement.setParameters(parameters); } + @Override + public Boolean isUpdate() { + return preparedStatement.isUpdate(); + } + @Override public void close() { try { diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightPreparedStatementTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightPreparedStatementTest.java index 0369c3a162..078837adf3 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightPreparedStatementTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightPreparedStatementTest.java @@ -98,6 +98,40 @@ public void testSimpleQueryNoParameterBindingWithExecute() throws SQLException { } } + @Test + public void testSimpleQueryNoParameterBindingWithExecuteV2() throws SQLException { + final String query = "SELECT * FROM TEST_V2"; + final Schema schema = + new Schema(Collections.singletonList(Field.nullable("", Types.MinorType.INT.getType()))); + PRODUCER.addSelectQuery( + query, + schema, + Collections.singletonList( + listener -> { + try (final BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { + root.allocateNew(); + ((IntVector) root.getVector(0)).setSafe(0, 123); + root.setRowCount(1); + listener.start(root); + listener.putNext(); + } finally { + listener.completed(); + } + }), + false); + try (final PreparedStatement preparedStatement = connection.prepareStatement(query)) { + boolean isResultSet = preparedStatement.execute(); + assertTrue(isResultSet); + final ResultSet resultSet = preparedStatement.getResultSet(); + assertTrue(resultSet.next()); + assertEquals(123, resultSet.getInt(1)); + assertFalse(resultSet.next()); + assertFalse(preparedStatement.getMoreResults()); + assertEquals(-1, preparedStatement.getUpdateCount()); + } + } + @Test public void testQueryWithParameterBinding() throws SQLException { final String query = "Fake query with parameters"; @@ -203,6 +237,20 @@ public void testUpdateQueryWithExecute() throws SQLException { } } + @Test + public void testUpdateQueryWithExecuteV2() throws SQLException { + String query = "Fake update with execute V2"; + PRODUCER.addUpdateQuery(query, /*updatedRows*/ 99, true); + try (final PreparedStatement stmt = connection.prepareStatement(query)) { + boolean isResultSet = stmt.execute(); + assertFalse(isResultSet); + int updated = stmt.getUpdateCount(); + assertEquals(99, updated); + assertFalse(stmt.getMoreResults()); + assertEquals(-1, stmt.getUpdateCount()); + } + } + @Test public void testUpdateQueryWithParameters() throws SQLException { String query = "Fake update with parameters"; diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightStatementExecuteTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightStatementExecuteTest.java index 632cb0ba56..6acce9c2a6 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightStatementExecuteTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightStatementExecuteTest.java @@ -62,6 +62,9 @@ public class ArrowFlightStatementExecuteTest { private static final String SAMPLE_LARGE_UPDATE_QUERY = "UPDATE this_large_table SET this_large_field = that_large_field FROM this_large_test WHERE this_large_condition"; private static final long SAMPLE_LARGE_UPDATE_COUNT = Long.MAX_VALUE; + private static final String SAMPLE_QUERY_CMD_V2 = "SELECT * FROM this_test_v2"; + private static final String SAMPLE_LARGE_UPDATE_QUERY_V2 = + "UPDATE this_large_table_v2 SET this_large_field = that_large_field FROM this_large_test WHERE this_large_condition"; private static final MockFlightSqlProducer PRODUCER = new MockFlightSqlProducer(); @RegisterExtension @@ -96,6 +99,31 @@ public static void setUpBeforeClass() { })); PRODUCER.addUpdateQuery(SAMPLE_UPDATE_QUERY, SAMPLE_UPDATE_COUNT); PRODUCER.addUpdateQuery(SAMPLE_LARGE_UPDATE_QUERY, SAMPLE_LARGE_UPDATE_COUNT); + + // V2 queries with is_update field set + PRODUCER.addSelectQuery( + SAMPLE_QUERY_CMD_V2, + SAMPLE_QUERY_SCHEMA, + Collections.singletonList( + listener -> { + try (final BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + final VectorSchemaRoot root = + VectorSchemaRoot.create(SAMPLE_QUERY_SCHEMA, allocator)) { + final UInt1Vector vector = (UInt1Vector) root.getVector(VECTOR_NAME); + IntStream.range(0, SAMPLE_QUERY_ROWS) + .forEach(index -> vector.setSafe(index, index)); + vector.setValueCount(SAMPLE_QUERY_ROWS); + root.setRowCount(SAMPLE_QUERY_ROWS); + listener.start(root); + listener.putNext(); + } catch (final Throwable throwable) { + listener.error(throwable); + } finally { + listener.completed(); + } + }), + false); + PRODUCER.addUpdateQuery(SAMPLE_LARGE_UPDATE_QUERY_V2, SAMPLE_LARGE_UPDATE_COUNT, true); } @BeforeEach @@ -168,4 +196,42 @@ public void testUpdateCountShouldStartOnZero() throws SQLException { is(allOf(equalTo(statement.getLargeUpdateCount()), equalTo(0L)))); assertThat(statement.getResultSet(), is(nullValue())); } + + @Test + public void testExecuteShouldRunSelectQueryV2() throws SQLException { + assertThat(statement.execute(SAMPLE_QUERY_CMD_V2), is(true)); + final Set numbers = + IntStream.range(0, SAMPLE_QUERY_ROWS) + .boxed() + .map(Integer::byteValue) + .collect(Collectors.toCollection(HashSet::new)); + try (final ResultSet resultSet = statement.getResultSet()) { + final int columnCount = resultSet.getMetaData().getColumnCount(); + assertThat(columnCount, is(1)); + int rowCount = 0; + for (; resultSet.next(); rowCount++) { + assertThat(numbers.remove(resultSet.getByte(1)), is(true)); + } + assertThat(rowCount, is(equalTo(SAMPLE_QUERY_ROWS))); + } + assertThat(numbers, is(Collections.emptySet())); + assertThat( + (long) statement.getUpdateCount(), + is(allOf(equalTo(statement.getLargeUpdateCount()), equalTo(-1L)))); + } + + @Test + public void testExecuteShouldRunUpdateQueryForLargeUpdateV2() throws SQLException { + assertThat(statement.execute(SAMPLE_LARGE_UPDATE_QUERY_V2), is(false)); // UPDATE query. + final long updateCountSmall = statement.getUpdateCount(); + final long updateCountLarge = statement.getLargeUpdateCount(); + assertThat(updateCountLarge, is(equalTo(SAMPLE_LARGE_UPDATE_COUNT))); + assertThat( + updateCountSmall, + is( + allOf( + equalTo((long) AvaticaUtils.toSaturatedInt(updateCountLarge)), + not(equalTo(updateCountLarge))))); + assertThat(statement.getResultSet(), is(nullValue())); + } } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/MockFlightSqlProducer.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/MockFlightSqlProducer.java index 45c2a96404..6627d91ab6 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/MockFlightSqlProducer.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/MockFlightSqlProducer.java @@ -87,6 +87,7 @@ import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.JsonStringArrayList; import org.apache.calcite.avatica.Meta.StatementType; +import org.checkerframework.checker.nullness.qual.Nullable; /** An ad-hoc {@link FlightSqlProducer} for tests. */ public final class MockFlightSqlProducer implements FlightSqlProducer { @@ -101,6 +102,7 @@ public final class MockFlightSqlProducer implements FlightSqlProducer { private final SqlInfoBuilder sqlInfoBuilder = new SqlInfoBuilder(); private final Map parameterSchemas = new HashMap<>(); private final Map>> expectedParameterValues = new HashMap<>(); + private final Map isUpdateMap = new HashMap<>(); private final Map actionTypeCounter = new HashMap<>(); @@ -176,6 +178,40 @@ public void addUpdateQuery(final String sqlCommand, final long updatedRows) { }); } + /** + * Registers a new {@link StatementType#SELECT} SQL query, optionally setting the is_update field. + * + * @param sqlCommand the SQL command under which to register the new query. + * @param schema the schema to use for the query result. + * @param resultProviders the result provider for this query. + * @param isUpdate value to report for the is_update field, or {@code null} to leave it unset. + */ + public void addSelectQuery( + final String sqlCommand, + final Schema schema, + final List> resultProviders, + final @Nullable Boolean isUpdate) { + addSelectQuery(sqlCommand, schema, resultProviders); + if (isUpdate != null) { + isUpdateMap.put(sqlCommand, isUpdate); + } + } + + /** + * Registers a new {@link StatementType#UPDATE} SQL query, optionally setting the is_update field. + * + * @param sqlCommand the SQL command. + * @param updatedRows the number of rows affected. + * @param isUpdate value to report for the is_update field, or {@code null} to leave it unset. + */ + public void addUpdateQuery( + final String sqlCommand, final long updatedRows, final @Nullable Boolean isUpdate) { + addUpdateQuery(sqlCommand, updatedRows); + if (isUpdate != null) { + isUpdateMap.put(sqlCommand, isUpdate); + } + } + /** * Adds a catalog query to the results. * @@ -247,6 +283,12 @@ public void createPreparedStatement( resultBuilder.setParameterSchema(ByteString.copyFrom(outputStream.toByteArray())); } + // Set is_update field if present + final Boolean isUpdate = isUpdateMap.get(query); + if (isUpdate != null) { + resultBuilder.setIsUpdate(isUpdate); + } + listener.onNext(new Result(pack(resultBuilder.build()).toByteArray())); } catch (final Throwable t) { listener.onError(t); diff --git a/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlClient.java b/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlClient.java index 623f9311e8..0af09faee1 100644 --- a/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlClient.java +++ b/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlClient.java @@ -1284,6 +1284,19 @@ public Schema getParameterSchema() { return parameterSchema; } + /** + * Returns whether the server indicated this prepared statement is an update query. + * + * @return true if the server indicated this is an update query, false if the server indicated + * this is a select query, or null if the server did not provide this information. + */ + public Boolean isUpdate() { + if (preparedStatementResult.hasIsUpdate()) { + return preparedStatementResult.getIsUpdate(); + } + return null; + } + /** Get the schema of the result set (should be identical to {@link #getResultSetSchema()}). */ public SchemaResult fetchSchema(CallOption... options) { checkOpen(); From fcba1b0345faed30621863af85a102c6d2fc1362 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Jul 2026 07:51:48 +0200 Subject: [PATCH 390/408] MINOR: Bump org.cyclonedx:cyclonedx-maven-plugin from 2.9.1 to 2.9.2 (#1198) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.cyclonedx:cyclonedx-maven-plugin](https://github.com/CycloneDX/cyclonedx-maven-plugin) from 2.9.1 to 2.9.2.
Release notes

Sourced from org.cyclonedx:cyclonedx-maven-plugin's releases.

2.9.2

🚀 New features and improvements

  • chore: upgrade maven-dependency-analyzer/asm, support Java 25 (#630) @​shihyuho

📦 Dependency updates

🔧 Build

Commits
  • 0fe189d [maven-release-plugin] prepare release cyclonedx-maven-plugin-2.9.2
  • 96c218c update scm urls
  • 0fe08b4 Revert "Bump JamesIves/github-pages-deploy-action from 4.7.3 to 4.8.0"
  • 6779e48 Revert "Bump release-drafter/release-drafter from 6 to 7"
  • 955fead switch to Central Publishing Portal
  • 50dbac7 Bump release-drafter/release-drafter from 6 to 7
  • d50bc58 Bump org.apache.maven.plugins:maven-project-info-reports-plugin
  • 1034644 Bump plugin-tools.version from 3.15.0 to 3.15.2
  • 018ab8e Bump commons-codec:commons-codec from 1.17.1 to 1.22.0
  • e359705 Bump JamesIves/github-pages-deploy-action from 4.7.3 to 4.8.0
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.cyclonedx:cyclonedx-maven-plugin&package-manager=maven&previous-version=2.9.1&new-version=2.9.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c39eafc209..6a97ac4620 100644 --- a/pom.xml +++ b/pom.xml @@ -522,7 +522,7 @@ under the License. org.cyclonedx cyclonedx-maven-plugin - 2.9.1 + 2.9.2 org.apache.drill.tools From bd8cd52fc8426b485a8b3e64997fb178c84c9de2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Jul 2026 07:54:25 +0200 Subject: [PATCH 391/408] MINOR: Bump com.github.luben:zstd-jni from 1.5.7-10 to 1.5.7-11 (#1184) Bumps [com.github.luben:zstd-jni](https://github.com/luben/zstd-jni) from 1.5.7-10 to 1.5.7-11.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.github.luben:zstd-jni&package-manager=maven&previous-version=1.5.7-10&new-version=1.5.7-11)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- compression/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compression/pom.xml b/compression/pom.xml index 41cdb03796..aa7dee6f89 100644 --- a/compression/pom.xml +++ b/compression/pom.xml @@ -55,7 +55,7 @@ under the License. com.github.luben zstd-jni - 1.5.7-10 + 1.5.7-11 From 04471f4f8900e0a91efe0d6ae717c3211aa5efbd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Jul 2026 08:11:17 +0200 Subject: [PATCH 392/408] MINOR: Bump logback.version from 1.5.32 to 1.5.34 (#1171) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps `logback.version` from 1.5.32 to 1.5.34. Updates `ch.qos.logback:logback-classic` from 1.5.32 to 1.5.34
Release notes

Sourced from ch.qos.logback:logback-classic's releases.

Logback 1.5.34

2026-06-01 Release of logback version 1.5.34

• In case certain StackTraceElement values returned by the Throwable.getStackTrace method are null, StackTraceElementProxy substitutes a dummy instance instead of throwing an IllegalArgumentException. This resolves [issues #1040](qos-ch/logback#1040), reported by Naotsugu Kobayashi.

• HardenedObjectInputStream will now throw an InvalidClassException during deserialization attempts of Proxy classes. This change addresses potential deserialization whitelist bypass vulnerability reported by York Shen and registered as CVE-2026-10532.

• A bitwise identical binary of this version can be reproduced by building from source code at commit e62272ac152469aec1ede056c3c7d0d7314e7bfe associated with the tag v_1.5.34. This release was built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Logback 1.5.33

2026-05-27 Release of logback version 1.5.33

• PropertiesConfiguratorModelHandler now registers properties file URLs to the ConfigurationWatchList when scan is enabled (via local scan="true" attribute or top-level configuration scan), ensuring changes are detected and reconfiguration occurs. This problem was reported in issues/1034.

• When processing <conversionRule> elements and both class and converterClass attributes are specified, silently use the class attribute without issuing a warning. However, if the attribute values differ, a warning will be issued. This change was requested in issues/1031.

• HardenedModelInputStream will no longer accept to deserialize all classes located under the "java.lang" and "java.util" packages but a limited number of explicitly authorized classes in those packages. This potential deserialization whitelist bypass vulnerability was reported by York Shen and registered as CVE-2026-9828.

• SSL parameters for SSLSocketAppender now enable hostname verification by default. Moreover, the default protocol is now "TLSv1.2". This potential vulnerability was reported by York Shen.

• When printing the status message field, ViewStatusMessagesServletBase now escapes special characters such as "&" as character entities. This potential vulnerability was reported by York Shen.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit 124e8b49b55ac34d08743a0646bd463410192647 associated with the tag v_1.5.33. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits
  • e62272a prepare release 1.5.34
  • 1e9e926 add resolveProxyClassRejectsDynamicProxies unit test
  • 2de5cbe added StackTraceElementProxyTest, minor edits to AGENTS.md
  • 0e9b927 in case StackTraceElement is null use a substitute, fixing issues/1040
  • f7a0654 prevent resolveProxyClass bypass
  • 249b81f docs are no longer distributed
  • 1c3b26a start work on 1.5.34-SNAPSHOT
  • 124e8b4 prepare release 1.5.33
  • d8fd6f2 escapeTags in message field when printing status messages
  • 95edbeb hostnameVerification default to true in SSLParametersConfiguration, SSL.DEFAU...
  • Additional commits viewable in compare view

Updates `ch.qos.logback:logback-core` from 1.5.32 to 1.5.34
Release notes

Sourced from ch.qos.logback:logback-core's releases.

Logback 1.5.34

2026-06-01 Release of logback version 1.5.34

• In case certain StackTraceElement values returned by the Throwable.getStackTrace method are null, StackTraceElementProxy substitutes a dummy instance instead of throwing an IllegalArgumentException. This resolves [issues #1040](qos-ch/logback#1040), reported by Naotsugu Kobayashi.

• HardenedObjectInputStream will now throw an InvalidClassException during deserialization attempts of Proxy classes. This change addresses potential deserialization whitelist bypass vulnerability reported by York Shen and registered as CVE-2026-10532.

• A bitwise identical binary of this version can be reproduced by building from source code at commit e62272ac152469aec1ede056c3c7d0d7314e7bfe associated with the tag v_1.5.34. This release was built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Logback 1.5.33

2026-05-27 Release of logback version 1.5.33

• PropertiesConfiguratorModelHandler now registers properties file URLs to the ConfigurationWatchList when scan is enabled (via local scan="true" attribute or top-level configuration scan), ensuring changes are detected and reconfiguration occurs. This problem was reported in issues/1034.

• When processing <conversionRule> elements and both class and converterClass attributes are specified, silently use the class attribute without issuing a warning. However, if the attribute values differ, a warning will be issued. This change was requested in issues/1031.

• HardenedModelInputStream will no longer accept to deserialize all classes located under the "java.lang" and "java.util" packages but a limited number of explicitly authorized classes in those packages. This potential deserialization whitelist bypass vulnerability was reported by York Shen and registered as CVE-2026-9828.

• SSL parameters for SSLSocketAppender now enable hostname verification by default. Moreover, the default protocol is now "TLSv1.2". This potential vulnerability was reported by York Shen.

• When printing the status message field, ViewStatusMessagesServletBase now escapes special characters such as "&" as character entities. This potential vulnerability was reported by York Shen.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit 124e8b49b55ac34d08743a0646bd463410192647 associated with the tag v_1.5.33. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits
  • e62272a prepare release 1.5.34
  • 1e9e926 add resolveProxyClassRejectsDynamicProxies unit test
  • 2de5cbe added StackTraceElementProxyTest, minor edits to AGENTS.md
  • 0e9b927 in case StackTraceElement is null use a substitute, fixing issues/1040
  • f7a0654 prevent resolveProxyClass bypass
  • 249b81f docs are no longer distributed
  • 1c3b26a start work on 1.5.34-SNAPSHOT
  • 124e8b4 prepare release 1.5.33
  • d8fd6f2 escapeTags in message field when printing status messages
  • 95edbeb hostnameVerification default to true in SSLParametersConfiguration, SSL.DEFAU...
  • Additional commits viewable in compare view

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6a97ac4620..231720eb44 100644 --- a/pom.xml +++ b/pom.xml @@ -113,7 +113,7 @@ under the License. true 2.42.0 4.2.0 - 1.5.32 + 1.5.34 none -Xdoclint:none From 8ce39730fe969b41c8620b02f3fa77c974d90784 Mon Sep 17 00:00:00 2001 From: Jordan Epstein <32082339+jordepic@users.noreply.github.com> Date: Thu, 2 Jul 2026 18:16:18 -0500 Subject: [PATCH 393/408] GH-1179: Correct the size of var-width vector with >0 start offset during vector append (#1180) ## What's Changed Fix VectorAppender data size computation for variable-width vectors with non-zero start offsets When appending a variable width offset vector in DataFusion comet I was receiving exceptions due to allocating too much memory. This is because Comet passes variable width arrays back to Java where the initial offset vector entry is greater than 0. Prior to this change, arrow-java determines how many bytes to copy by just looking at the last offset entry in the buffer, completely disregarding the value of the first. If first = 100 and last = 200, Java will still copy 200 bytes instead of 100. In this change we fix that. Closes #1179 --------- Co-authored-by: Jordan Epstein --- .../arrow/vector/util/VectorAppender.java | 92 ++++++--- .../arrow/vector/util/TestVectorAppender.java | 189 ++++++++++++++++++ 2 files changed, 257 insertions(+), 24 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java b/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java index e7c0d11cb9..2cfeb0a04d 100644 --- a/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java +++ b/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java @@ -125,10 +125,15 @@ public ValueVector visit(BaseVariableWidthVector deltaVector, Void value) { targetVector .getOffsetBuffer() .getInt((long) targetVector.getValueCount() * BaseVariableWidthVector.OFFSET_WIDTH); + // The delta vector's offset buffer need not start at zero (e.g. a vector imported through + // the C data interface from a sliced array), so the amount of data to append is the + // distance between its first and last offsets, not the last offset itself. + int deltaDataStart = deltaVector.getOffsetBuffer().getInt(0); int deltaDataSize = deltaVector - .getOffsetBuffer() - .getInt((long) deltaVector.getValueCount() * BaseVariableWidthVector.OFFSET_WIDTH); + .getOffsetBuffer() + .getInt((long) deltaVector.getValueCount() * BaseVariableWidthVector.OFFSET_WIDTH) + - deltaDataStart; int newValueCapacity = targetDataSize + deltaDataSize; // make sure there is enough capacity @@ -149,7 +154,7 @@ public ValueVector visit(BaseVariableWidthVector deltaVector, Void value) { // append data buffer MemoryUtil.copyMemory( - deltaVector.getDataBuffer().memoryAddress(), + deltaVector.getDataBuffer().memoryAddress() + deltaDataStart, targetVector.getDataBuffer().memoryAddress() + targetDataSize, deltaDataSize); @@ -160,7 +165,7 @@ public ValueVector visit(BaseVariableWidthVector deltaVector, Void value) { + (targetVector.getValueCount() + 1) * BaseVariableWidthVector.OFFSET_WIDTH, deltaVector.getValueCount() * BaseVariableWidthVector.OFFSET_WIDTH); - // increase each offset from the second buffer + // rebase each appended offset to the target's data, accounting for the delta's start offset for (int i = 0; i < deltaVector.getValueCount(); i++) { int oldOffset = targetVector @@ -172,7 +177,7 @@ public ValueVector visit(BaseVariableWidthVector deltaVector, Void value) { .getOffsetBuffer() .setInt( (long) (targetVector.getValueCount() + 1 + i) * BaseVariableWidthVector.OFFSET_WIDTH, - oldOffset + targetDataSize); + oldOffset - deltaDataStart + targetDataSize); } ((BaseVariableWidthVector) targetVector).setLastSet(newValueCount - 1); targetVector.setValueCount(newValueCount); @@ -196,11 +201,15 @@ public ValueVector visit(BaseLargeVariableWidthVector deltaVector, Void value) { .getOffsetBuffer() .getLong( (long) targetVector.getValueCount() * BaseLargeVariableWidthVector.OFFSET_WIDTH); + // see the corresponding comment in visit(BaseVariableWidthVector, Void): the delta's + // offset buffer need not start at zero + long deltaDataStart = deltaVector.getOffsetBuffer().getLong(0); long deltaDataSize = deltaVector - .getOffsetBuffer() - .getLong( - (long) deltaVector.getValueCount() * BaseLargeVariableWidthVector.OFFSET_WIDTH); + .getOffsetBuffer() + .getLong( + (long) deltaVector.getValueCount() * BaseLargeVariableWidthVector.OFFSET_WIDTH) + - deltaDataStart; long newValueCapacity = targetDataSize + deltaDataSize; // make sure there is enough capacity @@ -221,7 +230,7 @@ public ValueVector visit(BaseLargeVariableWidthVector deltaVector, Void value) { // append data buffer MemoryUtil.copyMemory( - deltaVector.getDataBuffer().memoryAddress(), + deltaVector.getDataBuffer().memoryAddress() + deltaDataStart, targetVector.getDataBuffer().memoryAddress() + targetDataSize, deltaDataSize); @@ -232,7 +241,7 @@ public ValueVector visit(BaseLargeVariableWidthVector deltaVector, Void value) { + (targetVector.getValueCount() + 1) * BaseLargeVariableWidthVector.OFFSET_WIDTH, deltaVector.getValueCount() * BaseLargeVariableWidthVector.OFFSET_WIDTH); - // increase each offset from the second buffer + // rebase each appended offset to the target's data, accounting for the delta's start offset for (int i = 0; i < deltaVector.getValueCount(); i++) { long oldOffset = targetVector @@ -245,7 +254,7 @@ public ValueVector visit(BaseLargeVariableWidthVector deltaVector, Void value) { .setLong( (long) (targetVector.getValueCount() + 1 + i) * BaseLargeVariableWidthVector.OFFSET_WIDTH, - oldOffset + targetDataSize); + oldOffset - deltaDataStart + targetDataSize); } ((BaseLargeVariableWidthVector) targetVector).setLastSet(newValueCount - 1); targetVector.setValueCount(newValueCount); @@ -331,16 +340,20 @@ public ValueVector visit(ListVector deltaVector, Void value) { targetVector .getOffsetBuffer() .getInt((long) targetVector.getValueCount() * ListVector.OFFSET_WIDTH); - int deltaListSize = + // see the corresponding comment in visit(BaseVariableWidthVector, Void): the delta's + // offset buffer need not start at zero + int deltaListStart = deltaVector.getOffsetBuffer().getInt(0); + int deltaListEnd = deltaVector .getOffsetBuffer() .getInt((long) deltaVector.getValueCount() * ListVector.OFFSET_WIDTH); + int deltaListSize = deltaListEnd - deltaListStart; ListVector targetListVector = (ListVector) targetVector; // make sure the underlying vector has value count set targetListVector.getDataVector().setValueCount(targetListSize); - deltaVector.getDataVector().setValueCount(deltaListSize); + deltaVector.getDataVector().setValueCount(deltaListEnd); // make sure there is enough capacity while (targetVector.getValueCapacity() < newValueCount) { @@ -372,13 +385,16 @@ public ValueVector visit(ListVector deltaVector, Void value) { .getOffsetBuffer() .setInt( (long) (targetVector.getValueCount() + 1 + i) * ListVector.OFFSET_WIDTH, - oldOffset + targetListSize); + oldOffset - deltaListStart + targetListSize); } targetListVector.setLastSet(newValueCount - 1); // append underlying vectors - VectorAppender innerAppender = new VectorAppender(targetListVector.getDataVector()); - deltaVector.getDataVector().accept(innerAppender, null); + appendDataVector( + targetListVector.getDataVector(), + deltaVector.getDataVector(), + deltaListStart, + deltaListSize); targetVector.setValueCount(newValueCount); return targetVector; @@ -400,17 +416,21 @@ public ValueVector visit(LargeListVector deltaVector, Void value) { targetVector .getOffsetBuffer() .getLong((long) targetVector.getValueCount() * LargeListVector.OFFSET_WIDTH); - long deltaListSize = + // see the corresponding comment in visit(BaseVariableWidthVector, Void): the delta's + // offset buffer need not start at zero + long deltaListStart = deltaVector.getOffsetBuffer().getLong(0); + long deltaListEnd = deltaVector .getOffsetBuffer() .getLong((long) deltaVector.getValueCount() * LargeListVector.OFFSET_WIDTH); + long deltaListSize = deltaListEnd - deltaListStart; - ListVector targetListVector = (ListVector) targetVector; + LargeListVector targetListVector = (LargeListVector) targetVector; // make sure the underlying vector has value count set // todo recheck these casts when int64 vectors are supported targetListVector.getDataVector().setValueCount(checkedCastToInt(targetListSize)); - deltaVector.getDataVector().setValueCount(checkedCastToInt(deltaListSize)); + deltaVector.getDataVector().setValueCount(checkedCastToInt(deltaListEnd)); // make sure there is enough capacity while (targetVector.getValueCapacity() < newValueCount) { @@ -427,10 +447,10 @@ public ValueVector visit(LargeListVector deltaVector, Void value) { // append offset buffer MemoryUtil.copyMemory( - deltaVector.getOffsetBuffer().memoryAddress() + ListVector.OFFSET_WIDTH, + deltaVector.getOffsetBuffer().memoryAddress() + LargeListVector.OFFSET_WIDTH, targetVector.getOffsetBuffer().memoryAddress() + (targetVector.getValueCount() + 1) * LargeListVector.OFFSET_WIDTH, - (long) deltaVector.getValueCount() * ListVector.OFFSET_WIDTH); + (long) deltaVector.getValueCount() * LargeListVector.OFFSET_WIDTH); // increase each offset from the second buffer for (int i = 0; i < deltaVector.getValueCount(); i++) { @@ -443,18 +463,42 @@ public ValueVector visit(LargeListVector deltaVector, Void value) { .getOffsetBuffer() .setLong( (long) (targetVector.getValueCount() + 1 + i) * LargeListVector.OFFSET_WIDTH, - oldOffset + targetListSize); + oldOffset - deltaListStart + targetListSize); } targetListVector.setLastSet(newValueCount - 1); // append underlying vectors - VectorAppender innerAppender = new VectorAppender(targetListVector.getDataVector()); - deltaVector.getDataVector().accept(innerAppender, null); + appendDataVector( + targetListVector.getDataVector(), + deltaVector.getDataVector(), + checkedCastToInt(deltaListStart), + checkedCastToInt(deltaListSize)); targetVector.setValueCount(newValueCount); return targetVector; } + /** + * Appends the range [start, start + length) of the delta vector's data vector to the target + * vector's data vector. The range may not cover the whole delta data vector when the delta's + * offset buffer does not start at zero. + */ + private static void appendDataVector( + ValueVector targetDataVector, ValueVector deltaDataVector, int start, int length) { + if (start == 0 && length == deltaDataVector.getValueCount()) { + VectorAppender innerAppender = new VectorAppender(targetDataVector); + deltaDataVector.accept(innerAppender, null); + return; + } + TransferPair transferPair = + deltaDataVector.getTransferPair(deltaDataVector.getField(), deltaDataVector.getAllocator()); + transferPair.splitAndTransfer(start, length); + try (ValueVector slicedDeltaDataVector = transferPair.getTo()) { + VectorAppender innerAppender = new VectorAppender(targetDataVector); + slicedDeltaDataVector.accept(innerAppender, null); + } + } + @Override public ValueVector visit(FixedSizeListVector deltaVector, Void value) { Preconditions.checkArgument( diff --git a/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java b/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java index df5521a1ad..9a8143f51b 100644 --- a/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java +++ b/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java @@ -26,10 +26,13 @@ import java.util.List; import java.util.stream.IntStream; import java.util.stream.Stream; +import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.memory.util.CommonUtil; +import org.apache.arrow.vector.BaseLargeVariableWidthVector; import org.apache.arrow.vector.BaseValueVector; +import org.apache.arrow.vector.BaseVariableWidthVector; import org.apache.arrow.vector.BaseVariableWidthViewVector; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; @@ -53,6 +56,7 @@ import org.apache.arrow.vector.holders.NullableBigIntHolder; import org.apache.arrow.vector.holders.NullableFloat4Holder; import org.apache.arrow.vector.holders.NullableIntHolder; +import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.vector.testing.ValueVectorDataPopulator; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -178,6 +182,82 @@ public void testAppendVariableWidthVector() { } } + @Test + public void testAppendVariableWidthVectorWithNonZeroStartOffset() { + try (VarCharVector target = new VarCharVector("", allocator); + VarCharVector delta = new VarCharVector("", allocator)) { + + target.allocateNew(64, 4); + ValueVectorDataPopulator.setVector(target, "a0", "a1"); + + // Build a delta vector whose offset buffer does not start at zero, as produced e.g. by + // importing a sliced array through the C data interface. The values are "BBBB" and + // "CCCC"; the data buffer additionally holds 4 bytes of unreferenced prefix ("AAAA"). + try (ArrowBuf validity = allocator.buffer(1); + ArrowBuf offsets = allocator.buffer(12); + ArrowBuf data = allocator.buffer(12)) { + validity.setByte(0, 0b11); + offsets.setInt(0, 4); + offsets.setInt(4, 8); + offsets.setInt(8, 12); + data.setBytes(0, "AAAABBBBCCCC".getBytes(StandardCharsets.UTF_8)); + delta.loadFieldBuffers(new ArrowFieldNode(2, 0), Arrays.asList(validity, offsets, data)); + } + + VectorAppender appender = new VectorAppender(target); + delta.accept(appender, null); + + // the unreferenced prefix must not be appended + assertEquals( + 4 + 8, + target + .getOffsetBuffer() + .getInt((long) target.getValueCount() * BaseVariableWidthVector.OFFSET_WIDTH)); + + try (VarCharVector expected = new VarCharVector("expected", allocator)) { + expected.allocateNew(); + ValueVectorDataPopulator.setVector(expected, "a0", "a1", "BBBB", "CCCC"); + assertVectorsEqual(expected, target); + } + } + } + + @Test + public void testAppendLargeVariableWidthVectorWithNonZeroStartOffset() { + try (LargeVarCharVector target = new LargeVarCharVector("", allocator); + LargeVarCharVector delta = new LargeVarCharVector("", allocator)) { + + target.allocateNew(64, 4); + ValueVectorDataPopulator.setVector(target, "a0", "a1"); + + try (ArrowBuf validity = allocator.buffer(1); + ArrowBuf offsets = allocator.buffer(24); + ArrowBuf data = allocator.buffer(12)) { + validity.setByte(0, 0b11); + offsets.setLong(0, 4); + offsets.setLong(8, 8); + offsets.setLong(16, 12); + data.setBytes(0, "AAAABBBBCCCC".getBytes(StandardCharsets.UTF_8)); + delta.loadFieldBuffers(new ArrowFieldNode(2, 0), Arrays.asList(validity, offsets, data)); + } + + VectorAppender appender = new VectorAppender(target); + delta.accept(appender, null); + + assertEquals( + 4 + 8, + target + .getOffsetBuffer() + .getLong((long) target.getValueCount() * BaseLargeVariableWidthVector.OFFSET_WIDTH)); + + try (LargeVarCharVector expected = new LargeVarCharVector("expected", allocator)) { + expected.allocateNew(); + ValueVectorDataPopulator.setVector(expected, "a0", "a1", "BBBB", "CCCC"); + assertVectorsEqual(expected, target); + } + } + } + @Test public void testAppendVariableWidthViewVector() { final int length1 = 10; @@ -431,6 +511,115 @@ public void testAppendListVector() { } } + @Test + public void testAppendListVectorWithNonZeroStartOffset() { + try (ListVector target = ListVector.empty("target", allocator); + ListVector delta = ListVector.empty("delta", allocator)) { + + target.allocateNew(); + ValueVectorDataPopulator.setVector(target, Arrays.asList(0, 1), Arrays.asList(2, 3)); + + // Build a delta vector whose offset buffer does not start at zero, as produced e.g. by + // importing a sliced array through the C data interface: lists [10, 11] and [12, 13], + // with one unreferenced prefix element (9) in the data vector. + delta.addOrGetVector(FieldType.nullable(Types.MinorType.INT.getType())); + IntVector deltaDataVector = (IntVector) delta.getDataVector(); + deltaDataVector.allocateNew(5); + for (int i = 0; i < 5; i++) { + deltaDataVector.set(i, 9 + i); + } + deltaDataVector.setValueCount(5); + try (ArrowBuf validity = allocator.buffer(1); + ArrowBuf offsets = allocator.buffer(12)) { + validity.setByte(0, 0b11); + offsets.setInt(0, 1); + offsets.setInt(4, 3); + offsets.setInt(8, 5); + delta.loadFieldBuffers(new ArrowFieldNode(2, 0), Arrays.asList(validity, offsets)); + } + assertEquals(Arrays.asList(10, 11), delta.getObject(0)); + + VectorAppender appender = new VectorAppender(target); + delta.accept(appender, null); + + assertEquals(4, target.getValueCount()); + // the unreferenced prefix element must not be appended + assertEquals( + 4 + 4, + target.getOffsetBuffer().getInt((long) target.getValueCount() * ListVector.OFFSET_WIDTH)); + assertEquals(Arrays.asList(0, 1), target.getObject(0)); + assertEquals(Arrays.asList(2, 3), target.getObject(1)); + assertEquals(Arrays.asList(10, 11), target.getObject(2)); + assertEquals(Arrays.asList(12, 13), target.getObject(3)); + } + } + + @Test + public void testAppendLargeListVector() { + try (LargeListVector target = LargeListVector.empty("target", allocator); + LargeListVector delta = LargeListVector.empty("delta", allocator)) { + + target.allocateNew(); + ValueVectorDataPopulator.setVector(target, Arrays.asList(0, 1), null, Arrays.asList(4, 5)); + + delta.allocateNew(); + ValueVectorDataPopulator.setVector(delta, Arrays.asList(10, 11, 12), Arrays.asList(13, 14)); + + VectorAppender appender = new VectorAppender(target); + delta.accept(appender, null); + + assertEquals(5, target.getValueCount()); + assertEquals(Arrays.asList(0, 1), target.getObject(0)); + assertTrue(target.isNull(1)); + assertEquals(Arrays.asList(4, 5), target.getObject(2)); + assertEquals(Arrays.asList(10, 11, 12), target.getObject(3)); + assertEquals(Arrays.asList(13, 14), target.getObject(4)); + } + } + + @Test + public void testAppendLargeListVectorWithNonZeroStartOffset() { + try (LargeListVector target = LargeListVector.empty("target", allocator); + LargeListVector delta = LargeListVector.empty("delta", allocator)) { + + target.allocateNew(); + ValueVectorDataPopulator.setVector(target, Arrays.asList(0, 1), Arrays.asList(2, 3)); + + // same as testAppendListVectorWithNonZeroStartOffset, with 8-byte offsets + delta.addOrGetVector(FieldType.nullable(Types.MinorType.INT.getType())); + IntVector deltaDataVector = (IntVector) delta.getDataVector(); + deltaDataVector.allocateNew(5); + for (int i = 0; i < 5; i++) { + deltaDataVector.set(i, 9 + i); + } + deltaDataVector.setValueCount(5); + try (ArrowBuf validity = allocator.buffer(1); + ArrowBuf offsets = allocator.buffer(24)) { + validity.setByte(0, 0b11); + offsets.setLong(0, 1); + offsets.setLong(8, 3); + offsets.setLong(16, 5); + delta.loadFieldBuffers(new ArrowFieldNode(2, 0), Arrays.asList(validity, offsets)); + } + assertEquals(Arrays.asList(10, 11), delta.getObject(0)); + + VectorAppender appender = new VectorAppender(target); + delta.accept(appender, null); + + assertEquals(4, target.getValueCount()); + // the unreferenced prefix element must not be appended + assertEquals( + 4 + 4, + target + .getOffsetBuffer() + .getLong((long) target.getValueCount() * LargeListVector.OFFSET_WIDTH)); + assertEquals(Arrays.asList(0, 1), target.getObject(0)); + assertEquals(Arrays.asList(2, 3), target.getObject(1)); + assertEquals(Arrays.asList(10, 11), target.getObject(2)); + assertEquals(Arrays.asList(12, 13), target.getObject(3)); + } + } + @Test public void testAppendEmptyListVector() { try (ListVector target = ListVector.empty("target", allocator); From 9e863319b1e3bedda45dbb12d5989f6b1330f664 Mon Sep 17 00:00:00 2001 From: Abdul Rawoof Khan Date: Mon, 6 Jul 2026 06:39:33 +0530 Subject: [PATCH 394/408] GH-1206: validate decompressed length in Lz4CompressionCodec (#1207) ## What's Changed `Lz4CompressionCodec.doDecompress` sizes the output buffer to the bytes it actually decompressed, but sets `writerIndex` to the length taken from the untrusted 8-byte prefix. A buffer whose prefix claims more than the real output leaves the returned `ArrowBuf` with a `writerIndex` past its capacity, and consumers then read off-heap memory beyond the allocation. This adds the actual-vs-claimed length check the ZSTD codec already does, so a mismatch throws instead of producing an over-long buffer. Closes #1206. --- .../compression/Lz4CompressionCodec.java | 7 ++++++ .../compression/TestCompressionCodec.java | 22 +++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/compression/src/main/java/org/apache/arrow/compression/Lz4CompressionCodec.java b/compression/src/main/java/org/apache/arrow/compression/Lz4CompressionCodec.java index 91cefc2a9e..f268e815fe 100644 --- a/compression/src/main/java/org/apache/arrow/compression/Lz4CompressionCodec.java +++ b/compression/src/main/java/org/apache/arrow/compression/Lz4CompressionCodec.java @@ -80,6 +80,13 @@ protected ArrowBuf doDecompress(BufferAllocator allocator, ArrowBuf compressedBu } byte[] outBytes = out.toByteArray(); + if (outBytes.length != decompressedLength) { + throw new RuntimeException( + "Expected != actual decompressed length: " + + decompressedLength + + " != " + + outBytes.length); + } ArrowBuf decompressedBuffer = allocator.buffer(outBytes.length); decompressedBuffer.setBytes(/* index= */ 0, outBytes); decompressedBuffer.writerIndex(decompressedLength); diff --git a/compression/src/test/java/org/apache/arrow/compression/TestCompressionCodec.java b/compression/src/test/java/org/apache/arrow/compression/TestCompressionCodec.java index b8fb4e28b9..d2d2921649 100644 --- a/compression/src/test/java/org/apache/arrow/compression/TestCompressionCodec.java +++ b/compression/src/test/java/org/apache/arrow/compression/TestCompressionCodec.java @@ -20,6 +20,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayOutputStream; @@ -59,6 +60,7 @@ import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -231,6 +233,26 @@ void testEmptyBuffer(int vectorLength, CompressionCodec codec) throws Exception AutoCloseables.close(decompressedBuffers); } + @Test + void testLz4DecompressRejectsWrongLength() { + byte[] data = new byte[512]; // all zeros, highly compressible + ArrowBuf orig = allocator.buffer(data.length); + orig.setBytes(0, data); + orig.writerIndex(data.length); + + CompressionCodec codec = new Lz4CompressionCodec(); + ArrowBuf compressed = codec.compress(allocator, orig); + + // tamper with the 8-byte uncompressed-length prefix so it no longer matches + // the real decompressed size + compressed.setLong(0, 1_000_000L); + + RuntimeException e = + assertThrows(RuntimeException.class, () -> codec.decompress(allocator, compressed)); + assertTrue(e.getMessage().contains("decompressed length")); + compressed.close(); + } + private static Stream codecTypes() { return Arrays.stream(CompressionUtil.CodecType.values()); } From c7e8e75c9978c60234dbcbd31311ac3ee2975fa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Pupier?= Date: Mon, 6 Jul 2026 08:50:26 +0200 Subject: [PATCH 395/408] MINOR: Restrict trigger push branch for GitHub Workflow (#1204) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Feature branches rarely need their own CI runs: the code is already tested when a pull request is opened against a release branch. If the push trigger has no branch restriction and pull_request is also configured, every push to a branch with an open PR runs the workflow twice: once for the push and once for the PR synchronisation. Always give the push trigger an explicit list of branches: this stops branches created from a release branch from inheriting its workflow runs. see https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=430408443#GitHubActionsRecommendedPractices-Restrictthepushtriggertospecificbranches ## What's Changed Please fill in a description of the changes here. **This contains breaking changes.** Closes #NNN. Note that I needed to recreate a PR as the previous one was closed https://github.com/apache/arrow-java/pull/1202#issuecomment-4864548916 Signed-off-by: Aurélien Pupier --- .github/workflows/dev.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 25b08700fd..7c590c749a 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -19,7 +19,9 @@ name: Dev on: pull_request: {} - push: {} + push: + branches-ignore: + - dependabot/** concurrency: group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} From 9e100a3b7f681a64f11eb4405e9e43431caf3647 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:21:40 +0200 Subject: [PATCH 396/408] MINOR: Bump com.squareup.okhttp3:mockwebserver3 from 5.3.2 to 5.4.0 (#1215) Bumps [com.squareup.okhttp3:mockwebserver3](https://github.com/square/okhttp) from 5.3.2 to 5.4.0.
Changelog

Sourced from com.squareup.okhttp3:mockwebserver3's changelog.

Version 5.4.0

2026-06-08

  • New: Add superpowers to interceptors. Interceptors can now override anything settable on OkHttpClient.Builder, such as the cache, connection pool, socket factory, and DNS. We expect this will allow most users to use interceptors everywhere, insted of mixing and matching interceptors with custom Call.Factory wrappers.
  • Fix: Limit each HTTP/2 response to 256 KiB of total headers.
  • Upgrade: [kotlinx.coroutines 1.11.0][coroutines_1_11_0]. This is used by the optional okhttp-coroutines artifact.
  • Upgrade: [GraalVM 25.0.3][graalvm_25].
  • Upgrade: [Okio 3.17.0][okio_3_17_0].
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.squareup.okhttp3:mockwebserver3&package-manager=maven&previous-version=5.3.2&new-version=5.4.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 9a8406cf31..e25d8438af 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -123,7 +123,7 @@ under the License. com.squareup.okhttp3 mockwebserver3 - 5.3.2 + 5.4.0 test From e087824935d283d3636d8edb908a79df2de217c4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:25:41 +0200 Subject: [PATCH 397/408] MINOR: Bump com.diffplug.spotless:spotless-maven-plugin from 3.6.0 to 3.8.0 (#1214) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [com.diffplug.spotless:spotless-maven-plugin](https://github.com/diffplug/spotless) from 3.6.0 to 3.8.0.
Release notes

Sourced from com.diffplug.spotless:spotless-maven-plugin's releases.

Maven Plugin v3.8.0

Added

  • Add support for custom string format for license header copyright year via yearStringFormat(). (#2965)

Fixed

  • <expandWildcardImports> no longer triggers a full transitive dependency resolution on every build. Dependency resolution is now deferred until the step actually runs, so projects that do not use <expandWildcardImports> (or that use version ranges) are no longer penalized. (#2983)

Maven Plugin v3.7.0

Fixed

  • Parse standard git year output in LicenseHeaderStep. (#2940)
  • <toggleOffOn> no longer disables lint-only steps such as <forbidWildcardImports>. (#2962)
  • Fix StringIndexOutOfBoundsException in scenarios where copyright year is surrounded by whitespace. (#2973)

Added

  • Add support for AsciiDoc formatting via adocfmt. (#2960)
  • <flexmark> step now supports arbitrary formatter options via <formatterOptions>. (#2968)
Changelog

Sourced from com.diffplug.spotless:spotless-maven-plugin's changelog.

spotless-lib and spotless-lib-extra releases

If you are a Spotless user (as opposed to developer), then you are probably looking for:

This document is intended for Spotless developers.

We adhere to the keepachangelog format (starting after version 1.27.0).

[Unreleased]

[4.8.0] - 2026-06-29

Added

  • Add support for custom string format for license header copyright year via yearStringFormat(). (#2965)

[4.7.0] - 2026-06-16

Added

  • Add support for AsciiDoc formatting via adocfmt. (#2960)
  • flexmark step now supports arbitrary formatter options via a formatterOptions map. (#2968)

Fixed

  • FenceStep.preserveWithin now forwards lints from nested steps while still suppressing lints inside preserved blocks. (#2962)
  • Support ktfmt 0.63 and use its new builder API for formatting options to better avoid future breaking changes.
  • Parse standard git year output in LicenseHeaderStep. (#2940)
  • Fix StringIndexOutOfBoundsException in scenarios where copyright year is surrounded by whitespace. (#2973)

Changes

  • Bump default greclipse version to latest 4.35 -> 4.39. (#2924)

[4.6.2] - 2026-05-27

Fixed

  • P2Provisioner now passes cache directory overrides directly to Solstice. (#2944)
  • forbidWildcardImports and forbidModuleImports now detect imports that have leading whitespace (indentation/tabs). (#2939)
  • versionCatalog step no longer splits long inline tables across multiple lines — Gradle's TOML 1.0 parser cannot read multi-line inline tables. The maxLineLength option has been removed. (#2948)

Changes

  • EclipseJdtFormtterStep now can conditionally set compiler source/compliance options. Allows for better parsing of AST Node for newer language features and more correct sorting; e.g. records or seal classes. (#2942)
  • Formatter no longer recomputes line-ending normalization (LineEnding.toUnix) a second time for every formatter step that changes content, removing redundant O(n) work from the core formatting loop. (#2934)
  • expandWildcardImports support pom type dependency. (#2839)

[4.6.1] - 2026-05-15

Fixed

  • LicenseHeaderStep in SET_FROM_GIT year mode no longer invokes git log through bash -c / cmd /c, eliminating a shell-injection vector when processing repositories that contain files whose names include shell metacharacters.

[4.6.0] - 2026-05-14

Added

  • scalafmt() now reads the version from the version field in the scalafmt config file when no version is explicitly set in the plugin config, falling back to the built-in default only if neither is available. (#2922)
  • Add versionCatalog step for formatting and sorting Gradle version catalog (.toml) files. (#2916)
  • Add javaparserVersion option to the Cleanthat step, allowing callers to override the JavaParser version pulled in transitively by Cleanthat. (#2903)

Fixed

... (truncated)

Commits
  • 03d43ba Published maven/3.8.0
  • 8b80c13 Published gradle/8.8.0
  • 8ee6cf9 Published lib/4.8.0
  • 6c02c0b Add missing changelog entry.
  • 264f4cc Add regression test for forbidWildcardImports inside toggleOffOn (#2982)
  • 6abb064 fix #2983, expandWildcardImports triggers a full transitive reso… (#2984)
  • f4536d4 Update plugin spotbugs to v6.5.8 (#2987)
  • 873454a Update plugin spotbugs to v6.5.8
  • 000b8a8 Update dependency org.junit.jupiter:junit-jupiter to v6.1.1 (#2985)
  • 84ebcab Update dependency org.junit.jupiter:junit-jupiter to v6.1.1
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.diffplug.spotless:spotless-maven-plugin&package-manager=maven&previous-version=3.6.0&new-version=3.8.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- bom/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index 3201761e81..2d1085b160 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -208,7 +208,7 @@ under the License. com.diffplug.spotless spotless-maven-plugin - 3.6.0 + 3.8.0 org.codehaus.mojo diff --git a/pom.xml b/pom.xml index 231720eb44..017e3acb81 100644 --- a/pom.xml +++ b/pom.xml @@ -492,7 +492,7 @@ under the License. com.diffplug.spotless spotless-maven-plugin - 3.6.0 + 3.8.0 org.codehaus.mojo From 1801a8a0937fb4aaac3654c66ec9109a6892666c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:28:19 +0200 Subject: [PATCH 398/408] MINOR: Bump com.gradle:common-custom-user-data-maven-extension from 2.2.0 to 2.3.0 (#1213) Bumps [com.gradle:common-custom-user-data-maven-extension](https://github.com/gradle/common-custom-user-data-maven-extension) from 2.2.0 to 2.3.0.
Release notes

Sourced from com.gradle:common-custom-user-data-maven-extension's releases.

2.3.0

  • [NEW] Capture Cursor as an AI agent via the CURSOR_AGENT environment variable
Commits
  • 61a5a45 [maven-release-plugin] prepare release v2.3.0
  • 398a231 [Renovate Bot] Update actions/setup-java digest to 1bcf9fb (#389)
  • f66a5c6 Merge pull request #391 from gradle/erichaagdev/capture-cursor-ai-agent
  • 8311f58 Capture Cursor as an AI agent
  • 0236fc8 [Renovate Bot] Update dependency org.eclipse.sisu:org.eclipse.sisu.inject to ...
  • 0d8c2c5 [Renovate Bot] Update GitHub Actions to v7 (#386)
  • 21ef159 [Renovate Bot] Update Maven dependencies (#387)
  • d9240f0 [Renovate Bot] Update Maven dependencies to v0.11.0 (#384)
  • 83935cc Auto-merge GitHub Actions digest re-pins (#385)
  • 9d03ae6 [Renovate Bot] Update GitHub Actions to ad2b381 (#383)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.gradle:common-custom-user-data-maven-extension&package-manager=maven&previous-version=2.2.0&new-version=2.3.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .mvn/extensions.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index 38b3c807b7..eb06ecc9fb 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -28,6 +28,6 @@ com.gradle common-custom-user-data-maven-extension - 2.2.0 + 2.3.0 From afd688e024753c75854e707748fe5ad4411d3361 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:29:40 +0200 Subject: [PATCH 399/408] MINOR: Bump com.gradle:develocity-maven-extension from 2.4.1 to 2.5.0 (#1212) Bumps com.gradle:develocity-maven-extension from 2.4.1 to 2.5.0. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.gradle:develocity-maven-extension&package-manager=maven&previous-version=2.4.1&new-version=2.5.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .mvn/extensions.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index eb06ecc9fb..74482cb2c4 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -23,7 +23,7 @@ com.gradle develocity-maven-extension - 2.4.1 + 2.5.0 com.gradle From 59010e63bb58c702ccdbcf6f7e0fb047bf8c8f73 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:31:03 +0200 Subject: [PATCH 400/408] MINOR: Bump checker.framework.version from 4.2.0 to 4.2.1 (#1211) Bumps `checker.framework.version` from 4.2.0 to 4.2.1. Updates `org.checkerframework:checker-qual` from 4.2.0 to 4.2.1
Release notes

Sourced from org.checkerframework:checker-qual's releases.

Checker Framework 4.2.1

Version 4.2.1 (2026-07-01)

Closed issues

#7726.

Changelog

Sourced from org.checkerframework:checker-qual's changelog.

Version 4.2.1 (2026-07-01)

Closed issues

#7726.

Commits

Updates `org.checkerframework:checker` from 4.2.0 to 4.2.1
Release notes

Sourced from org.checkerframework:checker's releases.

Checker Framework 4.2.1

Version 4.2.1 (2026-07-01)

Closed issues

#7726.

Changelog

Sourced from org.checkerframework:checker's changelog.

Version 4.2.1 (2026-07-01)

Closed issues

#7726.

Commits

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 017e3acb81..d9f9d59c43 100644 --- a/pom.xml +++ b/pom.xml @@ -112,7 +112,7 @@ under the License. 10.23.0 true 2.42.0 - 4.2.0 + 4.2.1 1.5.34 none -Xdoclint:none From f2594c99cc98923ad5949713d4519c5d4f24f4f6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:32:02 +0200 Subject: [PATCH 401/408] MINOR: [CI] Bump docker/login-action from 4.2.0 to 4.4.0 (#1210) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [docker/login-action](https://github.com/docker/login-action) from 4.2.0 to 4.4.0.
Release notes

Sourced from docker/login-action's releases.

v4.4.0

Full Changelog: https://github.com/docker/login-action/compare/v4.3.0...v4.4.0

v4.3.0

Full Changelog: https://github.com/docker/login-action/compare/v4.2.0...v4.3.0

Commits
  • af1e73f Merge pull request #1034 from docker/dependabot/npm_and_yarn/aws-sdk-dependen...
  • da722bd [dependabot skip] chore: update generated content
  • 2916ad6 build(deps): bump the aws-sdk-dependencies group across 1 directory with 2 up...
  • ca0a662 Merge pull request #1035 from crazy-max/fix-registry-auth-empty-mask
  • c455755 chore: update generated content
  • 4835190 skip empty registry-auth secret mask
  • 992421c Merge pull request #1033 from docker/dependabot/github_actions/docker/bake-ac...
  • b249b43 Merge pull request #1032 from docker/dependabot/github_actions/docker/bake-ac...
  • 1b67977 build(deps): bump docker/bake-action from 7.2.0 to 7.3.0
  • 9d49d6a build(deps): bump docker/bake-action/subaction/matrix
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=docker/login-action&package-manager=github_actions&previous-version=4.2.0&new-version=4.4.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index fdfcd1cce3..5d2fb6683a 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -127,7 +127,7 @@ jobs: with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing - - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 with: registry: ghcr.io username: ${{ github.actor }} From 915428751b8871406268aeb27bd2f055dcfba6bf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 16:00:46 +0200 Subject: [PATCH 402/408] MINOR: Bump logback.version from 1.5.34 to 1.5.37 (#1209) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps `logback.version` from 1.5.34 to 1.5.37. Updates `ch.qos.logback:logback-classic` from 1.5.34 to 1.5.37
Release notes

Sourced from ch.qos.logback:logback-classic's releases.

Logback 1.5.37

2026-06-26 Release of logback version 1.5.37

  1. • Given the numerous vulnerabilities related to conditional configuration processing based on the evaluation of Java expressions using the Janino library, support for such expressions has been removed. Users are offered the an online migration service or the <condition> element introduced in version 1.5.20. See the relevant documentation for more details.

• A bitwise identical binary of this version can be reproduced by building from source code at commit c1df7f522e648eec7b4ef6a12c8758fec0f00048 associated with the tag v_1.5.37. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Logback 1.5.36

2026-06-25 Release of logback version 1.5.36

• The 'condition' attribute in <if> elements now reject certain references that are associated with ACE attacks. This issue was reported by "yulate" (yulate531@gmail.com.com) and registered as CVE-2026-13006. Please note that version 1.5.37 provides the full fix to this vulnerability.

• A bitwise identical binary of this version can be reproduced by building from source code at commit 9b94c37562bf25a6a944146701d42ee6c4eee888 associated with the tag v_1.5.36. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Logback 1.5.35

026-06-23 Release of logback version 1.5.35

• The 'condition' attribute in <if> elements now rejects unicode escape sequences (\u and \U). This closes a bypass of the existing prohibition on the new operator in Janino-evaluated conditions. This issue was reported by IcySun (icysun@qq.com) and registered as CVE-2026-13006. Please note that version 1.5.37 provides the full fix to this vulnerability.

• Added ConfiguratorRank.AUTHENTICATING (rank 100), the highest configurator rank, for certified/authenticating configurators discovered via the ServiceLoader mechanism. ContextInitializer now requires that at most one such configurator exist on the classpath; if more than one is found, initialization aborts with an error.

• ConsoleCharsetPropertyDefiner is no longer shipped. The Java 21 multi-release compilation of logback-core has been disabled, which removes this class from the published artifact. Configurations that referenced ch.qos.logback.core.property.ConsoleCharsetPropertyDefiner will need an alternative approach for console charset detection.

• The logback-examples module is now included in artifacts published to Maven Central.

• JoranConfigurator.makeAnotherInstance() and DefaultJoranConfigurator.performMultiStepConfigurationFileSearch() are now protected, allowing derived configurators to override these methods.

• A bitwise identical binary of this version can be reproduced by building from source code at commit 08bd1598d565d83444f72983935e7da4746783b7 associated with the tag v_1.5.35. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits

Updates `ch.qos.logback:logback-core` from 1.5.34 to 1.5.37
Release notes

Sourced from ch.qos.logback:logback-core's releases.

Logback 1.5.37

2026-06-26 Release of logback version 1.5.37

  1. • Given the numerous vulnerabilities related to conditional configuration processing based on the evaluation of Java expressions using the Janino library, support for such expressions has been removed. Users are offered the an online migration service or the <condition> element introduced in version 1.5.20. See the relevant documentation for more details.

• A bitwise identical binary of this version can be reproduced by building from source code at commit c1df7f522e648eec7b4ef6a12c8758fec0f00048 associated with the tag v_1.5.37. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Logback 1.5.36

2026-06-25 Release of logback version 1.5.36

• The 'condition' attribute in <if> elements now reject certain references that are associated with ACE attacks. This issue was reported by "yulate" (yulate531@gmail.com.com) and registered as CVE-2026-13006. Please note that version 1.5.37 provides the full fix to this vulnerability.

• A bitwise identical binary of this version can be reproduced by building from source code at commit 9b94c37562bf25a6a944146701d42ee6c4eee888 associated with the tag v_1.5.36. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Logback 1.5.35

026-06-23 Release of logback version 1.5.35

• The 'condition' attribute in <if> elements now rejects unicode escape sequences (\u and \U). This closes a bypass of the existing prohibition on the new operator in Janino-evaluated conditions. This issue was reported by IcySun (icysun@qq.com) and registered as CVE-2026-13006. Please note that version 1.5.37 provides the full fix to this vulnerability.

• Added ConfiguratorRank.AUTHENTICATING (rank 100), the highest configurator rank, for certified/authenticating configurators discovered via the ServiceLoader mechanism. ContextInitializer now requires that at most one such configurator exist on the classpath; if more than one is found, initialization aborts with an error.

• ConsoleCharsetPropertyDefiner is no longer shipped. The Java 21 multi-release compilation of logback-core has been disabled, which removes this class from the published artifact. Configurations that referenced ch.qos.logback.core.property.ConsoleCharsetPropertyDefiner will need an alternative approach for console charset detection.

• The logback-examples module is now included in artifacts published to Maven Central.

• JoranConfigurator.makeAnotherInstance() and DefaultJoranConfigurator.performMultiStepConfigurationFileSearch() are now protected, allowing derived configurators to override these methods.

• A bitwise identical binary of this version can be reproduced by building from source code at commit 08bd1598d565d83444f72983935e7da4746783b7 associated with the tag v_1.5.35. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: JB Onofré --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d9f9d59c43..78137eb4e9 100644 --- a/pom.xml +++ b/pom.xml @@ -113,7 +113,7 @@ under the License. true 2.42.0 4.2.1 - 1.5.34 + 1.5.37 none -Xdoclint:none From a9f0086090e649b83c89ea864c82ed7fcc84ab2f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 22:19:02 +0200 Subject: [PATCH 403/408] MINOR: Bump com.squareup.okhttp3:okhttp-jvm from 5.3.2 to 5.4.0 (#1208) Bumps [com.squareup.okhttp3:okhttp-jvm](https://github.com/square/okhttp) from 5.3.2 to 5.4.0.
Changelog

Sourced from com.squareup.okhttp3:okhttp-jvm's changelog.

Version 5.4.0

2026-06-08

  • New: Add superpowers to interceptors. Interceptors can now override anything settable on OkHttpClient.Builder, such as the cache, connection pool, socket factory, and DNS. We expect this will allow most users to use interceptors everywhere, insted of mixing and matching interceptors with custom Call.Factory wrappers.
  • Fix: Limit each HTTP/2 response to 256 KiB of total headers.
  • Upgrade: [kotlinx.coroutines 1.11.0][coroutines_1_11_0]. This is used by the optional okhttp-coroutines artifact.
  • Upgrade: [GraalVM 25.0.3][graalvm_25].
  • Upgrade: [Okio 3.17.0][okio_3_17_0].
Commits

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index e25d8438af..be2ee32868 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -135,7 +135,7 @@ under the License. com.squareup.okhttp3 okhttp-jvm - 5.3.2 + 5.4.0 test From 7f1f9f588af610b842ee1c3bccf6afbd528ae4db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Thu, 9 Jul 2026 14:42:25 +0100 Subject: [PATCH 404/408] GH-1188: Reduce test workflow waste (#1189) ## Summary This PR reduces wasted GitHub Actions time in the `Test` workflow by removing redundant rebuilds, improving caching, and skipping integration work when it is not relevant. ## What changed - Removed `clean` from `ci/scripts/test.sh` so the test phase reuses the classes compiled earlier in the job instead of deleting and recompiling them. - Narrowed the Maven/Docker cache key to POM changes, so source-only edits do not invalidate the dependency cache. - Enabled Maven dependency caching on the macOS and Windows test jobs. - Added `pull-requests: read` so the integration job can inspect changed files. Closes #1188 Note: The changes on this PR were highlighted and addressed with AI assistance --- .github/workflows/test.yml | 24 +++++++++++++----------- ci/scripts/test.sh | 7 ++++--- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ac8080d075..41a42c05ed 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -66,7 +66,7 @@ jobs: uses: actions/cache@v6 with: path: .docker - key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('compose.yaml', '**/pom.xml', '**/*.java') }} + key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} restore-keys: maven-${{ matrix.jdk }}-${{ matrix.maven }}- - name: Execute Docker Build env: @@ -94,16 +94,17 @@ jobs: jdk: 17 macos: latest steps: - - name: Set up Java - uses: actions/setup-java@v5 - with: - distribution: 'temurin' - java-version: ${{ matrix.jdk }} - name: Checkout Arrow uses: actions/checkout@v7 with: fetch-depth: 0 submodules: recursive + - name: Set up Java + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: ${{ matrix.jdk }} + cache: 'maven' - name: Build shell: bash env: @@ -125,16 +126,17 @@ jobs: matrix: jdk: [17] steps: - - name: Set up Java - uses: actions/setup-java@v5 - with: - java-version: ${{ matrix.jdk }} - distribution: 'temurin' - name: Checkout Arrow uses: actions/checkout@v7 with: fetch-depth: 0 submodules: recursive + - name: Set up Java + uses: actions/setup-java@v5 + with: + java-version: ${{ matrix.jdk }} + distribution: 'temurin' + cache: 'maven' - name: Build shell: bash env: diff --git a/ci/scripts/test.sh b/ci/scripts/test.sh index cacc20034e..8061ee455d 100755 --- a/ci/scripts/test.sh +++ b/ci/scripts/test.sh @@ -34,10 +34,11 @@ fi mvn="mvn -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" # Use `2 * ncores` threads mvn="${mvn} -T 2C" +mvn="${mvn} -Denforcer.skip=true" pushd "${build_dir}" -${mvn} -Darrow.test.dataRoot="${source_dir}/testing/data" clean test +${mvn} -Darrow.test.dataRoot="${source_dir}/testing/data" test projects=() if [ "${ARROW_JAVA_JNI}" = "ON" ]; then @@ -46,7 +47,7 @@ if [ "${ARROW_JAVA_JNI}" = "ON" ]; then projects+=(gandiva) fi if [ "${#projects[@]}" -gt 0 ]; then - ${mvn} clean test \ + ${mvn} test \ -Parrow-jni \ -pl "$( IFS=, @@ -56,7 +57,7 @@ if [ "${#projects[@]}" -gt 0 ]; then fi if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then - ${mvn} clean test -Parrow-c-data -pl c -Darrow.c.jni.dist.dir="${java_jni_dist_dir}" + ${mvn} test -Parrow-c-data -pl c -Darrow.c.jni.dist.dir="${java_jni_dist_dir}" fi popd From 21b6a05154b7713b12ace4c0ba3572ba5cc8fe66 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2026 10:19:32 +0900 Subject: [PATCH 405/408] MINOR: [CI] Bump actions/setup-python from 6 to 7 (#1235) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/setup-python](https://github.com/actions/setup-python) from 6 to 7.
Release notes

Sourced from actions/setup-python's releases.

v7.0.0

What's Changed

Enhancements

Bug Fix

Dependency Upgrade

New Contributors

Full Changelog: https://github.com/actions/setup-python/compare/v6...v7.0.0

v6.3.0

What's Changed

Enhancement

Dependency update

Documentation

New Contributors

Full Changelog: https://github.com/actions/setup-python/compare/v6.2.0...v6.3.0

v6.2.0

What's Changed

Dependency Upgrades

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/setup-python&package-manager=github_actions&previous-version=6&new-version=7)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/dev.yml | 2 +- .github/workflows/rc.yml | 4 ++-- .github/workflows/test.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 7c590c749a..f4dc9ad6f1 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -40,7 +40,7 @@ jobs: fetch-depth: 0 persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: python-version: '3.x' - name: pre-commit (cache) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 5d2fb6683a..18a721ac02 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -195,7 +195,7 @@ jobs: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: cache: 'pip' python-version: 3.12 @@ -446,7 +446,7 @@ jobs: contents: read packages: write steps: - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: cache: 'pip' - name: Download source archive diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 41a42c05ed..853b7cec09 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -198,7 +198,7 @@ jobs: key: integration-conda-${{ hashFiles('cpp/**') }} restore-keys: integration-conda- - name: Setup Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: 3.12 - name: Setup Archery From 06170242bde2f492e068235efdd2183a3cbd87d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Sat, 25 Jul 2026 14:35:09 +0100 Subject: [PATCH 406/408] GH-1244: Move integration tests to separate workflow (#1245) ## What's Changed Move the `integration` job from `test.yml` into its own `integration.yml` workflow. The new workflow uses top-level `paths` filters, so integration tests run only when changes affect relevant code or build inputs. This should also help to reduce the amount of runner time. Topic initially triggered in the [mailing list](https://lists.apache.org/thread/drf0o5kzg1zfmok7gc09k8qz8hh9ymvh) Closes #1244 --- .github/workflows/integration.yml | 127 ++++++++++++++++++++++++++++++ .github/workflows/test.yml | 69 ---------------- 2 files changed, 127 insertions(+), 69 deletions(-) create mode 100644 .github/workflows/integration.yml diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000000..3872d03d2f --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,127 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Integration + +on: + push: + branches: + - '**' + - '!dependabot/**' + tags: + - '**' + paths: + - '.github/workflows/integration.yml' + - '**/pom.xml' + - 'c/**' + - 'ci/scripts/**' + - 'compose.yaml' + - 'flight/**' + - 'format/**' + - 'testing/data/**' + - 'vector/**' + pull_request: + paths: + - '.github/workflows/integration.yml' + - '**/pom.xml' + - 'c/**' + - 'ci/scripts/**' + - 'compose.yaml' + - 'flight/**' + - 'format/**' + - 'testing/data/**' + - 'vector/**' + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +env: + DOCKER_VOLUME_PREFIX: ".docker/" + +jobs: + integration: + name: AMD64 integration + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout Arrow + uses: actions/checkout@v7 + with: + fetch-depth: 0 + repository: apache/arrow + submodules: recursive + - name: Checkout Arrow Rust + uses: actions/checkout@v7 + with: + repository: apache/arrow-rs + path: rust + - name: Checkout Arrow nanoarrow + uses: actions/checkout@v7 + with: + repository: apache/arrow-nanoarrow + path: nanoarrow + - name: Checkout Arrow .NET + uses: actions/checkout@v7 + with: + repository: apache/arrow-dotnet + path: dotnet + - name: Checkout Arrow Go + uses: actions/checkout@v7 + with: + repository: apache/arrow-go + path: go + - name: Checkout Arrow Java + uses: actions/checkout@v7 + with: + path: java + - name: Checkout Arrow JavaScript + uses: actions/checkout@v7 + with: + repository: apache/arrow-js + path: js + - name: Free up disk space + run: | + ci/scripts/util_free_space.sh + - name: Cache Docker Volumes + uses: actions/cache@v6 + with: + path: .docker + key: integration-conda-${{ hashFiles('cpp/**') }} + restore-keys: integration-conda- + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: 3.12 + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + run: | + source ci/scripts/util_enable_core_dumps.sh + archery docker run \ + -e ARCHERY_DEFAULT_BRANCH=main \ + -e ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS=java \ + -e ARCHERY_INTEGRATION_WITH_DOTNET=1 \ + -e ARCHERY_INTEGRATION_WITH_GO=1 \ + -e ARCHERY_INTEGRATION_WITH_JAVA=1 \ + -e ARCHERY_INTEGRATION_WITH_JS=1 \ + -e ARCHERY_INTEGRATION_WITH_NANOARROW=1 \ + -e ARCHERY_INTEGRATION_WITH_RUST=1 \ + conda-integration diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 853b7cec09..653b16fa32 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -147,72 +147,3 @@ jobs: env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} run: ci/scripts/test.sh . build jni - - integration: - name: AMD64 integration - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - fetch-depth: 0 - repository: apache/arrow - submodules: recursive - - name: Checkout Arrow Rust - uses: actions/checkout@v7 - with: - repository: apache/arrow-rs - path: rust - - name: Checkout Arrow nanoarrow - uses: actions/checkout@v7 - with: - repository: apache/arrow-nanoarrow - path: nanoarrow - - name: Checkout Arrow .NET - uses: actions/checkout@v7 - with: - repository: apache/arrow-dotnet - path: dotnet - - name: Checkout Arrow Go - uses: actions/checkout@v7 - with: - repository: apache/arrow-go - path: go - - name: Checkout Arrow Java - uses: actions/checkout@v7 - with: - path: java - - name: Checkout Arrow JavaScript - uses: actions/checkout@v7 - with: - repository: apache/arrow-js - path: js - - name: Free up disk space - run: | - ci/scripts/util_free_space.sh - - name: Cache Docker Volumes - uses: actions/cache@v6 - with: - path: .docker - key: integration-conda-${{ hashFiles('cpp/**') }} - restore-keys: integration-conda- - - name: Setup Python - uses: actions/setup-python@v7 - with: - python-version: 3.12 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: | - source ci/scripts/util_enable_core_dumps.sh - archery docker run \ - -e ARCHERY_DEFAULT_BRANCH=main \ - -e ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS=java \ - -e ARCHERY_INTEGRATION_WITH_DOTNET=1 \ - -e ARCHERY_INTEGRATION_WITH_GO=1 \ - -e ARCHERY_INTEGRATION_WITH_JAVA=1 \ - -e ARCHERY_INTEGRATION_WITH_JS=1 \ - -e ARCHERY_INTEGRATION_WITH_NANOARROW=1 \ - -e ARCHERY_INTEGRATION_WITH_RUST=1 \ - conda-integration From 85d7ef30ee5f623cf3442dd989df09c3c5754ad2 Mon Sep 17 00:00:00 2001 From: Sandesh Kumar Date: Thu, 6 Aug 2026 17:29:51 -0700 Subject: [PATCH 407/408] GH-1239: Fix memory leak when C Data import hits allocator limit mid-array (#1240) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ReferenceCountedArrowArray.unsafeAssociateAllocation` calls `retain()` before `wrapForeignAllocation`. If `wrapForeignAllocation` throws (allocator over its limit), the retain is never balanced, the reference count stays elevated, and the C Data release callback never fires — leaking the producer's native memory. **Fix:** call `retain()` after `wrapForeignAllocation` returns. Same behavior on the success path; on failure the existing `finally` in `ArrayImporter.importArray` drives the count to zero and fires the release callback. **Test:** `ImportOutOfMemoryTest` — exports a batch from a "producer" allocator, tries to import it into a too-small consumer, and asserts the producer drains to zero after the OOM. Fails on the original code, passes with the fix. Fixes: https://github.com/apache/arrow-java/issues/1239 --------- Signed-off-by: Sandesh Kumar Co-authored-by: Sandesh Kumar --- .../arrow/c/ReferenceCountedArrowArray.java | 19 ++- .../apache/arrow/c/ImportOutOfMemoryTest.java | 139 ++++++++++++++++++ 2 files changed, 151 insertions(+), 7 deletions(-) create mode 100644 c/src/test/java/org/apache/arrow/c/ImportOutOfMemoryTest.java diff --git a/c/src/main/java/org/apache/arrow/c/ReferenceCountedArrowArray.java b/c/src/main/java/org/apache/arrow/c/ReferenceCountedArrowArray.java index cf50f9417b..f51fb25105 100644 --- a/c/src/main/java/org/apache/arrow/c/ReferenceCountedArrowArray.java +++ b/c/src/main/java/org/apache/arrow/c/ReferenceCountedArrowArray.java @@ -64,13 +64,18 @@ void release() { */ ArrowBuf unsafeAssociateAllocation( BufferAllocator trackingAllocator, long capacity, long memoryAddress) { + // Retain only after wrapForeignAllocation succeeds. On the allocator-limit OOM path, + // wrapForeignAllocation throws before the ForeignAllocation is associated, so release0() + // is not called; retaining first would leave the count elevated with no matching release0(). + ArrowBuf buf = + trackingAllocator.wrapForeignAllocation( + new ForeignAllocation(capacity, memoryAddress) { + @Override + protected void release0() { + ReferenceCountedArrowArray.this.release(); + } + }); retain(); - return trackingAllocator.wrapForeignAllocation( - new ForeignAllocation(capacity, memoryAddress) { - @Override - protected void release0() { - ReferenceCountedArrowArray.this.release(); - } - }); + return buf; } } diff --git a/c/src/test/java/org/apache/arrow/c/ImportOutOfMemoryTest.java b/c/src/test/java/org/apache/arrow/c/ImportOutOfMemoryTest.java new file mode 100644 index 0000000000..7c099f2ef0 --- /dev/null +++ b/c/src/test/java/org/apache/arrow/c/ImportOutOfMemoryTest.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.c; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.List; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.OutOfMemoryException; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Regression test: a mid-import {@link OutOfMemoryException} must not leak the imported array. + * + *

A "producer" allocator owns the exported batch; if the C Data release callback fires, the + * producer drains to zero. A too-small consumer allocator forces an OOM part-way through the + * import. The test asserts the producer drains, confirming the release callback fired despite the + * failure. + */ +final class ImportOutOfMemoryTest { + private static final int ROWS = 1024; + private static final int VALUE_BYTES = 256; + private static final int COLUMNS = 4; + // Far smaller than the exported batch, so the import OOMs part-way through the buffers. + private static final long TINY_LIMIT = 16 * 1024; + + private RootAllocator root; + + @BeforeEach + public void setUp() { + root = new RootAllocator(Long.MAX_VALUE); + } + + @AfterEach + public void tearDown() { + root.close(); + } + + @Test + public void importOomDoesNotLeakExportedArray() { + // "producer" owns only the exported batch buffers; the C Data struct containers live on a + // separate allocator (they are consumed/closed by import, which would otherwise muddy the + // producer's balance). So producer draining to zero is an exact signal that the array's release + // callback fired. + try (BufferAllocator producer = root.newChildAllocator("producer", 0, Long.MAX_VALUE); + BufferAllocator structs = root.newChildAllocator("structs", 0, Long.MAX_VALUE)) { + try (ArrowArray array = ArrowArray.allocateNew(structs); + ArrowSchema schema = ArrowSchema.allocateNew(structs)) { + exportBatch(producer, array, schema); + assertTrue( + producer.getAllocatedMemory() > 0, "producer holds the exported batch before import"); + + // A consumer allocator far too small to hold the batch: the import throws part-way through. + try (BufferAllocator consumer = root.newChildAllocator("consumer", 0, TINY_LIMIT); + CDataDictionaryProvider provider = new CDataDictionaryProvider()) { + Schema importSchema = Data.importSchema(consumer, schema, provider); + try (VectorSchemaRoot importRoot = VectorSchemaRoot.create(importSchema, consumer)) { + Exception thrown = + assertThrows( + Exception.class, + () -> Data.importIntoVectorSchemaRoot(consumer, array, importRoot, provider)); + assertTrue( + hasOutOfMemoryCause(thrown), + "mid-import failure must be an allocator OOM: " + thrown); + } + } + + // The array's release callback must have fired despite the mid-import OOM, freeing the + // whole exported batch. On the unfixed retain-before-wrap code the batch is stranded. + assertEquals( + 0L, + producer.getAllocatedMemory(), + "import OOM leaked the exported batch (producer not drained)"); + } + } + } + + /** True if {@code t} is, or is caused by, an Arrow {@link OutOfMemoryException}. */ + private static boolean hasOutOfMemoryCause(Throwable t) { + for (Throwable cause = t; cause != null; cause = cause.getCause()) { + if (cause instanceof OutOfMemoryException) { + return true; + } + } + return false; + } + + /** + * Builds a wide multi-column VarChar batch on {@code alloc} and exports it into the C structs. + */ + private void exportBatch(BufferAllocator alloc, ArrowArray array, ArrowSchema schema) { + byte[] value = new byte[VALUE_BYTES]; + for (int i = 0; i < value.length; i++) { + value[i] = (byte) 'x'; + } + List vectors = new ArrayList<>(COLUMNS); + for (int c = 0; c < COLUMNS; c++) { + VarCharVector vector = new VarCharVector("col" + c, alloc); + vector.allocateNew((long) ROWS * VALUE_BYTES, ROWS); + for (int r = 0; r < ROWS; r++) { + vector.setSafe(r, value); + } + vector.setValueCount(ROWS); + vectors.add(vector); + } + try (VectorSchemaRoot source = new VectorSchemaRoot(vectors)) { + long total = 0; + for (FieldVector vector : source.getFieldVectors()) { + total += vector.getBufferSize(); + } + assertTrue(total > TINY_LIMIT, "test setup: batch must exceed the consumer limit"); + Data.exportVectorSchemaRoot(alloc, source, null, array, schema); + } + } +} From fa20039f39a7deacf5624ba8a6ee10e9d31e98ce Mon Sep 17 00:00:00 2001 From: David Li Date: Fri, 14 Aug 2026 10:57:43 +0900 Subject: [PATCH 408/408] GH-1241: Bump vcpkg version to stay in sync with apache/arrow (#1260) This fixes the CI for Linux. Closes #1241. --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 51daa0406c..ef0c5fb101 100644 --- a/.env +++ b/.env @@ -53,4 +53,4 @@ MAVEN=3.9.9 # Versions for various dependencies used to build artifacts # Keep in sync with apache/arrow ARROW_REPO_ROOT=./arrow -VCPKG="66c0373dc7fca549e5803087b9487edfe3aca0a1" # 2026.01.16 Release +VCPKG="9b965a116838c6cdcd36bca60d1b81b030c8ab8d" # 2026.05.27 (not release, upstream commit)