diff --git a/c/pom.xml b/c/pom.xml
index c90b6dc0ef..27b6619c4c 100644
--- a/c/pom.xml
+++ b/c/pom.xml
@@ -22,7 +22,7 @@ under the License.
org.apache.arrow
arrow-java-root
- 19.0.0-SNAPSHOT
+ 20.0.0-SNAPSHOT
arrow-c-data
diff --git a/c/src/main/java/org/apache/arrow/c/ReferenceCountedArrowArray.java b/c/src/main/java/org/apache/arrow/c/ReferenceCountedArrowArray.java
index cf50f9417b..f51fb25105 100644
--- a/c/src/main/java/org/apache/arrow/c/ReferenceCountedArrowArray.java
+++ b/c/src/main/java/org/apache/arrow/c/ReferenceCountedArrowArray.java
@@ -64,13 +64,18 @@ void release() {
*/
ArrowBuf unsafeAssociateAllocation(
BufferAllocator trackingAllocator, long capacity, long memoryAddress) {
+ // Retain only after wrapForeignAllocation succeeds. On the allocator-limit OOM path,
+ // wrapForeignAllocation throws before the ForeignAllocation is associated, so release0()
+ // is not called; retaining first would leave the count elevated with no matching release0().
+ ArrowBuf buf =
+ trackingAllocator.wrapForeignAllocation(
+ new ForeignAllocation(capacity, memoryAddress) {
+ @Override
+ protected void release0() {
+ ReferenceCountedArrowArray.this.release();
+ }
+ });
retain();
- return trackingAllocator.wrapForeignAllocation(
- new ForeignAllocation(capacity, memoryAddress) {
- @Override
- protected void release0() {
- ReferenceCountedArrowArray.this.release();
- }
- });
+ return buf;
}
}
diff --git a/c/src/test/java/org/apache/arrow/c/ImportOutOfMemoryTest.java b/c/src/test/java/org/apache/arrow/c/ImportOutOfMemoryTest.java
new file mode 100644
index 0000000000..7c099f2ef0
--- /dev/null
+++ b/c/src/test/java/org/apache/arrow/c/ImportOutOfMemoryTest.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.c;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Regression test: a mid-import {@link OutOfMemoryException} must not leak the imported array.
+ *
+ * A "producer" allocator owns the exported batch; if the C Data release callback fires, the
+ * producer drains to zero. A too-small consumer allocator forces an OOM part-way through the
+ * import. The test asserts the producer drains, confirming the release callback fired despite the
+ * failure.
+ */
+final class ImportOutOfMemoryTest {
+ private static final int ROWS = 1024;
+ private static final int VALUE_BYTES = 256;
+ private static final int COLUMNS = 4;
+ // Far smaller than the exported batch, so the import OOMs part-way through the buffers.
+ private static final long TINY_LIMIT = 16 * 1024;
+
+ private RootAllocator root;
+
+ @BeforeEach
+ public void setUp() {
+ root = new RootAllocator(Long.MAX_VALUE);
+ }
+
+ @AfterEach
+ public void tearDown() {
+ root.close();
+ }
+
+ @Test
+ public void importOomDoesNotLeakExportedArray() {
+ // "producer" owns only the exported batch buffers; the C Data struct containers live on a
+ // separate allocator (they are consumed/closed by import, which would otherwise muddy the
+ // producer's balance). So producer draining to zero is an exact signal that the array's release
+ // callback fired.
+ try (BufferAllocator producer = root.newChildAllocator("producer", 0, Long.MAX_VALUE);
+ BufferAllocator structs = root.newChildAllocator("structs", 0, Long.MAX_VALUE)) {
+ try (ArrowArray array = ArrowArray.allocateNew(structs);
+ ArrowSchema schema = ArrowSchema.allocateNew(structs)) {
+ exportBatch(producer, array, schema);
+ assertTrue(
+ producer.getAllocatedMemory() > 0, "producer holds the exported batch before import");
+
+ // A consumer allocator far too small to hold the batch: the import throws part-way through.
+ try (BufferAllocator consumer = root.newChildAllocator("consumer", 0, TINY_LIMIT);
+ CDataDictionaryProvider provider = new CDataDictionaryProvider()) {
+ Schema importSchema = Data.importSchema(consumer, schema, provider);
+ try (VectorSchemaRoot importRoot = VectorSchemaRoot.create(importSchema, consumer)) {
+ Exception thrown =
+ assertThrows(
+ Exception.class,
+ () -> Data.importIntoVectorSchemaRoot(consumer, array, importRoot, provider));
+ assertTrue(
+ hasOutOfMemoryCause(thrown),
+ "mid-import failure must be an allocator OOM: " + thrown);
+ }
+ }
+
+ // The array's release callback must have fired despite the mid-import OOM, freeing the
+ // whole exported batch. On the unfixed retain-before-wrap code the batch is stranded.
+ assertEquals(
+ 0L,
+ producer.getAllocatedMemory(),
+ "import OOM leaked the exported batch (producer not drained)");
+ }
+ }
+ }
+
+ /** True if {@code t} is, or is caused by, an Arrow {@link OutOfMemoryException}. */
+ private static boolean hasOutOfMemoryCause(Throwable t) {
+ for (Throwable cause = t; cause != null; cause = cause.getCause()) {
+ if (cause instanceof OutOfMemoryException) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Builds a wide multi-column VarChar batch on {@code alloc} and exports it into the C structs.
+ */
+ private void exportBatch(BufferAllocator alloc, ArrowArray array, ArrowSchema schema) {
+ byte[] value = new byte[VALUE_BYTES];
+ for (int i = 0; i < value.length; i++) {
+ value[i] = (byte) 'x';
+ }
+ List vectors = new ArrayList<>(COLUMNS);
+ for (int c = 0; c < COLUMNS; c++) {
+ VarCharVector vector = new VarCharVector("col" + c, alloc);
+ vector.allocateNew((long) ROWS * VALUE_BYTES, ROWS);
+ for (int r = 0; r < ROWS; r++) {
+ vector.setSafe(r, value);
+ }
+ vector.setValueCount(ROWS);
+ vectors.add(vector);
+ }
+ try (VectorSchemaRoot source = new VectorSchemaRoot(vectors)) {
+ long total = 0;
+ for (FieldVector vector : source.getFieldVectors()) {
+ total += vector.getBufferSize();
+ }
+ assertTrue(total > TINY_LIMIT, "test setup: batch must exceed the consumer limit");
+ Data.exportVectorSchemaRoot(alloc, source, null, array, schema);
+ }
+ }
+}
diff --git a/ci/docker/conda-jni.dockerfile b/ci/docker/conda-jni.dockerfile
index e14db73688..3f31b74052 100644
--- a/ci/docker/conda-jni.dockerfile
+++ b/ci/docker/conda-jni.dockerfile
@@ -17,7 +17,7 @@
FROM ghcr.io/mamba-org/micromamba:ubuntu24.04
-ARG jdk=11
+ARG jdk=17
ARG maven=3.9.9
RUN micromamba install -y \
diff --git a/ci/docker/vcpkg-jni.dockerfile b/ci/docker/vcpkg-jni.dockerfile
index d6bd322a39..f2f5d0d45a 100644
--- a/ci/docker/vcpkg-jni.dockerfile
+++ b/ci/docker/vcpkg-jni.dockerfile
@@ -20,7 +20,7 @@ FROM ${base}
# Install Java
# We need Java for JNI headers, but we don't invoke Maven in this build.
-ARG java=11
+ARG java=17
RUN dnf install -y java-$java-openjdk-devel && dnf clean all
# For ci/scripts/{cpp,java}_*.sh
diff --git a/ci/scripts/test.sh b/ci/scripts/test.sh
index cacc20034e..8061ee455d 100755
--- a/ci/scripts/test.sh
+++ b/ci/scripts/test.sh
@@ -34,10 +34,11 @@ fi
mvn="mvn -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
# Use `2 * ncores` threads
mvn="${mvn} -T 2C"
+mvn="${mvn} -Denforcer.skip=true"
pushd "${build_dir}"
-${mvn} -Darrow.test.dataRoot="${source_dir}/testing/data" clean test
+${mvn} -Darrow.test.dataRoot="${source_dir}/testing/data" test
projects=()
if [ "${ARROW_JAVA_JNI}" = "ON" ]; then
@@ -46,7 +47,7 @@ if [ "${ARROW_JAVA_JNI}" = "ON" ]; then
projects+=(gandiva)
fi
if [ "${#projects[@]}" -gt 0 ]; then
- ${mvn} clean test \
+ ${mvn} test \
-Parrow-jni \
-pl "$(
IFS=,
@@ -56,7 +57,7 @@ if [ "${#projects[@]}" -gt 0 ]; then
fi
if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then
- ${mvn} clean test -Parrow-c-data -pl c -Darrow.c.jni.dist.dir="${java_jni_dist_dir}"
+ ${mvn} test -Parrow-c-data -pl c -Darrow.c.jni.dist.dir="${java_jni_dist_dir}"
fi
popd
diff --git a/compose.yaml b/compose.yaml
index f5082a22aa..4fd825e5a5 100644
--- a/compose.yaml
+++ b/compose.yaml
@@ -40,7 +40,7 @@ services:
# docker compose run ubuntu
# Parameters:
# MAVEN: 3.9.9
- # JDK: 11, 17, 21
+ # JDK: 17, 21
image: ${ARCH}/maven:${MAVEN}-eclipse-temurin-${JDK}
volumes:
- .:/arrow-java:delegated
@@ -60,7 +60,7 @@ services:
# docker compose run conda-jni-cdata
# Parameters:
# MAVEN: 3.9.9
- # JDK: 11, 17, 21
+ # JDK: 17, 21
image: ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration
build:
context: .
@@ -109,5 +109,6 @@ services:
ARROW_JAVA_CDATA: "ON"
CCACHE_DIR: "/ccache"
command:
- ["git config --global --add safe.directory /arrow-java && \
+ ["/bin/bash", "-c",
+ "git config --global --add safe.directory /arrow-java &&
/arrow-java/ci/scripts/jni_manylinux_build.sh /arrow-java /arrow /build/java /arrow-java/jni"]
diff --git a/compression/pom.xml b/compression/pom.xml
index 29f8b41788..aa7dee6f89 100644
--- a/compression/pom.xml
+++ b/compression/pom.xml
@@ -22,7 +22,7 @@ under the License.
org.apache.arrow
arrow-java-root
- 19.0.0-SNAPSHOT
+ 20.0.0-SNAPSHOT
arrow-compression
Arrow Compression
@@ -55,7 +55,7 @@ under the License.
com.github.luben
zstd-jni
- 1.5.7-6
+ 1.5.7-11
diff --git a/compression/src/main/java/org/apache/arrow/compression/Lz4CompressionCodec.java b/compression/src/main/java/org/apache/arrow/compression/Lz4CompressionCodec.java
index 91cefc2a9e..f268e815fe 100644
--- a/compression/src/main/java/org/apache/arrow/compression/Lz4CompressionCodec.java
+++ b/compression/src/main/java/org/apache/arrow/compression/Lz4CompressionCodec.java
@@ -80,6 +80,13 @@ protected ArrowBuf doDecompress(BufferAllocator allocator, ArrowBuf compressedBu
}
byte[] outBytes = out.toByteArray();
+ if (outBytes.length != decompressedLength) {
+ throw new RuntimeException(
+ "Expected != actual decompressed length: "
+ + decompressedLength
+ + " != "
+ + outBytes.length);
+ }
ArrowBuf decompressedBuffer = allocator.buffer(outBytes.length);
decompressedBuffer.setBytes(/* index= */ 0, outBytes);
decompressedBuffer.writerIndex(decompressedLength);
diff --git a/compression/src/main/java/org/apache/arrow/compression/ZstdCompressionCodec.java b/compression/src/main/java/org/apache/arrow/compression/ZstdCompressionCodec.java
index 290723608d..ed46fe81b4 100644
--- a/compression/src/main/java/org/apache/arrow/compression/ZstdCompressionCodec.java
+++ b/compression/src/main/java/org/apache/arrow/compression/ZstdCompressionCodec.java
@@ -44,7 +44,7 @@ protected ArrowBuf doCompress(BufferAllocator allocator, ArrowBuf uncompressedBu
long bytesWritten =
Zstd.compressUnsafe(
compressedBuffer.memoryAddress() + CompressionUtil.SIZE_OF_UNCOMPRESSED_LENGTH,
- dstSize,
+ maxSize,
/*src*/ uncompressedBuffer.memoryAddress(),
/* srcSize= */ uncompressedBuffer.writerIndex(),
/* level= */ this.compressionLevel);
diff --git a/compression/src/test/java/org/apache/arrow/compression/TestCompressionCodec.java b/compression/src/test/java/org/apache/arrow/compression/TestCompressionCodec.java
index b8fb4e28b9..d2d2921649 100644
--- a/compression/src/test/java/org/apache/arrow/compression/TestCompressionCodec.java
+++ b/compression/src/test/java/org/apache/arrow/compression/TestCompressionCodec.java
@@ -20,6 +20,7 @@
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.ByteArrayOutputStream;
@@ -59,6 +60,7 @@
import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
@@ -231,6 +233,26 @@ void testEmptyBuffer(int vectorLength, CompressionCodec codec) throws Exception
AutoCloseables.close(decompressedBuffers);
}
+ @Test
+ void testLz4DecompressRejectsWrongLength() {
+ byte[] data = new byte[512]; // all zeros, highly compressible
+ ArrowBuf orig = allocator.buffer(data.length);
+ orig.setBytes(0, data);
+ orig.writerIndex(data.length);
+
+ CompressionCodec codec = new Lz4CompressionCodec();
+ ArrowBuf compressed = codec.compress(allocator, orig);
+
+ // tamper with the 8-byte uncompressed-length prefix so it no longer matches
+ // the real decompressed size
+ compressed.setLong(0, 1_000_000L);
+
+ RuntimeException e =
+ assertThrows(RuntimeException.class, () -> codec.decompress(allocator, compressed));
+ assertTrue(e.getMessage().contains("decompressed length"));
+ compressed.close();
+ }
+
private static Stream codecTypes() {
return Arrays.stream(CompressionUtil.CodecType.values());
}
diff --git a/dataset/pom.xml b/dataset/pom.xml
index 686a234358..5acc837860 100644
--- a/dataset/pom.xml
+++ b/dataset/pom.xml
@@ -22,7 +22,7 @@ under the License.
org.apache.arrow
arrow-java-root
- 19.0.0-SNAPSHOT
+ 20.0.0-SNAPSHOT
arrow-dataset
@@ -32,7 +32,7 @@ under the License.
../../../cpp/release-build/
- 1.17.0
+ 1.17.1
1.12.1
@@ -130,7 +130,7 @@ under the License.
org.apache.orc
orc-core
- 2.2.2
+ 2.3.0
test
@@ -156,7 +156,7 @@ under the License.
commons-io
commons-io
- 2.21.0
+ 2.22.0
test
diff --git a/dev/release/bump_version.sh b/dev/release/bump_version.sh
index 458e930f98..68cafb99bd 100755
--- a/dev/release/bump_version.sh
+++ b/dev/release/bump_version.sh
@@ -37,6 +37,7 @@ if [ ! -f "${SOURCE_DIR}/.env" ]; then
exit 1
fi
. "${SOURCE_DIR}/.env"
+export GH_TOKEN
cd "${SOURCE_TOP_DIR}"
diff --git a/dev/release/release.sh b/dev/release/release.sh
index f08a618c4f..d1db7ad05a 100755
--- a/dev/release/release.sh
+++ b/dev/release/release.sh
@@ -36,6 +36,7 @@ if [ ! -f "${SOURCE_DIR}/.env" ]; then
exit 1
fi
. "${SOURCE_DIR}/.env"
+export GH_TOKEN
git_origin_url="$(git remote get-url origin)"
repository="${git_origin_url#*github.com?}"
diff --git a/dev/release/release_rc.sh b/dev/release/release_rc.sh
index ff77718b8d..0920edbe35 100755
--- a/dev/release/release_rc.sh
+++ b/dev/release/release_rc.sh
@@ -42,6 +42,7 @@ if [ ! -f "${SOURCE_DIR}/.env" ]; then
exit 1
fi
. "${SOURCE_DIR}/.env"
+export GH_TOKEN
cd "${SOURCE_TOP_DIR}"
diff --git a/docs/source/cdata.rst b/docs/source/cdata.rst
index 9643d88df3..7b2924d259 100644
--- a/docs/source/cdata.rst
+++ b/docs/source/cdata.rst
@@ -101,8 +101,8 @@ without writing JNI bindings ourselves.
1.0-SNAPSHOT
- 8
- 8
+ 17
+ 17
9.0.0
@@ -237,8 +237,8 @@ For this example, we will build a JAR with all dependencies bundled.
cpptojava
1.0-SNAPSHOT
- 8
- 8
+ 17
+ 17
9.0.0
diff --git a/docs/source/developers/building.rst b/docs/source/developers/building.rst
index f9ef7daea8..b682957714 100644
--- a/docs/source/developers/building.rst
+++ b/docs/source/developers/building.rst
@@ -32,7 +32,7 @@ Arrow Java uses the `Maven `_ build system.
Building requires:
-* JDK 11+
+* JDK 17+
* Maven 3+
.. note::
@@ -345,7 +345,7 @@ configuration file usually located under ``${HOME}/.m2`` with the following snip
jdk
- 21
+ 21
temurin
@@ -383,11 +383,11 @@ Arrow repository, and update the following settings:
right click the directory, and select Mark Directory as > Generated Sources
Root. There is no need to mark other generated sources directories, as only
the ``vector`` module generates sources.
-* For JDK 11, due to an `IntelliJ bug
- `__, you must go into
+* Due to an `IntelliJ bug
+ `__, you may need to go into
Settings > Build, Execution, Deployment > Compiler > Java Compiler and disable
"Use '--release' option for cross-compilation (Java 9 and later)". Otherwise
- you will get an error like "package sun.misc does not exist".
+ you may get an error like "package sun.misc does not exist".
* You may want to disable error-prone entirely if it gives spurious
warnings (disable both error-prone profiles in the Maven tool window
and "Reload All Maven Projects").
@@ -397,7 +397,7 @@ Arrow repository, and update the following settings:
* To enable debugging JNI-based modules like ``dataset``,
activate specific profiles in the Maven tab under "Profiles".
Ensure the profiles ``arrow-c-data``, ``arrow-jni``, ``generate-libs-cdata-all-os``,
- ``generate-libs-jni-macos-linux``, and ``jdk11+`` are enabled, so that the
+ ``generate-libs-jni-macos-linux``, and ``jdk17+`` are enabled, so that the
IDE can build them and enable debugging.
You may not need to update all of these settings if you build/test with the
@@ -478,8 +478,8 @@ Installing Manually
.. code-block:: xml
- 8
- 8
+ 17
+ 17
9.0.0.dev501
diff --git a/docs/source/flight_sql_jdbc_driver.rst b/docs/source/flight_sql_jdbc_driver.rst
index 4deb726b33..6d40434a22 100644
--- a/docs/source/flight_sql_jdbc_driver.rst
+++ b/docs/source/flight_sql_jdbc_driver.rst
@@ -27,7 +27,7 @@ Flight SQL.
Installation and Requirements
=============================
-The driver is compatible with JDK 11+. Note that the following JVM
+The driver is compatible with JDK 17+. Note that the following JVM
parameter is required:
.. code-block:: shell
diff --git a/docs/source/install.rst b/docs/source/install.rst
index b2b1c7163f..e0b34515ef 100644
--- a/docs/source/install.rst
+++ b/docs/source/install.rst
@@ -27,8 +27,8 @@ Java modules are regularly built and tested on macOS and Linux distributions.
Java Compatibility
==================
-Java modules are compatible with JDK 11 and above. Currently, JDK versions
-11, 17, 21, and latest are tested in CI.
+Java modules are compatible with JDK 17 and above. Currently, JDK versions
+17, 21, and latest are tested in CI.
Note that some JDK internals must be exposed by
adding ``--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED`` to the ``java`` command:
diff --git a/docs/source/jdbc.rst b/docs/source/jdbc.rst
index 2f57c34bf8..e054127faa 100644
--- a/docs/source/jdbc.rst
+++ b/docs/source/jdbc.rst
@@ -276,7 +276,7 @@ mapping, with additional support for the UUID extension type noted below.
JDBC value, because a JDBC Timestamp is in UTC, and we have no
timezone information. In this case, the default binder will call
`setTimestamp(int, Timestamp)
- `_,
+ `_,
which will lead to the driver using the "default timezone" (that of
the Java VM).
* \(3) For the Flight SQL JDBC driver, the Arrow UUID extension type
diff --git a/docs/source/memory.rst b/docs/source/memory.rst
index 4a71ed846a..5b9148223a 100644
--- a/docs/source/memory.rst
+++ b/docs/source/memory.rst
@@ -341,7 +341,7 @@ How this works:
.. _`newChildAllocator`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/RootAllocator.html#newChildAllocator-java.lang.String-org.apache.arrow.memory.AllocationListener-long-long-
.. _`Netty`: https://netty.io/wiki/
.. _`sun.misc.unsafe`: https://web.archive.org/web/20210929024401/http://www.docjar.com/html/api/sun/misc/Unsafe.java.html
-.. _`Direct Memory`: https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/nio/ByteBuffer.html
+.. _`Direct Memory`: https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/nio/ByteBuffer.html
.. _`ReferenceManager`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/ReferenceManager.html
.. _`ReferenceManager.release`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/ReferenceManager.html#release--
.. _`ReferenceManager.retain`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/ReferenceManager.html#retain--
diff --git a/docs/source/overview.rst b/docs/source/overview.rst
index be579c1495..1188054114 100644
--- a/docs/source/overview.rst
+++ b/docs/source/overview.rst
@@ -45,6 +45,9 @@ but some modules are JNI bindings to the C++ library.
* - arrow-vector
- An off-heap reference implementation for Arrow columnar data format.
- Native
+ * - arrow-vector-codegen
+ - Template files for Arrow datatypes suitable for code generation.
+ - Native
* - arrow-tools
- Java applications for working with Arrow ValueVectors.
- Native
diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml
index b1f755844e..9ae402cdc4 100644
--- a/flight/flight-core/pom.xml
+++ b/flight/flight-core/pom.xml
@@ -22,7 +22,7 @@ under the License.
org.apache.arrow
arrow-flight
- 19.0.0-SNAPSHOT
+ 20.0.0-SNAPSHOT
flight-core
@@ -134,7 +134,7 @@ under the License.
com.google.api.grpc
proto-google-common-protos
- 2.63.2
+ 2.72.0
test
diff --git a/flight/flight-core/src/main/java/org/apache/arrow/flight/CallHeaders.java b/flight/flight-core/src/main/java/org/apache/arrow/flight/CallHeaders.java
index f4f6486a3c..0939d232cf 100644
--- a/flight/flight-core/src/main/java/org/apache/arrow/flight/CallHeaders.java
+++ b/flight/flight-core/src/main/java/org/apache/arrow/flight/CallHeaders.java
@@ -26,10 +26,20 @@ public interface CallHeaders {
/** Get the value of a metadata key. If multiple values are present, then get the last one. */
byte[] getByte(String key);
- /** Get all values present for the given metadata key. */
+ /**
+ * Get all values present for the given metadata key.
+ *
+ * @param key the metadata key
+ * @return an iterable of all values for the key. Returns an empty iterable if no value to return.
+ */
Iterable getAll(String key);
- /** Get all values present for the given metadata key. */
+ /**
+ * Get all values present for the given metadata key.
+ *
+ * @param key the metadata key
+ * @return an iterable of all values for the key. Returns an empty iterable if no value to return.
+ */
Iterable getAllByte(String key);
/**
diff --git a/flight/flight-core/src/main/java/org/apache/arrow/flight/ServerSessionMiddleware.java b/flight/flight-core/src/main/java/org/apache/arrow/flight/ServerSessionMiddleware.java
index 47fd6f1366..5ec01b9c83 100644
--- a/flight/flight-core/src/main/java/org/apache/arrow/flight/ServerSessionMiddleware.java
+++ b/flight/flight-core/src/main/java/org/apache/arrow/flight/ServerSessionMiddleware.java
@@ -80,20 +80,18 @@ public ServerSessionMiddleware onCallStarted(
String sessionId = null;
final Iterable it = incomingHeaders.getAll("cookie");
- if (it != null) {
- findIdCookie:
- for (final String headerValue : it) {
- for (final String cookie : headerValue.split(" ;")) {
- final String[] cookiePair = cookie.split("=");
- if (cookiePair.length != 2) {
- // Soft failure: Ignore invalid cookie list field
- break;
- }
-
- if (sessionCookieName.equals(cookiePair[0]) && cookiePair[1].length() > 0) {
- sessionId = cookiePair[1];
- break findIdCookie;
- }
+ findIdCookie:
+ for (final String headerValue : it) {
+ for (final String cookie : headerValue.split(" ;")) {
+ final String[] cookiePair = cookie.split("=");
+ if (cookiePair.length != 2) {
+ // Soft failure: Ignore invalid cookie list field
+ break;
+ }
+
+ if (sessionCookieName.equals(cookiePair[0]) && cookiePair[1].length() > 0) {
+ sessionId = cookiePair[1];
+ break findIdCookie;
}
}
}
diff --git a/flight/flight-core/src/main/java/org/apache/arrow/flight/client/ClientCookieMiddleware.java b/flight/flight-core/src/main/java/org/apache/arrow/flight/client/ClientCookieMiddleware.java
index e5eb934001..b33e6b7ecc 100644
--- a/flight/flight-core/src/main/java/org/apache/arrow/flight/client/ClientCookieMiddleware.java
+++ b/flight/flight-core/src/main/java/org/apache/arrow/flight/client/ClientCookieMiddleware.java
@@ -100,10 +100,7 @@ public void onBeforeSendingHeaders(CallHeaders outgoingHeaders) {
@Override
public void onHeadersReceived(CallHeaders incomingHeaders) {
- final Iterable setCookieHeaders = incomingHeaders.getAll(SET_COOKIE_HEADER);
- if (setCookieHeaders != null) {
- factory.updateCookies(setCookieHeaders);
- }
+ factory.updateCookies(incomingHeaders.getAll(SET_COOKIE_HEADER));
}
@Override
diff --git a/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/GetReadableBuffer.java b/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/GetReadableBuffer.java
index 45c32a86c6..fcba88d212 100644
--- a/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/GetReadableBuffer.java
+++ b/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/GetReadableBuffer.java
@@ -87,13 +87,13 @@ public static void readIntoBuffer(
final InputStream stream, final ArrowBuf buf, final int size, final boolean fastPath)
throws IOException {
ReadableBuffer readableBuffer = fastPath ? getReadableBuffer(stream) : null;
+ byte[] heapBytes = new byte[size];
if (readableBuffer != null) {
- readableBuffer.readBytes(buf.nioBuffer(0, size));
+ readableBuffer.readBytes(heapBytes, 0, size);
} else {
- byte[] heapBytes = new byte[size];
ByteStreams.readFully(stream, heapBytes);
- buf.writeBytes(heapBytes);
}
+ buf.writeBytes(heapBytes);
buf.writerIndex(size);
}
}
diff --git a/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/MetadataAdapter.java b/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/MetadataAdapter.java
index a1de16ede6..64a0769d63 100644
--- a/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/MetadataAdapter.java
+++ b/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/MetadataAdapter.java
@@ -18,6 +18,7 @@
import io.grpc.Metadata;
import java.nio.charset.StandardCharsets;
+import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import java.util.stream.Collectors;
@@ -53,13 +54,17 @@ public byte[] getByte(String key) {
@Override
public Iterable getAll(String key) {
- return this.metadata.getAll(Metadata.Key.of(key, Metadata.ASCII_STRING_MARSHALLER));
+ final Iterable all =
+ this.metadata.getAll(Metadata.Key.of(key, Metadata.ASCII_STRING_MARSHALLER));
+ return all != null ? all : Collections.emptyList();
}
@Override
public Iterable getAllByte(String key) {
if (key.endsWith(Metadata.BINARY_HEADER_SUFFIX)) {
- return this.metadata.getAll(Metadata.Key.of(key, Metadata.BINARY_BYTE_MARSHALLER));
+ final Iterable all =
+ this.metadata.getAll(Metadata.Key.of(key, Metadata.BINARY_BYTE_MARSHALLER));
+ return all != null ? all : Collections.emptyList();
}
return StreamSupport.stream(getAll(key).spliterator(), false)
.map(String::getBytes)
diff --git a/flight/flight-core/src/test/java/org/apache/arrow/flight/TestCallOptions.java b/flight/flight-core/src/test/java/org/apache/arrow/flight/TestCallOptions.java
index a54ce69812..8aef9c69a1 100644
--- a/flight/flight-core/src/test/java/org/apache/arrow/flight/TestCallOptions.java
+++ b/flight/flight-core/src/test/java/org/apache/arrow/flight/TestCallOptions.java
@@ -21,6 +21,7 @@
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
@@ -110,6 +111,16 @@ public void mixedProperties() {
testHeaders(headers);
}
+ @Test
+ public void getAllReturnsEmptyIterableForMissingKey() {
+ FlightCallHeaders headers = new FlightCallHeaders();
+
+ assertNotNull(headers.getAll("missing"));
+ assertFalse(headers.getAll("missing").iterator().hasNext());
+ assertNotNull(headers.getAllByte("missing-bin"));
+ assertFalse(headers.getAllByte("missing-bin").iterator().hasNext());
+ }
+
private void testHeaders(CallHeaders headers) {
try (BufferAllocator a = new RootAllocator(Long.MAX_VALUE);
HeaderProducer producer = new HeaderProducer();
diff --git a/flight/flight-core/src/test/java/org/apache/arrow/flight/TestErrorMetadata.java b/flight/flight-core/src/test/java/org/apache/arrow/flight/TestErrorMetadata.java
index a9a3e355bc..214614defd 100644
--- a/flight/flight-core/src/test/java/org/apache/arrow/flight/TestErrorMetadata.java
+++ b/flight/flight-core/src/test/java/org/apache/arrow/flight/TestErrorMetadata.java
@@ -20,6 +20,7 @@
import static org.apache.arrow.flight.Location.forGrpcInsecure;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
@@ -119,6 +120,16 @@ public void testFlightMetadata() throws Exception {
}
}
+ @Test
+ public void getAllReturnsEmptyIterableForMissingKey() {
+ ErrorFlightMetadata metadata = new ErrorFlightMetadata();
+
+ assertNotNull(metadata.getAll("missing"));
+ assertFalse(metadata.getAll("missing").iterator().hasNext());
+ assertNotNull(metadata.getAllByte("missing-bin"));
+ assertFalse(metadata.getAllByte("missing-bin").iterator().hasNext());
+ }
+
private static class StatusRuntimeExceptionProducer extends NoOpFlightProducer {
private final PerfOuterClass.Perf perf;
diff --git a/flight/flight-core/src/test/java/org/apache/arrow/flight/auth/TestBasicAuth.java b/flight/flight-core/src/test/java/org/apache/arrow/flight/auth/TestBasicAuth.java
index 0c63785c88..0f202ba2d9 100644
--- a/flight/flight-core/src/test/java/org/apache/arrow/flight/auth/TestBasicAuth.java
+++ b/flight/flight-core/src/test/java/org/apache/arrow/flight/auth/TestBasicAuth.java
@@ -178,6 +178,12 @@ public static void shutdown() throws Exception {
AutoCloseables.close(server);
allocator.getChildAllocators().forEach(BufferAllocator::close);
+
+ // gRPC/Netty may still be releasing Arrow buffers asynchronously after server shutdown.
+ // Poll briefly to allow in-flight buffer releases to complete before closing the allocator.
+ for (int i = 0; i < 20 && allocator.getAllocatedMemory() > 0; i++) {
+ Thread.sleep(100);
+ }
AutoCloseables.close(allocator);
}
}
diff --git a/flight/flight-core/src/test/java/org/apache/arrow/flight/grpc/TestMetadataAdapter.java b/flight/flight-core/src/test/java/org/apache/arrow/flight/grpc/TestMetadataAdapter.java
new file mode 100644
index 0000000000..b0f5dcfcfc
--- /dev/null
+++ b/flight/flight-core/src/test/java/org/apache/arrow/flight/grpc/TestMetadataAdapter.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.flight.grpc;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+import io.grpc.Metadata;
+import org.junit.jupiter.api.Test;
+
+public class TestMetadataAdapter {
+
+ @Test
+ public void getAllReturnsEmptyIterableForMissingKey() {
+ MetadataAdapter headers = new MetadataAdapter(new Metadata());
+
+ assertNotNull(headers.getAll("missing"));
+ assertFalse(headers.getAll("missing").iterator().hasNext());
+ assertNotNull(headers.getAllByte("missing-bin"));
+ assertFalse(headers.getAllByte("missing-bin").iterator().hasNext());
+ }
+}
diff --git a/flight/flight-integration-tests/pom.xml b/flight/flight-integration-tests/pom.xml
index 78a2d08ee1..f6ae8e16a5 100644
--- a/flight/flight-integration-tests/pom.xml
+++ b/flight/flight-integration-tests/pom.xml
@@ -22,7 +22,7 @@ under the License.
org.apache.arrow
arrow-flight
- 19.0.0-SNAPSHOT
+ 20.0.0-SNAPSHOT
flight-integration-tests
@@ -58,7 +58,7 @@ under the License.
commons-cli
commons-cli
- 1.9.0
+ 1.11.0
org.slf4j
@@ -101,7 +101,7 @@ under the License.
-
+
META-INF/LICENSE.txt
src/shade/LICENSE.txt
diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml
index da00baf32a..be2ee32868 100644
--- a/flight/flight-sql-jdbc-core/pom.xml
+++ b/flight/flight-sql-jdbc-core/pom.xml
@@ -22,7 +22,7 @@ under the License.
org.apache.arrow
arrow-flight
- 19.0.0-SNAPSHOT
+ 20.0.0-SNAPSHOT
flight-sql-jdbc-core
@@ -105,7 +105,7 @@ under the License.
commons-io
commons-io
- 2.21.0
+ 2.22.0
test
@@ -123,25 +123,25 @@ under the License.
com.squareup.okhttp3
mockwebserver3
- 5.3.2
+ 5.4.0
test
com.squareup.okhttp3
mockwebserver3-junit5
- 5.3.2
+ 5.4.0
test
com.squareup.okhttp3
okhttp-jvm
- 5.3.2
+ 5.4.0
test
com.squareup.okio
okio-jvm
- 3.16.4
+ 3.17.0
test
@@ -159,13 +159,13 @@ under the License.
org.apache.calcite.avatica
avatica
- 1.26.0
+ 1.27.0
org.bouncycastle
bcpkix-jdk18on
- 1.83
+ 1.84
@@ -176,13 +176,13 @@ under the License.
com.github.ben-manes.caffeine
caffeine
- 3.2.3
+ 3.2.4
com.nimbusds
oauth2-oidc-sdk
- 11.20.1
+ 11.37.2
diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java
index 502270e1cd..0110525fea 100644
--- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java
+++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java
@@ -45,6 +45,7 @@
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Arrays;
+import java.util.Collections;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.List;
@@ -86,9 +87,12 @@
import org.apache.arrow.vector.util.Text;
import org.apache.calcite.avatica.AvaticaConnection;
import org.apache.calcite.avatica.AvaticaDatabaseMetaData;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/** Arrow Flight JDBC's implementation of {@link DatabaseMetaData}. */
public class ArrowDatabaseMetadata extends AvaticaDatabaseMetaData {
+ private static final Logger LOGGER = LoggerFactory.getLogger(ArrowDatabaseMetadata.class);
private static final String JAVA_REGEX_SPECIALS = "[]()|^-+*?{}$\\.";
private static final Charset CHARSET = StandardCharsets.UTF_8;
private static final byte[] EMPTY_BYTE_ARRAY = new byte[0];
@@ -774,7 +778,34 @@ private T getSqlInfoAndCacheIfCacheIsEmpty(
}
}
}
- return desiredType.cast(cachedSqlInfo.get(sqlInfoCommand));
+ T value = desiredType.cast(cachedSqlInfo.get(sqlInfoCommand));
+ if (value != null) {
+ return value;
+ }
+ LOGGER.debug(
+ "SqlInfo {} not provided by server, returning default for type {}",
+ sqlInfoCommand.name(),
+ desiredType.getSimpleName());
+
+ // Return sensible defaults when SqlInfo is unavailable
+ if (desiredType == Long.class) {
+ return desiredType.cast(0L);
+ } else if (desiredType == Integer.class) {
+ return desiredType.cast(0);
+ } else if (desiredType == Boolean.class) {
+ return desiredType.cast(false);
+ } else if (desiredType == String.class) {
+ return desiredType.cast("");
+ } else if (desiredType == Map.class) {
+ return desiredType.cast(Collections.emptyMap());
+ } else if (desiredType == List.class) {
+ return desiredType.cast(Collections.emptyList());
+ }
+
+ throw new SQLException(
+ String.format(
+ "The value of the SqlInfo %s is null and it could not be cast to %s.",
+ sqlInfoCommand.name(), desiredType.getName()));
}
private Optional convertListSqlInfoToString(final List sqlInfoList) {
diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriver.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriver.java
index 53e6120f62..12ef8030d7 100644
--- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriver.java
+++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriver.java
@@ -75,7 +75,9 @@ public Logger getParentLogger() {
public ArrowFlightConnection connect(final String url, final Properties info)
throws SQLException {
final Properties properties = new Properties(info);
- properties.putAll(info);
+ if (info != null) {
+ properties.putAll(info);
+ }
if (url != null) {
final Optional> maybeProperties = getUrlsArgs(url);
diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcFlightStreamResultSet.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcFlightStreamResultSet.java
index 2885f7895b..376e5b11e7 100644
--- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcFlightStreamResultSet.java
+++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcFlightStreamResultSet.java
@@ -106,7 +106,7 @@ static ArrowFlightJdbcFlightStreamResultSet fromFlightInfo(
final TimeZone timeZone = TimeZone.getDefault();
final QueryState state = new QueryState();
- final Meta.Signature signature = ArrowFlightMetaImpl.newSignature(null, null, null);
+ final Meta.Signature signature = ArrowFlightMetaImpl.newSignature(null, null, null, null);
final AvaticaResultSetMetaData resultSetMetaData =
new AvaticaResultSetMetaData(null, null, signature);
diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java
index 49334951de..ad6670a001 100644
--- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java
+++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java
@@ -73,7 +73,7 @@ public static ArrowFlightJdbcVectorSchemaRootResultSet fromVectorSchemaRoot(
final TimeZone timeZone = TimeZone.getDefault();
final QueryState state = new QueryState();
- final Meta.Signature signature = ArrowFlightMetaImpl.newSignature(null, null, null);
+ final Meta.Signature signature = ArrowFlightMetaImpl.newSignature(null, null, null, null);
final AvaticaResultSetMetaData resultSetMetaData =
new AvaticaResultSetMetaData(null, null, signature);
@@ -128,7 +128,7 @@ public Object getObject(int columnIndex) throws SQLException {
if (metaData.type.id == Types.TIMESTAMP_WITH_TIMEZONE) {
return accessor.getTimestamp(localCalendar);
} else {
- return AvaticaSite.get(accessor, metaData.type.id, localCalendar);
+ return AvaticaSite.get(accessor, metaData.type.id, true, localCalendar);
}
}
diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java
index 64529b50c8..0d85b5eddb 100644
--- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java
+++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java
@@ -53,7 +53,8 @@ public ArrowFlightMetaImpl(final AvaticaConnection connection) {
}
/** Construct a signature. */
- static Signature newSignature(final String sql, Schema resultSetSchema, Schema parameterSchema) {
+ static Signature newSignature(
+ final String sql, Schema resultSetSchema, Schema parameterSchema, Boolean isUpdate) {
List columnMetaData =
resultSetSchema == null
? new ArrayList<>()
@@ -62,10 +63,17 @@ static Signature newSignature(final String sql, Schema resultSetSchema, Schema p
parameterSchema == null
? new ArrayList<>()
: ConvertUtils.convertArrowFieldsToAvaticaParameters(parameterSchema.getFields());
- StatementType statementType =
- resultSetSchema == null || resultSetSchema.getFields().isEmpty()
- ? StatementType.IS_DML
- : StatementType.SELECT;
+ // If the server provided the is_update field, use it to determine the statement type
+ StatementType statementType;
+ if (isUpdate != null) {
+ statementType = isUpdate ? StatementType.IS_DML : StatementType.SELECT;
+ } else {
+ // Fall back to the legacy logic: check if the result set schema is empty
+ statementType =
+ resultSetSchema == null || resultSetSchema.getFields().isEmpty()
+ ? StatementType.IS_DML
+ : StatementType.SELECT;
+ }
return new Signature(
columnMetaData,
sql,
@@ -178,7 +186,10 @@ private PreparedStatement prepareForHandle(final String query, StatementHandle h
((ArrowFlightConnection) connection).getClientHandler().prepare(query);
handle.signature =
newSignature(
- query, preparedStatement.getDataSetSchema(), preparedStatement.getParameterSchema());
+ query,
+ preparedStatement.getDataSetSchema(),
+ preparedStatement.getParameterSchema(),
+ preparedStatement.isUpdate());
statementHandlePreparedStatementMap.put(new StatementHandleKey(handle), preparedStatement);
return preparedStatement;
}
diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessor.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessor.java
index f0fa55fa82..cd762fb1ac 100644
--- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessor.java
+++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessor.java
@@ -36,6 +36,10 @@
import java.util.Calendar;
import java.util.Map;
import java.util.function.IntSupplier;
+import org.joou.UByte;
+import org.joou.UInteger;
+import org.joou.ULong;
+import org.joou.UShort;
/** Base Jdbc Accessor. */
public abstract class ArrowFlightJdbcAccessor implements Accessor {
@@ -99,6 +103,26 @@ public long getLong() throws SQLException {
throw getOperationNotSupported(this.getClass());
}
+ @Override
+ public UByte getUByte() throws SQLException {
+ throw getOperationNotSupported(this.getClass());
+ }
+
+ @Override
+ public UShort getUShort() throws SQLException {
+ throw getOperationNotSupported(this.getClass());
+ }
+
+ @Override
+ public UInteger getUInt() throws SQLException {
+ throw getOperationNotSupported(this.getClass());
+ }
+
+ @Override
+ public ULong getULong() throws SQLException {
+ throw getOperationNotSupported(this.getClass());
+ }
+
@Override
public float getFloat() throws SQLException {
throw getOperationNotSupported(this.getClass());
diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java
index f0ea284239..719cc38a2b 100644
--- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java
+++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java
@@ -388,6 +388,14 @@ public interface PreparedStatement extends AutoCloseable {
*/
Schema getParameterSchema();
+ /**
+ * Gets whether this {@link PreparedStatement} is an update statement.
+ *
+ * @return {@code true} if this is an update statement, {@code false} if it's a query, or {@code
+ * null} if the server did not provide this information.
+ */
+ @Nullable Boolean isUpdate();
+
void setParameters(VectorSchemaRoot parameters);
@Override
@@ -456,6 +464,12 @@ public long executeUpdate() {
@Override
public StatementType getType() {
+ // If the server provided the is_update field, use it to determine the statement type
+ final Boolean isUpdate = preparedStatement.isUpdate();
+ if (isUpdate != null) {
+ return isUpdate ? StatementType.UPDATE : StatementType.SELECT;
+ }
+ // Fall back to the legacy logic: check if the result set schema is empty
final Schema schema = preparedStatement.getResultSetSchema();
return schema.getFields().isEmpty() ? StatementType.UPDATE : StatementType.SELECT;
}
@@ -475,6 +489,11 @@ public void setParameters(VectorSchemaRoot parameters) {
preparedStatement.setParameters(parameters);
}
+ @Override
+ public Boolean isUpdate() {
+ return preparedStatement.isUpdate();
+ }
+
@Override
public void close() {
try {
diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java
index 81579cc387..3ab1460b27 100644
--- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java
+++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java
@@ -1543,11 +1543,83 @@ public void testEmptySqlInfo() throws Exception {
try (final Connection testConnection =
FLIGHT_SERVER_EMPTY_SQLINFO_TEST_RULE.getConnection(false)) {
final DatabaseMetaData metaData = testConnection.getMetaData();
+
assertThat(metaData.getSQLKeywords(), is(""));
assertThat(metaData.getNumericFunctions(), is(""));
assertThat(metaData.getStringFunctions(), is(""));
assertThat(metaData.getSystemFunctions(), is(""));
assertThat(metaData.getTimeDateFunctions(), is(""));
+
+ assertThat(metaData.getMaxBinaryLiteralLength(), is(0));
+ assertThat(metaData.getMaxCharLiteralLength(), is(0));
+ assertThat(metaData.getMaxColumnNameLength(), is(0));
+ assertThat(metaData.getMaxColumnsInGroupBy(), is(0));
+ assertThat(metaData.getMaxColumnsInIndex(), is(0));
+ assertThat(metaData.getMaxColumnsInOrderBy(), is(0));
+ assertThat(metaData.getMaxColumnsInSelect(), is(0));
+ assertThat(metaData.getMaxColumnsInTable(), is(0));
+ assertThat(metaData.getMaxConnections(), is(0));
+ assertThat(metaData.getMaxCursorNameLength(), is(0));
+ assertThat(metaData.getMaxIndexLength(), is(0));
+ assertThat(metaData.getMaxSchemaNameLength(), is(0));
+ assertThat(metaData.getMaxProcedureNameLength(), is(0));
+ assertThat(metaData.getMaxCatalogNameLength(), is(0));
+ assertThat(metaData.getMaxRowSize(), is(0));
+ assertThat(metaData.getMaxStatementLength(), is(0));
+ assertThat(metaData.getMaxStatements(), is(0));
+ assertThat(metaData.getMaxTableNameLength(), is(0));
+ assertThat(metaData.getMaxTablesInSelect(), is(0));
+ assertThat(metaData.getMaxUserNameLength(), is(0));
+
+ assertThat(metaData.supportsColumnAliasing(), is(false));
+ assertThat(metaData.nullPlusNonNullIsNull(), is(false));
+ assertThat(metaData.supportsTableCorrelationNames(), is(false));
+ assertThat(metaData.supportsDifferentTableCorrelationNames(), is(false));
+ assertThat(metaData.supportsExpressionsInOrderBy(), is(false));
+ assertThat(metaData.supportsOrderByUnrelated(), is(false));
+ assertThat(metaData.supportsLikeEscapeClause(), is(false));
+ assertThat(metaData.supportsNonNullableColumns(), is(false));
+ assertThat(metaData.supportsIntegrityEnhancementFacility(), is(false));
+ assertThat(metaData.isCatalogAtStart(), is(false));
+ assertThat(metaData.supportsSelectForUpdate(), is(false));
+ assertThat(metaData.supportsStoredProcedures(), is(false));
+ assertThat(metaData.supportsCorrelatedSubqueries(), is(false));
+ assertThat(metaData.doesMaxRowSizeIncludeBlobs(), is(false));
+ assertThat(metaData.supportsTransactions(), is(false));
+ assertThat(metaData.dataDefinitionCausesTransactionCommit(), is(false));
+ assertThat(metaData.dataDefinitionIgnoredInTransactions(), is(false));
+ assertThat(metaData.supportsBatchUpdates(), is(false));
+ assertThat(metaData.supportsSavepoints(), is(false));
+ assertThat(metaData.supportsNamedParameters(), is(false));
+ assertThat(metaData.locatorsUpdateCopy(), is(false));
+ assertThat(metaData.supportsStoredFunctionsUsingCallSyntax(), is(false));
+ assertThat(metaData.supportsGroupBy(), is(false));
+ assertThat(metaData.supportsGroupByUnrelated(), is(false));
+ assertThat(metaData.supportsMinimumSQLGrammar(), is(false));
+ assertThat(metaData.supportsCoreSQLGrammar(), is(false));
+ assertThat(metaData.supportsExtendedSQLGrammar(), is(false));
+ assertThat(metaData.supportsANSI92EntryLevelSQL(), is(false));
+ assertThat(metaData.supportsANSI92IntermediateSQL(), is(false));
+ assertThat(metaData.supportsANSI92FullSQL(), is(false));
+ assertThat(metaData.supportsOuterJoins(), is(false));
+ assertThat(metaData.supportsFullOuterJoins(), is(false));
+ assertThat(metaData.supportsLimitedOuterJoins(), is(false));
+ assertThat(metaData.supportsSchemasInProcedureCalls(), is(false));
+ assertThat(metaData.supportsSchemasInIndexDefinitions(), is(false));
+ assertThat(metaData.supportsSchemasInPrivilegeDefinitions(), is(false));
+ assertThat(metaData.supportsCatalogsInIndexDefinitions(), is(false));
+ assertThat(metaData.supportsCatalogsInPrivilegeDefinitions(), is(false));
+ assertThat(metaData.supportsPositionedDelete(), is(false));
+ assertThat(metaData.supportsPositionedUpdate(), is(false));
+ assertThat(metaData.supportsSubqueriesInComparisons(), is(false));
+ assertThat(metaData.supportsSubqueriesInExists(), is(false));
+ assertThat(metaData.supportsSubqueriesInIns(), is(false));
+ assertThat(metaData.supportsSubqueriesInQuantifieds(), is(false));
+ assertThat(metaData.supportsUnion(), is(false));
+ assertThat(metaData.supportsUnionAll(), is(false));
+ assertThat(metaData.supportsConvert(), is(false));
+
+ assertThat(metaData.getDefaultTransactionIsolation(), is(Connection.TRANSACTION_NONE));
}
}
}
diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriverTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriverTest.java
index ae355829d7..88fb9889b6 100644
--- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriverTest.java
+++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriverTest.java
@@ -201,6 +201,30 @@ public void testConnectWithInsensitiveCasePropertyKeys2() throws Exception {
}
}
+ /**
+ * Tests whether the {@link ArrowFlightJdbcDriver} can establish a successful connection to the
+ * Arrow Flight client when provided with null properties.
+ */
+ @Test
+ public void testConnectWithNullProperties() throws Exception {
+ final Driver driver = new ArrowFlightJdbcDriver();
+ try (Connection connection =
+ driver.connect(
+ "jdbc:arrow-flight://"
+ + dataSource.getConfig().getHost()
+ + ":"
+ + dataSource.getConfig().getPort()
+ + "?"
+ + "useEncryption=false"
+ + "&user="
+ + dataSource.getConfig().getUser()
+ + "&password="
+ + dataSource.getConfig().getPassword(),
+ null)) {
+ assertTrue(connection.isValid(300));
+ }
+ }
+
/**
* Tests whether an exception is thrown upon attempting to connect to a malformed URI.
*
diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightPreparedStatementTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightPreparedStatementTest.java
index 0369c3a162..078837adf3 100644
--- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightPreparedStatementTest.java
+++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightPreparedStatementTest.java
@@ -98,6 +98,40 @@ public void testSimpleQueryNoParameterBindingWithExecute() throws SQLException {
}
}
+ @Test
+ public void testSimpleQueryNoParameterBindingWithExecuteV2() throws SQLException {
+ final String query = "SELECT * FROM TEST_V2";
+ final Schema schema =
+ new Schema(Collections.singletonList(Field.nullable("", Types.MinorType.INT.getType())));
+ PRODUCER.addSelectQuery(
+ query,
+ schema,
+ Collections.singletonList(
+ listener -> {
+ try (final BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
+ final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) {
+ root.allocateNew();
+ ((IntVector) root.getVector(0)).setSafe(0, 123);
+ root.setRowCount(1);
+ listener.start(root);
+ listener.putNext();
+ } finally {
+ listener.completed();
+ }
+ }),
+ false);
+ try (final PreparedStatement preparedStatement = connection.prepareStatement(query)) {
+ boolean isResultSet = preparedStatement.execute();
+ assertTrue(isResultSet);
+ final ResultSet resultSet = preparedStatement.getResultSet();
+ assertTrue(resultSet.next());
+ assertEquals(123, resultSet.getInt(1));
+ assertFalse(resultSet.next());
+ assertFalse(preparedStatement.getMoreResults());
+ assertEquals(-1, preparedStatement.getUpdateCount());
+ }
+ }
+
@Test
public void testQueryWithParameterBinding() throws SQLException {
final String query = "Fake query with parameters";
@@ -203,6 +237,20 @@ public void testUpdateQueryWithExecute() throws SQLException {
}
}
+ @Test
+ public void testUpdateQueryWithExecuteV2() throws SQLException {
+ String query = "Fake update with execute V2";
+ PRODUCER.addUpdateQuery(query, /*updatedRows*/ 99, true);
+ try (final PreparedStatement stmt = connection.prepareStatement(query)) {
+ boolean isResultSet = stmt.execute();
+ assertFalse(isResultSet);
+ int updated = stmt.getUpdateCount();
+ assertEquals(99, updated);
+ assertFalse(stmt.getMoreResults());
+ assertEquals(-1, stmt.getUpdateCount());
+ }
+ }
+
@Test
public void testUpdateQueryWithParameters() throws SQLException {
String query = "Fake update with parameters";
diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightStatementExecuteTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightStatementExecuteTest.java
index 632cb0ba56..6acce9c2a6 100644
--- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightStatementExecuteTest.java
+++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightStatementExecuteTest.java
@@ -62,6 +62,9 @@ public class ArrowFlightStatementExecuteTest {
private static final String SAMPLE_LARGE_UPDATE_QUERY =
"UPDATE this_large_table SET this_large_field = that_large_field FROM this_large_test WHERE this_large_condition";
private static final long SAMPLE_LARGE_UPDATE_COUNT = Long.MAX_VALUE;
+ private static final String SAMPLE_QUERY_CMD_V2 = "SELECT * FROM this_test_v2";
+ private static final String SAMPLE_LARGE_UPDATE_QUERY_V2 =
+ "UPDATE this_large_table_v2 SET this_large_field = that_large_field FROM this_large_test WHERE this_large_condition";
private static final MockFlightSqlProducer PRODUCER = new MockFlightSqlProducer();
@RegisterExtension
@@ -96,6 +99,31 @@ public static void setUpBeforeClass() {
}));
PRODUCER.addUpdateQuery(SAMPLE_UPDATE_QUERY, SAMPLE_UPDATE_COUNT);
PRODUCER.addUpdateQuery(SAMPLE_LARGE_UPDATE_QUERY, SAMPLE_LARGE_UPDATE_COUNT);
+
+ // V2 queries with is_update field set
+ PRODUCER.addSelectQuery(
+ SAMPLE_QUERY_CMD_V2,
+ SAMPLE_QUERY_SCHEMA,
+ Collections.singletonList(
+ listener -> {
+ try (final BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
+ final VectorSchemaRoot root =
+ VectorSchemaRoot.create(SAMPLE_QUERY_SCHEMA, allocator)) {
+ final UInt1Vector vector = (UInt1Vector) root.getVector(VECTOR_NAME);
+ IntStream.range(0, SAMPLE_QUERY_ROWS)
+ .forEach(index -> vector.setSafe(index, index));
+ vector.setValueCount(SAMPLE_QUERY_ROWS);
+ root.setRowCount(SAMPLE_QUERY_ROWS);
+ listener.start(root);
+ listener.putNext();
+ } catch (final Throwable throwable) {
+ listener.error(throwable);
+ } finally {
+ listener.completed();
+ }
+ }),
+ false);
+ PRODUCER.addUpdateQuery(SAMPLE_LARGE_UPDATE_QUERY_V2, SAMPLE_LARGE_UPDATE_COUNT, true);
}
@BeforeEach
@@ -168,4 +196,42 @@ public void testUpdateCountShouldStartOnZero() throws SQLException {
is(allOf(equalTo(statement.getLargeUpdateCount()), equalTo(0L))));
assertThat(statement.getResultSet(), is(nullValue()));
}
+
+ @Test
+ public void testExecuteShouldRunSelectQueryV2() throws SQLException {
+ assertThat(statement.execute(SAMPLE_QUERY_CMD_V2), is(true));
+ final Set numbers =
+ IntStream.range(0, SAMPLE_QUERY_ROWS)
+ .boxed()
+ .map(Integer::byteValue)
+ .collect(Collectors.toCollection(HashSet::new));
+ try (final ResultSet resultSet = statement.getResultSet()) {
+ final int columnCount = resultSet.getMetaData().getColumnCount();
+ assertThat(columnCount, is(1));
+ int rowCount = 0;
+ for (; resultSet.next(); rowCount++) {
+ assertThat(numbers.remove(resultSet.getByte(1)), is(true));
+ }
+ assertThat(rowCount, is(equalTo(SAMPLE_QUERY_ROWS)));
+ }
+ assertThat(numbers, is(Collections.emptySet()));
+ assertThat(
+ (long) statement.getUpdateCount(),
+ is(allOf(equalTo(statement.getLargeUpdateCount()), equalTo(-1L))));
+ }
+
+ @Test
+ public void testExecuteShouldRunUpdateQueryForLargeUpdateV2() throws SQLException {
+ assertThat(statement.execute(SAMPLE_LARGE_UPDATE_QUERY_V2), is(false)); // UPDATE query.
+ final long updateCountSmall = statement.getUpdateCount();
+ final long updateCountLarge = statement.getLargeUpdateCount();
+ assertThat(updateCountLarge, is(equalTo(SAMPLE_LARGE_UPDATE_COUNT)));
+ assertThat(
+ updateCountSmall,
+ is(
+ allOf(
+ equalTo((long) AvaticaUtils.toSaturatedInt(updateCountLarge)),
+ not(equalTo(updateCountLarge)))));
+ assertThat(statement.getResultSet(), is(nullValue()));
+ }
}
diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/MockFlightSqlProducer.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/MockFlightSqlProducer.java
index 45c2a96404..6627d91ab6 100644
--- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/MockFlightSqlProducer.java
+++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/MockFlightSqlProducer.java
@@ -87,6 +87,7 @@
import org.apache.arrow.vector.types.pojo.Schema;
import org.apache.arrow.vector.util.JsonStringArrayList;
import org.apache.calcite.avatica.Meta.StatementType;
+import org.checkerframework.checker.nullness.qual.Nullable;
/** An ad-hoc {@link FlightSqlProducer} for tests. */
public final class MockFlightSqlProducer implements FlightSqlProducer {
@@ -101,6 +102,7 @@ public final class MockFlightSqlProducer implements FlightSqlProducer {
private final SqlInfoBuilder sqlInfoBuilder = new SqlInfoBuilder();
private final Map parameterSchemas = new HashMap<>();
private final Map>> expectedParameterValues = new HashMap<>();
+ private final Map isUpdateMap = new HashMap<>();
private final Map actionTypeCounter = new HashMap<>();
@@ -176,6 +178,40 @@ public void addUpdateQuery(final String sqlCommand, final long updatedRows) {
});
}
+ /**
+ * Registers a new {@link StatementType#SELECT} SQL query, optionally setting the is_update field.
+ *
+ * @param sqlCommand the SQL command under which to register the new query.
+ * @param schema the schema to use for the query result.
+ * @param resultProviders the result provider for this query.
+ * @param isUpdate value to report for the is_update field, or {@code null} to leave it unset.
+ */
+ public void addSelectQuery(
+ final String sqlCommand,
+ final Schema schema,
+ final List> resultProviders,
+ final @Nullable Boolean isUpdate) {
+ addSelectQuery(sqlCommand, schema, resultProviders);
+ if (isUpdate != null) {
+ isUpdateMap.put(sqlCommand, isUpdate);
+ }
+ }
+
+ /**
+ * Registers a new {@link StatementType#UPDATE} SQL query, optionally setting the is_update field.
+ *
+ * @param sqlCommand the SQL command.
+ * @param updatedRows the number of rows affected.
+ * @param isUpdate value to report for the is_update field, or {@code null} to leave it unset.
+ */
+ public void addUpdateQuery(
+ final String sqlCommand, final long updatedRows, final @Nullable Boolean isUpdate) {
+ addUpdateQuery(sqlCommand, updatedRows);
+ if (isUpdate != null) {
+ isUpdateMap.put(sqlCommand, isUpdate);
+ }
+ }
+
/**
* Adds a catalog query to the results.
*
@@ -247,6 +283,12 @@ public void createPreparedStatement(
resultBuilder.setParameterSchema(ByteString.copyFrom(outputStream.toByteArray()));
}
+ // Set is_update field if present
+ final Boolean isUpdate = isUpdateMap.get(query);
+ if (isUpdate != null) {
+ resultBuilder.setIsUpdate(isUpdate);
+ }
+
listener.onNext(new Result(pack(resultBuilder.build()).toByteArray()));
} catch (final Throwable t) {
listener.onError(t);
diff --git a/flight/flight-sql-jdbc-driver/pom.xml b/flight/flight-sql-jdbc-driver/pom.xml
index 559c42597d..801089d090 100644
--- a/flight/flight-sql-jdbc-driver/pom.xml
+++ b/flight/flight-sql-jdbc-driver/pom.xml
@@ -22,7 +22,7 @@ under the License.
org.apache.arrow
arrow-flight
- 19.0.0-SNAPSHOT
+ 20.0.0-SNAPSHOT
flight-sql-jdbc-driver
@@ -138,7 +138,7 @@ under the License.
-
+
META-INF/LICENSE.txt
src/shade/LICENSE.txt
@@ -159,6 +159,7 @@ under the License.
org.apache.calcite.avatica:*
META-INF/services/java.sql.Driver
+ META-INF/README.txt
diff --git a/flight/flight-sql/pom.xml b/flight/flight-sql/pom.xml
index 56c47f64dd..b7c8931391 100644
--- a/flight/flight-sql/pom.xml
+++ b/flight/flight-sql/pom.xml
@@ -22,7 +22,7 @@ under the License.
org.apache.arrow
arrow-flight
- 19.0.0-SNAPSHOT
+ 20.0.0-SNAPSHOT
flight-sql
@@ -119,7 +119,7 @@ under the License.
commons-cli
commons-cli
- 1.9.0
+ 1.11.0
true
diff --git a/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlClient.java b/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlClient.java
index 623f9311e8..0af09faee1 100644
--- a/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlClient.java
+++ b/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlClient.java
@@ -1284,6 +1284,19 @@ public Schema getParameterSchema() {
return parameterSchema;
}
+ /**
+ * Returns whether the server indicated this prepared statement is an update query.
+ *
+ * @return true if the server indicated this is an update query, false if the server indicated
+ * this is a select query, or null if the server did not provide this information.
+ */
+ public Boolean isUpdate() {
+ if (preparedStatementResult.hasIsUpdate()) {
+ return preparedStatementResult.getIsUpdate();
+ }
+ return null;
+ }
+
/** Get the schema of the result set (should be identical to {@link #getResultSetSchema()}). */
public SchemaResult fetchSchema(CallOption... options) {
checkOpen();
diff --git a/flight/pom.xml b/flight/pom.xml
index 2fc3e89ef8..30f75fa27e 100644
--- a/flight/pom.xml
+++ b/flight/pom.xml
@@ -22,7 +22,7 @@ under the License.
org.apache.arrow
arrow-java-root
- 19.0.0-SNAPSHOT
+ 20.0.0-SNAPSHOT
arrow-flight
diff --git a/format/pom.xml b/format/pom.xml
index d3578b63d2..8c2f2d3387 100644
--- a/format/pom.xml
+++ b/format/pom.xml
@@ -23,7 +23,7 @@ under the License.
org.apache.arrow
arrow-java-root
- 19.0.0-SNAPSHOT
+ 20.0.0-SNAPSHOT
arrow-format
diff --git a/gandiva/pom.xml b/gandiva/pom.xml
index 5367bfdedf..190bf016ce 100644
--- a/gandiva/pom.xml
+++ b/gandiva/pom.xml
@@ -22,7 +22,7 @@ under the License.
org.apache.arrow
arrow-java-root
- 19.0.0-SNAPSHOT
+ 20.0.0-SNAPSHOT
org.apache.arrow.gandiva
diff --git a/gandiva/src/main/cpp/jni_common.cc b/gandiva/src/main/cpp/jni_common.cc
index 2851250072..ec4888a512 100644
--- a/gandiva/src/main/cpp/jni_common.cc
+++ b/gandiva/src/main/cpp/jni_common.cc
@@ -221,7 +221,7 @@ DataTypePtr ProtoTypeToDataType(const gandiva::types::ExtGandivaType& ext_type)
return arrow::date64();
case gandiva::types::DECIMAL:
// TODO: error handling
- return arrow::decimal(ext_type.precision(), ext_type.scale());
+ return arrow::decimal128(ext_type.precision(), ext_type.scale());
case gandiva::types::TIME32:
return ProtoTypeToTime32(ext_type);
case gandiva::types::TIME64:
diff --git a/memory/memory-core/pom.xml b/memory/memory-core/pom.xml
index 72ee69d60a..825b3dae4b 100644
--- a/memory/memory-core/pom.xml
+++ b/memory/memory-core/pom.xml
@@ -22,7 +22,7 @@ under the License.
org.apache.arrow
arrow-memory
- 19.0.0-SNAPSHOT
+ 20.0.0-SNAPSHOT
arrow-memory-core
@@ -100,8 +100,8 @@ under the License.
test
-
-
+
+
**/TestOpens.java
diff --git a/memory/memory-core/src/main/java/org/apache/arrow/memory/Accountant.java b/memory/memory-core/src/main/java/org/apache/arrow/memory/Accountant.java
index 5d052c2cde..d4d76f57f4 100644
--- a/memory/memory-core/src/main/java/org/apache/arrow/memory/Accountant.java
+++ b/memory/memory-core/src/main/java/org/apache/arrow/memory/Accountant.java
@@ -16,7 +16,7 @@
*/
package org.apache.arrow.memory;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicLongFieldUpdater;
import org.apache.arrow.util.Preconditions;
import org.checkerframework.checker.nullness.qual.Nullable;
@@ -37,16 +37,24 @@ class Accountant implements AutoCloseable {
*/
protected final long reservation;
- private final AtomicLong peakAllocation = new AtomicLong();
+ // AtomicLongFieldUpdaters for memory accounting fields to reduce memory overhead
+ private static final AtomicLongFieldUpdater PEAK_ALLOCATION_UPDATER =
+ AtomicLongFieldUpdater.newUpdater(Accountant.class, "peakAllocation");
+ private static final AtomicLongFieldUpdater ALLOCATION_LIMIT_UPDATER =
+ AtomicLongFieldUpdater.newUpdater(Accountant.class, "allocationLimit");
+ private static final AtomicLongFieldUpdater LOCALLY_HELD_MEMORY_UPDATER =
+ AtomicLongFieldUpdater.newUpdater(Accountant.class, "locallyHeldMemory");
+
+ private volatile long peakAllocation = 0;
/**
* Maximum local memory that can be held. This can be externally updated. Changing it won't cause
* past memory to change but will change responses to future allocation efforts
*/
- private final AtomicLong allocationLimit = new AtomicLong();
+ private volatile long allocationLimit = 0;
/** Currently allocated amount of memory. */
- private final AtomicLong locallyHeldMemory = new AtomicLong();
+ private volatile long locallyHeldMemory = 0;
public Accountant(
@Nullable Accountant parent, String name, long reservation, long maxAllocation) {
@@ -64,7 +72,7 @@ public Accountant(
this.parent = parent;
this.name = name;
this.reservation = reservation;
- this.allocationLimit.set(maxAllocation);
+ ALLOCATION_LIMIT_UPDATER.set(this, maxAllocation);
if (reservation != 0) {
Preconditions.checkArgument(parent != null, "parent must not be null");
@@ -117,12 +125,12 @@ private AllocationOutcome.Status allocateBytesInternal(long size) {
}
private void updatePeak() {
- final long currentMemory = locallyHeldMemory.get();
+ final long currentMemory = locallyHeldMemory;
while (true) {
- final long previousPeak = peakAllocation.get();
+ final long previousPeak = peakAllocation;
if (currentMemory > previousPeak) {
- if (!peakAllocation.compareAndSet(previousPeak, currentMemory)) {
+ if (!PEAK_ALLOCATION_UPDATER.compareAndSet(this, previousPeak, currentMemory)) {
// peak allocation changed underneath us. try again.
continue;
}
@@ -166,7 +174,7 @@ private AllocationOutcome.Status allocate(
final boolean incomingUpdatePeak,
final boolean forceAllocation,
@Nullable AllocationOutcomeDetails details) {
- final long oldLocal = locallyHeldMemory.getAndAdd(size);
+ final long oldLocal = LOCALLY_HELD_MEMORY_UPDATER.getAndAdd(this, size);
final long newLocal = oldLocal + size;
// Borrowed from Math.addExact (but avoid exception here)
// Overflow if result has opposite sign of both arguments
@@ -174,7 +182,7 @@ private AllocationOutcome.Status allocate(
// failure
final boolean overflow = ((oldLocal ^ newLocal) & (size ^ newLocal)) < 0;
final long beyondReservation = newLocal - reservation;
- final boolean beyondLimit = overflow || newLocal > allocationLimit.get();
+ final boolean beyondLimit = overflow || newLocal > allocationLimit;
final boolean updatePeak = forceAllocation || (incomingUpdatePeak && !beyondLimit);
if (details != null) {
@@ -214,7 +222,7 @@ private AllocationOutcome.Status allocate(
public void releaseBytes(long size) {
// reduce local memory. all memory released above reservation should be released up the tree.
- final long newSize = locallyHeldMemory.addAndGet(-size);
+ final long newSize = LOCALLY_HELD_MEMORY_UPDATER.addAndGet(this, -size);
Preconditions.checkArgument(newSize >= 0, "Accounted size went negative.");
@@ -255,7 +263,7 @@ public String getName() {
* @return Limit in bytes.
*/
public long getLimit() {
- return allocationLimit.get();
+ return allocationLimit;
}
/**
@@ -274,7 +282,7 @@ public long getInitReservation() {
* @param newLimit The limit in bytes.
*/
public void setLimit(long newLimit) {
- allocationLimit.set(newLimit);
+ ALLOCATION_LIMIT_UPDATER.set(this, newLimit);
}
/**
@@ -284,7 +292,7 @@ public void setLimit(long newLimit) {
* @return Currently allocate memory in bytes.
*/
public long getAllocatedMemory() {
- return locallyHeldMemory.get();
+ return locallyHeldMemory;
}
/**
@@ -293,17 +301,17 @@ public long getAllocatedMemory() {
* @return The peak allocated memory in bytes.
*/
public long getPeakMemoryAllocation() {
- return peakAllocation.get();
+ return peakAllocation;
}
public long getHeadroom() {
- long localHeadroom = allocationLimit.get() - locallyHeldMemory.get();
+ long localHeadroom = allocationLimit - locallyHeldMemory;
if (parent == null) {
return localHeadroom;
}
// Amount of reserved memory left on top of what parent has
- long reservedHeadroom = Math.max(0, reservation - locallyHeldMemory.get());
+ long reservedHeadroom = Math.max(0, reservation - locallyHeldMemory);
return Math.min(localHeadroom, parent.getHeadroom() + reservedHeadroom);
}
}
diff --git a/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java b/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java
index b8012fe643..9712be34d7 100644
--- a/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java
+++ b/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java
@@ -24,7 +24,6 @@
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.ReadOnlyBufferException;
-import java.util.concurrent.atomic.AtomicLong;
import org.apache.arrow.memory.BaseAllocator.Verbosity;
import org.apache.arrow.memory.util.CommonUtil;
import org.apache.arrow.memory.util.HistoricalLog;
@@ -57,9 +56,8 @@ public final class ArrowBuf implements AutoCloseable {
private static final int DOUBLE_SIZE = Double.BYTES;
private static final int LONG_SIZE = Long.BYTES;
- private static final AtomicLong idGenerator = new AtomicLong(0);
private static final int LOG_BYTES_PER_ROW = 10;
- private final long id = idGenerator.incrementAndGet();
+
private final ReferenceManager referenceManager;
private final @Nullable BufferManager bufferManager;
private final long addr;
@@ -67,7 +65,8 @@ public final class ArrowBuf implements AutoCloseable {
private long writerIndex;
private final @Nullable HistoricalLog historicalLog =
BaseAllocator.DEBUG
- ? new HistoricalLog(BaseAllocator.DEBUG_LOG_LENGTH, "ArrowBuf[%d]", id)
+ ? new HistoricalLog(
+ BaseAllocator.DEBUG_LOG_LENGTH, "ArrowBuf[%d]", System.identityHashCode(this))
: null;
private volatile long capacity;
@@ -218,7 +217,8 @@ public long memoryAddress() {
@Override
public String toString() {
- return String.format("ArrowBuf[%d], address:%d, capacity:%d", id, memoryAddress(), capacity);
+ return String.format(
+ "ArrowBuf[%d], address:%d, capacity:%d", getId(), memoryAddress(), capacity);
}
@Override
@@ -1080,12 +1080,15 @@ public String toHexString(final long start, final int length) {
}
/**
- * Get the integer id assigned to this ArrowBuf for debugging purposes.
+ * Get the id assigned to this ArrowBuf for debugging purposes.
+ *
+ * Returns {@link System#identityHashCode(Object)} which provides a unique identifier for this
+ * buffer without any per-instance memory overhead.
*
- * @return integer id
+ * @return the identity hash code for this buffer
*/
public long getId() {
- return id;
+ return System.identityHashCode(this);
}
/**
diff --git a/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferLedger.java b/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferLedger.java
index b562a421e7..eb90efcbb5 100644
--- a/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferLedger.java
+++ b/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferLedger.java
@@ -17,8 +17,7 @@
package org.apache.arrow.memory;
import java.util.IdentityHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
import org.apache.arrow.memory.util.CommonUtil;
import org.apache.arrow.memory.util.HistoricalLog;
import org.apache.arrow.util.Preconditions;
@@ -32,12 +31,13 @@
public class BufferLedger implements ValueWithKeyIncluded, ReferenceManager {
private final @Nullable IdentityHashMap buffers =
BaseAllocator.DEBUG ? new IdentityHashMap<>() : null;
- private static final AtomicLong LEDGER_ID_GENERATOR = new AtomicLong(0);
- // unique ID assigned to each ledger
- private final long ledgerId = LEDGER_ID_GENERATOR.incrementAndGet();
- private final AtomicInteger bufRefCnt = new AtomicInteger(0); // start at zero so we can
- // manage request for retain
- // correctly
+
+ // AtomicIntegerFieldUpdater for bufRefCnt to reduce memory overhead
+ private static final AtomicIntegerFieldUpdater BUF_REF_CNT_UPDATER =
+ AtomicIntegerFieldUpdater.newUpdater(BufferLedger.class, "bufRefCnt");
+ // start at zero so we can manage request for retain correctly
+ private volatile int bufRefCnt = 0;
+
private final long lCreationTime = System.nanoTime();
private final BufferAllocator allocator;
private final AllocationManager allocationManager;
@@ -78,7 +78,7 @@ public BufferAllocator getAllocator() {
*/
@Override
public int getRefCount() {
- return bufRefCnt.get();
+ return bufRefCnt;
}
/**
@@ -86,7 +86,7 @@ public int getRefCount() {
* ArrowBufs managed by this ledger will share the ref count.
*/
void increment() {
- bufRefCnt.incrementAndGet();
+ BUF_REF_CNT_UPDATER.incrementAndGet(this);
}
/**
@@ -144,7 +144,7 @@ private int decrement(int decrement) {
allocator.assertOpen();
final int outcome;
synchronized (allocationManager) {
- outcome = bufRefCnt.addAndGet(-decrement);
+ outcome = BUF_REF_CNT_UPDATER.addAndGet(this, -decrement);
if (outcome == 0) {
lDestructionTime = System.nanoTime();
// refcount of this reference manager has dropped to 0
@@ -174,7 +174,7 @@ public void retain(int increment) {
if (historicalLog != null) {
historicalLog.recordEvent("retain(%d)", increment);
}
- final int originalReferenceCount = bufRefCnt.getAndAdd(increment);
+ final int originalReferenceCount = BUF_REF_CNT_UPDATER.getAndAdd(this, increment);
Preconditions.checkArgument(originalReferenceCount > 0);
}
@@ -472,13 +472,13 @@ public long getAccountedSize() {
void print(StringBuilder sb, int indent, BaseAllocator.Verbosity verbosity) {
CommonUtil.indent(sb, indent)
.append("ledger[")
- .append(ledgerId)
+ .append(System.identityHashCode(this))
.append("] allocator: ")
.append(allocator.getName())
.append("), isOwning: ")
.append(", size: ")
.append(", references: ")
- .append(bufRefCnt.get())
+ .append(bufRefCnt)
.append(", life: ")
.append(lCreationTime)
.append("..")
diff --git a/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java b/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java
index 91bd7cd905..be0749a215 100644
--- a/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java
+++ b/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java
@@ -18,6 +18,7 @@
import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
+import java.lang.reflect.InaccessibleObjectException;
import java.lang.reflect.InvocationTargetException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
@@ -81,9 +82,18 @@ public Object run() {
BYTE_ARRAY_BASE_OFFSET = UNSAFE.arrayBaseOffset(byte[].class);
// get the offset of the address field in a java.nio.Buffer object
+ long maybeOffset;
Field addressField = java.nio.Buffer.class.getDeclaredField("address");
- addressField.setAccessible(true);
- BYTE_BUFFER_ADDRESS_OFFSET = UNSAFE.objectFieldOffset(addressField);
+ try {
+ addressField.setAccessible(true);
+ maybeOffset = UNSAFE.objectFieldOffset(addressField);
+ } catch (InaccessibleObjectException e) {
+ maybeOffset = -1;
+ logger.debug(
+ "Cannot access the address field of java.nio.Buffer. DirectBuffer operations wont be available",
+ e);
+ }
+ BYTE_BUFFER_ADDRESS_OFFSET = maybeOffset;
Constructor directBufferConstructor;
long address = -1;
@@ -109,6 +119,9 @@ public Object run() {
} catch (SecurityException e) {
logger.debug("Cannot get constructor for direct buffer allocation", e);
return e;
+ } catch (InaccessibleObjectException e) {
+ logger.debug("Cannot get constructor for direct buffer allocation", e);
+ return e;
}
}
});
@@ -156,7 +169,11 @@ public Object run() {
* @return address of the underlying memory.
*/
public static long getByteBufferAddress(ByteBuffer buf) {
- return UNSAFE.getLong(buf, BYTE_BUFFER_ADDRESS_OFFSET);
+ if (BYTE_BUFFER_ADDRESS_OFFSET != -1) {
+ return UNSAFE.getLong(buf, BYTE_BUFFER_ADDRESS_OFFSET);
+ }
+ throw new UnsupportedOperationException(
+ "Byte buffer address cannot be obtained because sun.misc.Unsafe or java.nio.DirectByteBuffer.(long, int) is not available");
}
private MemoryUtil() {}
diff --git a/memory/memory-core/src/test/java/org/apache/arrow/memory/TestOpens.java b/memory/memory-core/src/test/java/org/apache/arrow/memory/TestOpens.java
index b5e0a71e7e..f74bf63f82 100644
--- a/memory/memory-core/src/test/java/org/apache/arrow/memory/TestOpens.java
+++ b/memory/memory-core/src/test/java/org/apache/arrow/memory/TestOpens.java
@@ -20,32 +20,27 @@
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.condition.JRE.JAVA_16;
+import org.apache.arrow.memory.util.MemoryUtil;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledForJreRange;
public class TestOpens {
- /** Instantiating the RootAllocator should poke MemoryUtil and fail. */
+ /** Accessing MemoryUtil.directBuffer should fail as add-opens is not configured. */
@Test
@EnabledForJreRange(min = JAVA_16)
public void testMemoryUtilFailsLoudly() {
// This test is configured by Maven to run WITHOUT add-opens. So this should fail on JDK16+
// (where JEP396 means that add-opens is required to access JDK internals).
// The test will likely fail in your IDE if it doesn't correctly pick this up.
- Throwable e =
- assertThrows(
- Throwable.class,
- () -> {
- BufferAllocator allocator = new RootAllocator();
- allocator.close();
- });
+ Throwable e = assertThrows(Throwable.class, () -> MemoryUtil.directBuffer(0, 10));
boolean found = false;
while (e != null) {
- e = e.getCause();
- if (e instanceof RuntimeException
- && e.getMessage().contains("Failed to initialize MemoryUtil")) {
+ if (e instanceof UnsupportedOperationException
+ && e.getMessage().contains("java.nio.DirectByteBuffer.(long, int) not available")) {
found = true;
break;
}
+ e = e.getCause();
}
assertTrue(found, "Expected exception was not thrown");
}
diff --git a/memory/memory-netty-buffer-patch/pom.xml b/memory/memory-netty-buffer-patch/pom.xml
index 07dc7d2403..039b2aa04a 100644
--- a/memory/memory-netty-buffer-patch/pom.xml
+++ b/memory/memory-netty-buffer-patch/pom.xml
@@ -22,7 +22,7 @@ under the License.
org.apache.arrow
arrow-memory
- 19.0.0-SNAPSHOT
+ 20.0.0-SNAPSHOT
arrow-memory-netty-buffer-patch
diff --git a/memory/memory-netty/pom.xml b/memory/memory-netty/pom.xml
index 6d660da117..4218910980 100644
--- a/memory/memory-netty/pom.xml
+++ b/memory/memory-netty/pom.xml
@@ -22,7 +22,7 @@ under the License.
org.apache.arrow
arrow-memory
- 19.0.0-SNAPSHOT
+ 20.0.0-SNAPSHOT
arrow-memory-netty
diff --git a/memory/memory-unsafe/pom.xml b/memory/memory-unsafe/pom.xml
index 92dc0c9fe5..3fafb42802 100644
--- a/memory/memory-unsafe/pom.xml
+++ b/memory/memory-unsafe/pom.xml
@@ -22,7 +22,7 @@ under the License.
org.apache.arrow
arrow-memory
- 19.0.0-SNAPSHOT
+ 20.0.0-SNAPSHOT
arrow-memory-unsafe
diff --git a/memory/pom.xml b/memory/pom.xml
index bc34c26050..4ea3d1f9ca 100644
--- a/memory/pom.xml
+++ b/memory/pom.xml
@@ -22,7 +22,7 @@ under the License.
org.apache.arrow
arrow-java-root
- 19.0.0-SNAPSHOT
+ 20.0.0-SNAPSHOT
arrow-memory
pom
diff --git a/performance/pom.xml b/performance/pom.xml
index 3f18188e3a..d994bff45b 100644
--- a/performance/pom.xml
+++ b/performance/pom.xml
@@ -22,7 +22,7 @@ under the License.
org.apache.arrow
arrow-java-root
- 19.0.0-SNAPSHOT
+ 20.0.0-SNAPSHOT
arrow-performance
jar
@@ -35,10 +35,10 @@ under the License.
true
.*
1
-
+
5
5
-
+
jmh-result.json
json
@@ -75,7 +75,7 @@ under the License.
com.h2database
h2
- 2.3.232
+ 2.4.240
runtime
@@ -143,7 +143,7 @@ under the License.
java
-classpath
-
+
org.openjdk.jmh.Main
${benchmark.filter}
-f
diff --git a/performance/src/main/java/org/apache/arrow/memory/MemoryFootprintBenchmarks.java b/performance/src/main/java/org/apache/arrow/memory/MemoryFootprintBenchmarks.java
new file mode 100644
index 0000000000..395ba13b9d
--- /dev/null
+++ b/performance/src/main/java/org/apache/arrow/memory/MemoryFootprintBenchmarks.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.memory;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
+import java.lang.management.MemoryUsage;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+/**
+ * Benchmarks for memory footprint of Arrow memory objects.
+ *
+ * This benchmark measures the heap memory overhead of creating many ArrowBuf instances. The
+ * optimizations using AtomicFieldUpdater instead of AtomicLong/AtomicInteger objects should reduce
+ * memory overhead significantly.
+ *
+ *
Expected savings per instance: - ArrowBuf: 8 bytes (id field removed) - BufferLedger: 28 bytes
+ * (20 from AtomicInteger + 8 from ledgerId) - Accountant: 48 bytes (3 × 16 bytes from AtomicLong
+ * objects)
+ *
+ *
For 1M ArrowBuf instances, this should save approximately 8 MB of heap memory.
+ */
+@State(Scope.Benchmark)
+@Fork(
+ value = 1,
+ jvmArgs = {"-Xms2g", "-Xmx2g"})
+@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
+public class MemoryFootprintBenchmarks {
+
+ /** Number of ArrowBuf instances to create for memory footprint measurement. */
+ private static final int NUM_BUFFERS = 100_000;
+
+ /** Size in bytes of each buffer allocation. */
+ private static final int BUFFER_SIZE = 1024;
+
+ /** Root allocator used for all buffer allocations in the benchmark. */
+ private RootAllocator allocator;
+
+ /** Array to hold references to allocated buffers, preventing garbage collection. */
+ private ArrowBuf[] buffers;
+
+ /** JMX bean for querying heap memory usage statistics. */
+ private MemoryMXBean memoryBean;
+
+ /**
+ * Sets up the benchmark state before each trial.
+ *
+ *
Initializes the memory monitoring bean, creates a root allocator with sufficient capacity,
+ * and allocates the buffer reference array.
+ */
+ @Setup(Level.Trial)
+ public void setup() {
+ memoryBean = ManagementFactory.getMemoryMXBean();
+ allocator = new RootAllocator((long) NUM_BUFFERS * BUFFER_SIZE);
+ buffers = new ArrowBuf[NUM_BUFFERS];
+ }
+
+ /**
+ * Cleans up buffers after each benchmark invocation.
+ *
+ *
Closes all allocated buffers to prevent memory leaks and ensure each iteration starts with a
+ * clean slate. This is critical for the memory footprint benchmark which allocates many buffers
+ * that would otherwise accumulate across warmup and measurement iterations.
+ */
+ @TearDown(Level.Invocation)
+ public void tearDown() {
+ for (int i = 0; i < NUM_BUFFERS; i++) {
+ if (buffers[i] != null) {
+ buffers[i].close();
+ buffers[i] = null;
+ }
+ }
+ }
+
+ /**
+ * Cleans up the allocator after the trial completes.
+ *
+ *
Closes the root allocator to release all resources after all warmup and measurement
+ * iterations are complete.
+ */
+ @TearDown(Level.Trial)
+ public void tearDownTrial() {
+ allocator.close();
+ }
+
+ /**
+ * Benchmark that measures heap memory usage when creating many ArrowBuf instances.
+ *
+ *
This benchmark creates {@value #NUM_BUFFERS} ArrowBuf instances and measures the heap memory
+ * used. With the AtomicFieldUpdater optimizations, we expect to save approximately 800 KB of heap
+ * memory (8 bytes × 100,000 instances) just from removing the id field in ArrowBuf.
+ *
+ *
The benchmark performs garbage collection before and after allocation to ensure accurate
+ * measurement of heap memory delta. Results are printed to stdout for analysis.
+ *
+ * @return the total heap memory used by the allocated buffers in bytes
+ */
+ @Benchmark
+ @BenchmarkMode(Mode.SingleShotTime)
+ @OutputTimeUnit(TimeUnit.MILLISECONDS)
+ public long measureArrowBufMemoryFootprint() {
+ // Force GC before measurement
+ System.gc();
+ System.gc();
+ System.gc();
+
+ MemoryUsage heapBefore = memoryBean.getHeapMemoryUsage();
+ long usedBefore = heapBefore.getUsed();
+
+ // Allocate buffers
+ for (int i = 0; i < NUM_BUFFERS; i++) {
+ buffers[i] = allocator.buffer(BUFFER_SIZE);
+ }
+
+ // Force GC to get accurate measurement
+ System.gc();
+ System.gc();
+ System.gc();
+
+ MemoryUsage heapAfter = memoryBean.getHeapMemoryUsage();
+ long usedAfter = heapAfter.getUsed();
+
+ long memoryUsed = usedAfter - usedBefore;
+
+ // Print memory usage for analysis
+ System.out.printf(
+ "Created %d ArrowBuf instances. Heap memory used: %d bytes (%.2f MB)%n",
+ NUM_BUFFERS, memoryUsed, memoryUsed / (1024.0 * 1024.0));
+ System.out.printf(
+ "Average memory per ArrowBuf: %.2f bytes%n", (double) memoryUsed / NUM_BUFFERS);
+
+ return memoryUsed;
+ }
+
+ /**
+ * Benchmark that measures allocation and deallocation performance.
+ *
+ *
This complements the memory footprint benchmark by measuring the time it takes to allocate
+ * and deallocate 1,000 buffers in a tight loop. This helps identify any performance regressions
+ * introduced by memory optimizations.
+ *
+ *
Uses a local buffer array to avoid interference with the shared {@link #buffers} array used
+ * by other benchmarks.
+ */
+ @Benchmark
+ @BenchmarkMode(Mode.AverageTime)
+ @OutputTimeUnit(TimeUnit.MICROSECONDS)
+ public void measureAllocationPerformance() {
+ ArrowBuf[] localBuffers = new ArrowBuf[1000];
+
+ for (int i = 0; i < 1000; i++) {
+ localBuffers[i] = allocator.buffer(BUFFER_SIZE);
+ }
+
+ for (int i = 0; i < 1000; i++) {
+ localBuffers[i].close();
+ }
+ }
+
+ /**
+ * Main entry point for running the benchmarks standalone.
+ *
+ *
This allows running the benchmarks directly from the command line or IDE without using the
+ * Maven JMH plugin. Example usage:
+ *
+ *
{@code
+ * java -cp target/benchmarks.jar org.apache.arrow.memory.MemoryFootprintBenchmarks
+ * }
+ *
+ * @param args command line arguments (not used)
+ * @throws RunnerException if the benchmark runner encounters an error
+ */
+ public static void main(String[] args) throws RunnerException {
+ Options opt =
+ new OptionsBuilder()
+ .include(MemoryFootprintBenchmarks.class.getSimpleName())
+ .forks(1)
+ .build();
+
+ new Runner(opt).run();
+ }
+}
diff --git a/pom.xml b/pom.xml
index 1f57039015..78137eb4e9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -23,12 +23,12 @@ under the License.
org.apache
apache
- 35
+ 38
org.apache.arrow
arrow-java-root
- 19.0.0-SNAPSHOT
+ 20.0.0-SNAPSHOT
pom
Apache Arrow Java Root POM
@@ -92,37 +92,37 @@ under the License.
- 1695310533
+ 1773644827
${project.build.directory}/generated-sources
1.9.0
- 5.12.2
- 2.0.17
- 33.4.8-jre
- 4.2.9.Final
- 1.78.0
- 4.33.4
- 2.21.0
- 3.4.2
+ 6.1.1
+ 2.0.18
+ 33.6.0-jre
+ 4.2.15.Final
+ 1.82.1
+ 4.35.1
+ 2.22.0
+ 3.5.0
25.2.10
1.12.1
- 1.17.0
- 5.21.0
-
+ 1.17.1
+ 5.23.0
+
2
10.23.0
true
2.42.0
- 3.53.0
- 1.5.26
+ 4.2.1
+ 1.5.37
none
-Xdoclint:none
--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED
- 11
- 11
- 11
- 11
+ 17
+ 17
+ 17
+ 17
+ codegen
+ ${basedir}/src/main/codegen
+
+ **/*.tdd
+ **/*.fmpp
+ **/*.ftl
+
+
+
+
+
diff --git a/vector/src/main/codegen/templates/HolderReaderImpl.java b/vector/src/main/codegen/templates/HolderReaderImpl.java
index 1151ea5d39..cdbb65c4f6 100644
--- a/vector/src/main/codegen/templates/HolderReaderImpl.java
+++ b/vector/src/main/codegen/templates/HolderReaderImpl.java
@@ -126,7 +126,7 @@ public void read(Nullable${name}Holder h) {
<#elseif minor.class == "Duration">
return DurationVector.toDuration(holder.value, holder.unit);
<#elseif minor.class == "Bit" >
- return new Boolean(holder.value != 0);
+ return Boolean.valueOf(holder.value != 0);
<#elseif minor.class == "Decimal">
byte[] bytes = new byte[${type.width}];
holder.buffer.getBytes(holder.start, bytes, 0, ${type.width});
@@ -151,7 +151,7 @@ public void read(Nullable${name}Holder h) {
<#elseif minor.class == "TimeStampNano">
return DateUtility.getLocalDateTimeFromEpochNano(holder.value);
<#else>
- ${friendlyType} value = new ${friendlyType}(this.holder.value);
+ ${friendlyType} value = ${friendlyType}.valueOf(this.holder.value);
return value;
}
diff --git a/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java b/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java
index f6e3f63caf..484199ab2a 100644
--- a/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java
+++ b/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java
@@ -35,6 +35,10 @@
<#include "/@includes/vv_imports.ftl" />
+<#function is_timestamp_tz type>
+ <#return type?starts_with("TimeStamp") && type?ends_with("TZ")>
+
+
/*
* This class is generated using freemarker and the ${.template_name} template.
*/
@@ -96,55 +100,30 @@ public void close() throws Exception {
public void setPosition(int index) {
super.setPosition(index);
}
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
- <#assign fields = minor.fields!type.fields />
- <#assign uncappedName = name?uncap_first/>
- <#if uncappedName == "int" ><#assign uncappedName = "integer" />
- <#if !minor.typeParams?? >
+ <#list vv.types as type><#list type.minor as minor>
+ <#assign lowerName = minor.class?uncap_first />
+ <#if lowerName == "int" ><#assign lowerName = "integer" />
+ <#assign upperName = minor.class?upper_case />
@Override
- public ${name}Writer ${uncappedName}() {
+ public ${minor.class}Writer ${lowerName}() {
return this;
}
+ <#if minor.typeParams?? >
@Override
- public ${name}Writer ${uncappedName}(String name) {
- structName = name;
- return writer.${uncappedName}(name);
+ public ${minor.class}Writer ${lowerName}(String name<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}) {
+ return writer.${lowerName}(name<#list minor.typeParams as typeParam>, ${typeParam.name});
}
-
-
- @Override
- public DecimalWriter decimal() {
- return this;
- }
-
- @Override
- public DecimalWriter decimal(String name, int scale, int precision) {
- return writer.decimal(name, scale, precision);
- }
-
- @Override
- public DecimalWriter decimal(String name) {
- return writer.decimal(name);
- }
-
@Override
- public Decimal256Writer decimal256() {
- return this;
- }
-
- @Override
- public Decimal256Writer decimal256(String name, int scale, int precision) {
- return writer.decimal256(name, scale, precision);
+ public ${minor.class}Writer ${lowerName}(String name) {
+ structName = name;
+ return writer.${lowerName}(name);
}
- @Override
- public Decimal256Writer decimal256(String name) {
- return writer.decimal256(name);
- }
+
@Override
public StructWriter struct() {
@@ -215,87 +194,86 @@ public void end() {
}
@Override
- public void write(DecimalHolder holder) {
- if (writer.idx() >= (idx() + 1) * listSize) {
- throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize));
- }
- writer.write(holder);
- writer.setPosition(writer.idx() + 1);
- }
-
- @Override
- public void write(Decimal256Holder holder) {
+ public void writeNull() {
if (writer.idx() >= (idx() + 1) * listSize) {
throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize));
}
- writer.write(holder);
- writer.setPosition(writer.idx() + 1);
+ writer.writeNull();
}
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ <#assign name = minor.class?cap_first />
+ <#assign fields = minor.fields!type.fields />
+ <#assign uncappedName = name?uncap_first/>
@Override
- public void writeNull() {
+ public void write${name}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, ) {
if (writer.idx() >= (idx() + 1) * listSize) {
throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize));
}
- writer.writeNull();
+ writer.write${name}(<#list fields as field>${field.name}<#if field_has_next>, );
+ writer.setPosition(writer.idx()+1);
}
- public void writeDecimal(long start, ArrowBuf buffer, ArrowType arrowType) {
+ <#if is_timestamp_tz(minor.class) || minor.class == "Duration" || minor.class == "FixedSizeBinary">
+ @Override
+ public void write(${name}Holder holder) {
if (writer.idx() >= (idx() + 1) * listSize) {
throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize));
}
- writer.writeDecimal(start, buffer, arrowType);
- writer.setPosition(writer.idx() + 1);
+ writer.write(holder);
+ writer.setPosition(writer.idx()+1);
}
- public void writeDecimal(BigDecimal value) {
+ <#elseif minor.class?starts_with("Decimal")>
+ @Override
+ public void write${name}(long start, ArrowBuf buffer, ArrowType arrowType) {
if (writer.idx() >= (idx() + 1) * listSize) {
throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize));
}
- writer.writeDecimal(value);
- writer.setPosition(writer.idx() + 1);
+ writer.write${name}(start, buffer, arrowType);
+ writer.setPosition(writer.idx()+1);
}
- public void writeBigEndianBytesToDecimal(byte[] value, ArrowType arrowType) {
+ @Override
+ public void write(${name}Holder holder) {
if (writer.idx() >= (idx() + 1) * listSize) {
throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize));
}
- writer.writeBigEndianBytesToDecimal(value, arrowType);
- writer.setPosition(writer.idx() + 1);
+ writer.write(holder);
+ writer.setPosition(writer.idx()+1);
}
- public void writeDecimal256(long start, ArrowBuf buffer, ArrowType arrowType) {
+ @Override
+ public void write${name}(BigDecimal value) {
if (writer.idx() >= (idx() + 1) * listSize) {
throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize));
}
- writer.writeDecimal256(start, buffer, arrowType);
- writer.setPosition(writer.idx() + 1);
+ writer.write${name}(value);
+ writer.setPosition(writer.idx()+1);
}
- public void writeDecimal256(BigDecimal value) {
+ @Override
+ public void writeBigEndianBytesTo${name}(byte[] value, ArrowType arrowType){
if (writer.idx() >= (idx() + 1) * listSize) {
throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize));
}
- writer.writeDecimal256(value);
+ writer.writeBigEndianBytesTo${name}(value, arrowType);
writer.setPosition(writer.idx() + 1);
}
-
- public void writeBigEndianBytesToDecimal256(byte[] value, ArrowType arrowType) {
+ <#else>
+ @Override
+ public void write(${name}Holder holder) {
if (writer.idx() >= (idx() + 1) * listSize) {
throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize));
}
- writer.writeBigEndianBytesToDecimal256(value, arrowType);
- writer.setPosition(writer.idx() + 1);
+ writer.write${name}(<#list fields as field>holder.${field.name}<#if field_has_next>, );
+ writer.setPosition(writer.idx()+1);
}
+
-
- <#list vv.types as type>
- <#list type.minor as minor>
- <#assign name = minor.class?cap_first />
- <#assign fields = minor.fields!type.fields />
- <#assign uncappedName = name?uncap_first/>
- <#if minor.class?ends_with("VarBinary")>
+ <#if minor.class?ends_with("VarBinary")>
@Override
public void write${minor.class}(byte[] value) {
if (writer.idx() >= (idx() + 1) * listSize) {
@@ -349,27 +327,8 @@ public void writeBigEndianBytesToDecimal256(byte[] value, ArrowType arrowType) {
writer.write${minor.class}(value);
writer.setPosition(writer.idx() + 1);
}
-
-
- <#if !minor.typeParams?? >
- @Override
- public void write${name}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, ) {
- if (writer.idx() >= (idx() + 1) * listSize) {
- throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize));
- }
- writer.write${name}(<#list fields as field>${field.name}<#if field_has_next>, );
- writer.setPosition(writer.idx() + 1);
- }
-
- public void write(${name}Holder holder) {
- if (writer.idx() >= (idx() + 1) * listSize) {
- throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize));
- }
- writer.write${name}(<#list fields as field>holder.${field.name}<#if field_has_next>, );
- writer.setPosition(writer.idx() + 1);
- }
+
-
}
diff --git a/vector/src/main/codegen/templates/UnionListWriter.java b/vector/src/main/codegen/templates/UnionListWriter.java
index 4b54739230..394348f029 100644
--- a/vector/src/main/codegen/templates/UnionListWriter.java
+++ b/vector/src/main/codegen/templates/UnionListWriter.java
@@ -123,8 +123,6 @@ public void setPosition(int index) {
<#assign lowerName = minor.class?uncap_first />
<#if lowerName == "int" ><#assign lowerName = "integer" />
<#assign upperName = minor.class?upper_case />
- <#assign capName = minor.class?cap_first />
- <#assign vectName = capName />
@Override
public ${minor.class}Writer ${lowerName}() {
return this;
@@ -370,6 +368,7 @@ public void write(${name}Holder holder) {
}
<#elseif minor.class?starts_with("Decimal")>
+ @Override
public void write${name}(long start, ArrowBuf buffer, ArrowType arrowType) {
writer.write${name}(start, buffer, arrowType);
writer.setPosition(writer.idx()+1);
@@ -381,11 +380,13 @@ public void write(${name}Holder holder) {
writer.setPosition(writer.idx()+1);
}
+ @Override
public void write${name}(BigDecimal value) {
writer.write${name}(value);
writer.setPosition(writer.idx()+1);
}
+ @Override
public void writeBigEndianBytesTo${name}(byte[] value, ArrowType arrowType){
writer.writeBigEndianBytesTo${name}(value, arrowType);
writer.setPosition(writer.idx() + 1);
@@ -429,6 +430,7 @@ public void write(${name}Holder holder) {
writer.setPosition(writer.idx() + 1);
}
+ @Override
public void write${minor.class}(String value) {
writer.write${minor.class}(value);
writer.setPosition(writer.idx() + 1);
diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java
index 6c451f10a7..3fac195786 100644
--- a/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java
+++ b/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java
@@ -373,14 +373,26 @@ private void setReaderAndWriterIndex() {
valueBuffer.readerIndex(0);
if (valueCount == 0) {
validityBuffer.writerIndex(0);
- offsetBuffer.writerIndex(0);
valueBuffer.writerIndex(0);
} else {
final long lastDataOffset = getStartOffset(valueCount);
validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount));
- offsetBuffer.writerIndex((long) (valueCount + 1) * OFFSET_WIDTH);
valueBuffer.writerIndex(lastDataOffset);
}
+ // IPC serializer will determine readable bytes based on `readerIndex` and `writerIndex`.
+ // Both are set to 0 means 0 bytes are written to the IPC stream which will crash IPC readers
+ // in other libraries. According to Arrow spec, we should still output the offset buffer which
+ // is [0].
+ final long requiredOffsetBufferSize = (long) (valueCount + 1) * OFFSET_WIDTH;
+ if (offsetBuffer.capacity() < requiredOffsetBufferSize) {
+ ArrowBuf newOffsetBuffer = allocateOffsetBuffer(requiredOffsetBufferSize);
+ if (offsetBuffer.capacity() > 0) {
+ newOffsetBuffer.setBytes(0, offsetBuffer, 0, offsetBuffer.capacity());
+ }
+ offsetBuffer.getReferenceManager().release();
+ offsetBuffer = newOffsetBuffer;
+ }
+ offsetBuffer.writerIndex(requiredOffsetBufferSize);
}
/** Same as {@link #allocateNewSafe()}. */
diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
index 96e2afbd29..d5bd167256 100644
--- a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
+++ b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
@@ -389,14 +389,26 @@ private void setReaderAndWriterIndex() {
valueBuffer.readerIndex(0);
if (valueCount == 0) {
validityBuffer.writerIndex(0);
- offsetBuffer.writerIndex(0);
valueBuffer.writerIndex(0);
} else {
final int lastDataOffset = getStartOffset(valueCount);
validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount));
- offsetBuffer.writerIndex((long) (valueCount + 1) * OFFSET_WIDTH);
valueBuffer.writerIndex(lastDataOffset);
}
+ // IPC serializer will determine readable bytes based on `readerIndex` and `writerIndex`.
+ // Both are set to 0 means 0 bytes are written to the IPC stream which will crash IPC readers
+ // in other libraries. According to Arrow spec, we should still output the offset buffer which
+ // is [0].
+ final long requiredOffsetBufferSize = (long) (valueCount + 1) * OFFSET_WIDTH;
+ if (offsetBuffer.capacity() < requiredOffsetBufferSize) {
+ ArrowBuf newOffsetBuffer = allocateOffsetBuffer(requiredOffsetBufferSize);
+ if (offsetBuffer.capacity() > 0) {
+ newOffsetBuffer.setBytes(0, offsetBuffer, 0, offsetBuffer.capacity());
+ }
+ offsetBuffer.getReferenceManager().release();
+ offsetBuffer = newOffsetBuffer;
+ }
+ offsetBuffer.writerIndex(requiredOffsetBufferSize);
}
/** Same as {@link #allocateNewSafe()}. */
diff --git a/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java b/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java
index a7cb9ced72..4c1fbf761a 100644
--- a/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java
+++ b/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java
@@ -199,13 +199,18 @@ public FieldVector getVector(int index) {
*/
public VectorSchemaRoot addVector(int index, FieldVector vector) {
Preconditions.checkNotNull(vector);
- Preconditions.checkArgument(index >= 0 && index < fieldVectors.size());
+ Preconditions.checkArgument(index >= 0 && index <= fieldVectors.size());
List