From c29fcfc72bed84c1dc9926bc28294b5ff193937f Mon Sep 17 00:00:00 2001 From: Vishal Boddu <9461605+bodduv@users.noreply.github.com> Date: Thu, 3 Apr 2025 17:25:44 +0200 Subject: [PATCH 001/271] GH-692: Preserve nullability information while transfering DecimalVector and Decimal256Vector (#693) ## What's Changed This PR proposes to use the "from" `ValueVector`'s field while transferring `DecimalVector` and `Decimal256Vector` to preserve nullability information. This is to have similar behavior with all the other primitive value vectors. Note that the `FieldType` of the value vector has nullability information. Closes #692 . --- .../apache/arrow/vector/Decimal256Vector.java | 7 +++-- .../apache/arrow/vector/DecimalVector.java | 5 +++- .../arrow/vector/TestDecimal256Vector.java | 28 +++++++++++++++++++ .../arrow/vector/TestDecimalVector.java | 26 +++++++++++++++++ 4 files changed, 63 insertions(+), 3 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java b/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java index 90f67798dd..f9d7e5cb9e 100644 --- a/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java +++ b/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java @@ -567,8 +567,11 @@ private class TransferImpl implements TransferPair { public TransferImpl(String ref, BufferAllocator allocator) { to = - new Decimal256Vector( - ref, allocator, Decimal256Vector.this.precision, Decimal256Vector.this.scale); + (Decimal256Vector.this.field != null + && Decimal256Vector.this.field.getFieldType() != null) + ? new Decimal256Vector(ref, Decimal256Vector.this.field.getFieldType(), allocator) + : new Decimal256Vector( + ref, allocator, Decimal256Vector.this.precision, Decimal256Vector.this.scale); } public TransferImpl(Field field, BufferAllocator allocator) { diff --git a/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java b/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java index b4c55680b7..9bf1812cc6 100644 --- a/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java @@ -565,7 +565,10 @@ private class TransferImpl implements TransferPair { public TransferImpl(String ref, BufferAllocator allocator) { to = - new DecimalVector(ref, allocator, DecimalVector.this.precision, DecimalVector.this.scale); + (DecimalVector.this.field != null && DecimalVector.this.field.getFieldType() != null) + ? new DecimalVector(ref, DecimalVector.this.field.getFieldType(), allocator) + : new DecimalVector( + ref, allocator, DecimalVector.this.precision, DecimalVector.this.scale); } public TransferImpl(Field field, BufferAllocator allocator) { diff --git a/vector/src/test/java/org/apache/arrow/vector/TestDecimal256Vector.java b/vector/src/test/java/org/apache/arrow/vector/TestDecimal256Vector.java index c155ab98fa..b995dc5d92 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestDecimal256Vector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestDecimal256Vector.java @@ -26,6 +26,7 @@ import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.TransferPair; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -375,6 +376,33 @@ public void testGetTransferPairWithField() { assertSame(fromVector.getField(), toVector.getField()); } + @Test + public void testGetTransferPairWithoutField() { + final Decimal256Vector fromVector = new Decimal256Vector("decimal", allocator, 10, scale); + final TransferPair transferPair = + fromVector.getTransferPair(fromVector.getField().getName(), allocator); + final Decimal256Vector toVector = (Decimal256Vector) transferPair.getTo(); + // A new Field created inside a new vector should reuse the field type (should be the same in + // memory as the original Field's field type). + assertSame(fromVector.getField().getFieldType(), toVector.getField().getFieldType()); + } + + @Test + public void testGetTransferPairWithoutFieldNonNullable() { + final FieldType decimal256NonNullableType = + new FieldType( + false, new ArrowType.Decimal(10, scale, Decimal256Vector.TYPE_WIDTH * 8), null); + final Decimal256Vector fromVector = + new Decimal256Vector("decimal", decimal256NonNullableType, allocator); + final TransferPair transferPair = + fromVector.getTransferPair(fromVector.getField().getName(), allocator); + final Decimal256Vector toVector = (Decimal256Vector) transferPair.getTo(); + // A new Field created inside a new vector should reuse the field type (should be the same in + // memory as the original Field's field type). + assertSame(fromVector.getField().getFieldType(), toVector.getField().getFieldType()); + assertSame(decimal256NonNullableType, toVector.getField().getFieldType()); + } + private void verifyWritingArrowBufWithBigEndianBytes( Decimal256Vector decimalVector, ArrowBuf buf, BigDecimal[] expectedValues, int length) { decimalVector.allocateNew(); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java b/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java index d5310bad0e..85c11e8f3d 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java @@ -26,6 +26,7 @@ import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.TransferPair; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -371,6 +372,31 @@ public void testGetTransferPairWithField() { assertSame(fromVector.getField(), toVector.getField()); } + @Test + public void testGetTransferPairWithoutField() { + final DecimalVector fromVector = new DecimalVector("decimal", allocator, 10, scale); + final TransferPair transferPair = + fromVector.getTransferPair(fromVector.getField().getName(), allocator); + final DecimalVector toVector = (DecimalVector) transferPair.getTo(); + // A new Field created inside a new vector should reuse the field type (should be the same in + // memory as the original Field's field type). + assertSame(fromVector.getField().getFieldType(), toVector.getField().getFieldType()); + } + + @Test + public void testGetTransferPairWithoutFieldNonNullable() { + final FieldType decimalNonNullableType = + new FieldType(false, new ArrowType.Decimal(10, scale), null); + final DecimalVector fromVector = + new DecimalVector("decimal", decimalNonNullableType, allocator); + final TransferPair transferPair = + fromVector.getTransferPair(fromVector.getField().getName(), allocator); + final DecimalVector toVector = (DecimalVector) transferPair.getTo(); + // A new Field created inside a new vector should reuse the field type (should be the same in + // memory as the original Field's field type). + assertSame(fromVector.getField().getFieldType(), toVector.getField().getFieldType()); + } + private void verifyWritingArrowBufWithBigEndianBytes( DecimalVector decimalVector, ArrowBuf buf, BigDecimal[] expectedValues, int length) { decimalVector.allocateNew(); From 8ffc1d3eb58bbe4fc023d2222cc4439dace7e8ea Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 8 Apr 2025 12:17:11 +0900 Subject: [PATCH 002/271] MINOR: Fix JNI code after upstream DCHECK change (#706) ## What's Changed Upstream renamed the public DCHECK macros to ARROW_DCHECK (https://github.com/apache/arrow/pull/46015) --- .github/workflows/rc.yml | 5 +++++ dataset/src/main/cpp/jni_util.cc | 4 ++-- gandiva/src/main/cpp/expression_registry_helper.cc | 2 +- gandiva/src/main/cpp/jni_common.cc | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 5b78cc9395..e039306ec3 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -261,6 +261,11 @@ jobs: # bundled Protobuf. brew uninstall protobuf + # We need Flatbuffers 24, not the latest version + # Homebrew does not offer older versions, so remove the Homebrew + # package and rely on Arrow using a bundled version instead + brew uninstall flatbuffers + brew bundle --file=Brewfile - name: Prepare ccache run: | diff --git a/dataset/src/main/cpp/jni_util.cc b/dataset/src/main/cpp/jni_util.cc index 1fd15696e6..35bfb328f0 100644 --- a/dataset/src/main/cpp/jni_util.cc +++ b/dataset/src/main/cpp/jni_util.cc @@ -187,7 +187,7 @@ ReservationListenableMemoryPool::~ReservationListenableMemoryPool() {} std::string Describe(JNIEnv* env, jthrowable t) { jclass describer_class = env->FindClass("org/apache/arrow/dataset/jni/JniExceptionDescriber"); - DCHECK_NE(describer_class, nullptr); + ARROW_DCHECK_NE(describer_class, nullptr); jmethodID describe_method = env->GetStaticMethodID( describer_class, "describe", "(Ljava/lang/Throwable;)Ljava/lang/String;"); std::string description = JStringToCString( @@ -197,7 +197,7 @@ std::string Describe(JNIEnv* env, jthrowable t) { bool IsErrorInstanceOf(JNIEnv* env, jthrowable t, std::string class_name) { jclass java_class = env->FindClass(class_name.c_str()); - DCHECK_NE(java_class, nullptr) << "Could not find Java class " << class_name; + ARROW_DCHECK_NE(java_class, nullptr) << "Could not find Java class " << class_name; return env->IsInstanceOf(t, java_class); } diff --git a/gandiva/src/main/cpp/expression_registry_helper.cc b/gandiva/src/main/cpp/expression_registry_helper.cc index 66b97c8b9e..21077ff1db 100644 --- a/gandiva/src/main/cpp/expression_registry_helper.cc +++ b/gandiva/src/main/cpp/expression_registry_helper.cc @@ -138,7 +138,7 @@ void ArrowToProtobuf(DataTypePtr type, gandiva::types::ExtGandivaType* gandiva_d default: // un-supported types. test ensures that // when one of these are added build breaks. - DCHECK(false); + ARROW_DCHECK(false); } } diff --git a/gandiva/src/main/cpp/jni_common.cc b/gandiva/src/main/cpp/jni_common.cc index ec1bb76234..2851250072 100644 --- a/gandiva/src/main/cpp/jni_common.cc +++ b/gandiva/src/main/cpp/jni_common.cc @@ -751,7 +751,7 @@ Status JavaResizableBuffer::Resize(const int64_t new_size, bool shrink_to_fit) { } RETURN_NOT_OK(Reserve(new_size)); - DCHECK_GE(capacity_, new_size); + ARROW_DCHECK_GE(capacity_, new_size); size_ = new_size; return Status::OK(); } From 7bd2b0d3853b6ef1066aa2e9819743f441082dd8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Apr 2025 01:21:37 -0400 Subject: [PATCH 003/271] MINOR: Bump checker.framework.version from 3.49.1 to 3.49.2 (#703) Bumps `checker.framework.version` from 3.49.1 to 3.49.2. Updates `org.checkerframework:checker-qual` from 3.49.1 to 3.49.2 Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index df32105804..5462d1416e 100644 --- a/pom.xml +++ b/pom.xml @@ -109,7 +109,7 @@ under the License. 10.22.0 true 2.37.0 - 3.49.1 + 3.49.2 1.5.18 none -Xdoclint:none From 8d55fd1b52863bc0d2606c623d34e3eaa3be7b3d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Apr 2025 01:26:19 -0400 Subject: [PATCH 004/271] MINOR: Bump org.mockito:mockito-bom from 5.16.1 to 5.17.0 (#701) Bumps [org.mockito:mockito-bom](https://github.com/mockito/mockito) from 5.16.1 to 5.17.0. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5462d1416e..9e6c8b782e 100644 --- a/pom.xml +++ b/pom.xml @@ -103,7 +103,7 @@ under the License. 3.4.1 25.2.10 1.12.0 - 5.16.1 + 5.17.0 2 10.22.0 From 42a126fd88a1430b474fb4571b0c5b99c83ee770 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Apr 2025 01:47:44 -0400 Subject: [PATCH 005/271] MINOR: Bump com.puppycrawl.tools:checkstyle from 10.22.0 to 10.23.0 (#702) Bumps [com.puppycrawl.tools:checkstyle](https://github.com/checkstyle/checkstyle) from 10.22.0 to 10.23.0. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9e6c8b782e..cb189dc8f9 100644 --- a/pom.xml +++ b/pom.xml @@ -106,7 +106,7 @@ under the License. 5.17.0 2 - 10.22.0 + 10.23.0 true 2.37.0 3.49.2 From f92585c272ea4b2db5b1717f6642f3edc8f99903 Mon Sep 17 00:00:00 2001 From: Kristin Cowalcijk Date: Tue, 8 Apr 2025 14:11:32 +0800 Subject: [PATCH 006/271] GH-704: Fix initialization of offset buffer when exporting VarChar vectors through C Data Interface (#705) ## What's Changed This patch fixes the initialization of offset buffers when exporting variable width arrays through Arrow C Data Interface. The original code incorrectly mess up with the member `this.offsetBuffer` while we should actually initialize the newly allocated offsetBuffer. I think the diff itself will be quite self-explanatory. Closes #704 and probably #88 . --- .../org/apache/arrow/c/RoundtripTest.java | 23 +++++++++++++++++++ .../vector/BaseLargeVariableWidthVector.java | 2 +- .../arrow/vector/BaseVariableWidthVector.java | 2 +- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/c/src/test/java/org/apache/arrow/c/RoundtripTest.java b/c/src/test/java/org/apache/arrow/c/RoundtripTest.java index 67ab282de5..6d68449c0b 100644 --- a/c/src/test/java/org/apache/arrow/c/RoundtripTest.java +++ b/c/src/test/java/org/apache/arrow/c/RoundtripTest.java @@ -38,6 +38,8 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.memory.util.hash.ArrowBufHasher; +import org.apache.arrow.vector.BaseLargeVariableWidthVector; +import org.apache.arrow.vector.BaseVariableWidthVector; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.DateDayVector; @@ -181,6 +183,13 @@ boolean roundtrip(FieldVector vector, Class clazz) { clazz.isInstance(imported), String.format("expected %s but was %s", clazz, imported.getClass())); result = VectorEqualsVisitor.vectorEquals(vector, imported); + + if (imported instanceof BaseVariableWidthVector + || imported instanceof BaseLargeVariableWidthVector) { + ArrowBuf offsetBuffer = imported.getOffsetBuffer(); + assertTrue(offsetBuffer.capacity() > 0); + assertEquals(0, offsetBuffer.getInt(0)); + } } // Check that the ref counts of the buffers are the same after the roundtrip @@ -602,6 +611,13 @@ public void testVarCharVector() { } } + @Test + public void testEmptyVarCharVector() { + try (final VarCharVector vector = new VarCharVector("v", allocator)) { + assertTrue(roundtrip(vector, VarCharVector.class)); + } + } + @Test public void testLargeVarBinaryVector() { try (final LargeVarBinaryVector vector = new LargeVarBinaryVector("", allocator)) { @@ -635,6 +651,13 @@ public void testLargeVarCharVector() { } } + @Test + public void testEmptyLargeVarCharVector() { + try (final LargeVarCharVector vector = new LargeVarCharVector("v", allocator)) { + assertTrue(roundtrip(vector, LargeVarCharVector.class)); + } + } + @Test public void testListVector() { try (final ListVector vector = ListVector.empty("v", allocator)) { diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java index 552a896ea8..7e0d0affc6 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java @@ -496,7 +496,7 @@ private void allocateBytes(final long valueBufferSize, final int valueCount) { private ArrowBuf allocateOffsetBuffer(final long size) { ArrowBuf offsetBuffer = allocator.buffer(size); offsetBuffer.readerIndex(0); - initOffsetBuffer(); + offsetBuffer.setZero(0, offsetBuffer.capacity()); return offsetBuffer; } diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java index aaccec602f..7b8d2cdfda 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java @@ -514,7 +514,7 @@ private ArrowBuf allocateOffsetBuffer(final long size) { final int curSize = (int) size; ArrowBuf offsetBuffer = allocator.buffer(curSize); offsetBuffer.readerIndex(0); - initOffsetBuffer(); + offsetBuffer.setZero(0, offsetBuffer.capacity()); return offsetBuffer; } From 222f30e75d079ae5005cfc5764f4c905eaa2ca5c Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 8 Apr 2025 15:58:08 +0900 Subject: [PATCH 007/271] GH-494: [Flight] Allow configuring connect timeout in JDBC (#495) Allow configuring the connect timeout via `connectTimeoutMs` so that the driver doesn't wait so long before giving up on unreachable locations. Fixes #494. --- flight/flight-sql-jdbc-core/pom.xml | 15 ++ .../driver/jdbc/ArrowFlightConnection.java | 1 + .../client/ArrowFlightSqlClientHandler.java | 52 +++++-- .../ArrowFlightConnectionConfigImpl.java | 15 +- .../arrow/driver/jdbc/ResultSetTest.java | 140 +++++++++++++++++- ...rrowFlightSqlClientHandlerBuilderTest.java | 1 + .../ArrowFlightConnectionConfigImplTest.java | 32 +++- .../jdbc/utils/FallbackFlightSqlProducer.java | 10 ++ 8 files changed, 240 insertions(+), 26 deletions(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 15368e918d..3362b60a8e 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -47,6 +47,21 @@ under the License. + + io.grpc + grpc-api + + + + io.grpc + grpc-netty + + + + io.netty + netty-transport + + org.apache.arrow arrow-memory-core diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java index c1b1c8f8e6..cf9804d68b 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java @@ -113,6 +113,7 @@ private static ArrowFlightSqlClientHandler createNewClientHandler( .withRetainCookies(config.retainCookies()) .withRetainAuth(config.retainAuth()) .withCatalog(config.getCatalog()) + .withConnectTimeout(config.getConnectTimeout()) .build(); } catch (final SQLException e) { try { diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java index 0e9c79a090..cbbe223eb8 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java @@ -17,10 +17,13 @@ package org.apache.arrow.driver.jdbc.client; import com.google.common.collect.ImmutableMap; +import io.grpc.netty.NettyChannelBuilder; +import io.netty.channel.ChannelOption; import java.io.IOException; import java.net.URI; import java.security.GeneralSecurityException; import java.sql.SQLException; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -36,6 +39,7 @@ import org.apache.arrow.flight.FlightClient; import org.apache.arrow.flight.FlightClientMiddleware; import org.apache.arrow.flight.FlightEndpoint; +import org.apache.arrow.flight.FlightGrpcUtils; import org.apache.arrow.flight.FlightInfo; import org.apache.arrow.flight.FlightRuntimeException; import org.apache.arrow.flight.FlightStatusCode; @@ -50,6 +54,7 @@ import org.apache.arrow.flight.auth2.ClientIncomingAuthHeaderMiddleware; import org.apache.arrow.flight.client.ClientCookieMiddleware; import org.apache.arrow.flight.grpc.CredentialCallOption; +import org.apache.arrow.flight.grpc.NettyClientBuilder; import org.apache.arrow.flight.sql.FlightSqlClient; import org.apache.arrow.flight.sql.impl.FlightSql.SqlInfo; import org.apache.arrow.flight.sql.util.TableRef; @@ -138,12 +143,11 @@ public List getStreams(final FlightInfo flightInfo) // Clone the builder and then set the new endpoint on it. // GH-38574: Currently a new FlightClient will be made for each partition that returns a - // non-empty Location - // then disposed of. It may be better to cache clients because a server may report the - // same Locations. - // It would also be good to identify when the reported location is the same as the - // original connection's - // Location and skip creating a FlightClient in that scenario. + // non-empty Location then disposed of. It may be better to cache clients because a server + // may report the same Locations. It would also be good to identify when the reported + // location + // is the same as the original connection's Location and skip creating a FlightClient in + // that scenario. List exceptions = new ArrayList<>(); CloseableEndpointStreamPair stream = null; for (Location location : endpoint.getLocations()) { @@ -158,7 +162,8 @@ public List getStreams(final FlightInfo flightInfo) new Builder(ArrowFlightSqlClientHandler.this.builder) .withHost(endpointUri.getHost()) .withPort(endpointUri.getPort()) - .withEncryption(endpointUri.getScheme().equals(LocationSchemes.GRPC_TLS)); + .withEncryption(endpointUri.getScheme().equals(LocationSchemes.GRPC_TLS)) + .withConnectTimeout(builder.connectTimeout); ArrowFlightSqlClientHandler endpointHandler = null; try { @@ -177,6 +182,7 @@ public List getStreams(final FlightInfo flightInfo) exceptions.add(ex); continue; } + break; } if (stream != null) { @@ -543,6 +549,8 @@ public static final class Builder { @VisibleForTesting Optional catalog = Optional.empty(); + @VisibleForTesting @Nullable Duration connectTimeout; + // These two middleware are for internal use within build() and should not be exposed by builder // APIs. // Note that these middleware may not necessarily be registered. @@ -825,6 +833,19 @@ public Builder withCatalog(@Nullable final String catalog) { return this; } + public Builder withConnectTimeout(Duration connectTimeout) { + this.connectTimeout = connectTimeout; + return this; + } + + /** Get the location that this client will connect to. */ + public Location getLocation() { + if (useEncryption) { + return Location.forGrpcTls(host, port); + } + return Location.forGrpcInsecure(host, port); + } + /** * Builds a new {@link ArrowFlightSqlClientHandler} from the provided fields. * @@ -845,17 +866,15 @@ public ArrowFlightSqlClientHandler build() throws SQLException { if (isUsingUserPasswordAuth) { buildTimeMiddlewareFactories.add(authFactory); } - final FlightClient.Builder clientBuilder = FlightClient.builder().allocator(allocator); + final NettyClientBuilder clientBuilder = new NettyClientBuilder(); + clientBuilder.allocator(allocator); buildTimeMiddlewareFactories.add(new ClientCookieMiddleware.Factory()); buildTimeMiddlewareFactories.forEach(clientBuilder::intercept); - Location location; if (useEncryption) { - location = Location.forGrpcTls(host, port); clientBuilder.useTls(); - } else { - location = Location.forGrpcInsecure(host, port); } + Location location = getLocation(); clientBuilder.location(location); if (useEncryption) { @@ -883,7 +902,14 @@ public ArrowFlightSqlClientHandler build() throws SQLException { } } - client = clientBuilder.build(); + NettyChannelBuilder channelBuilder = clientBuilder.build(); + if (connectTimeout != null) { + channelBuilder.withOption( + ChannelOption.CONNECT_TIMEOUT_MILLIS, (int) connectTimeout.toMillis()); + } + client = + FlightGrpcUtils.createFlightClient( + allocator, channelBuilder.build(), clientBuilder.middleware()); final ArrayList credentialOptions = new ArrayList<>(); if (isUsingUserPasswordAuth) { // If the authFactory has already been used for a handshake, use the existing token. diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java index e8bae2a207..ab6a5898b7 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.driver.jdbc.utils; +import java.time.Duration; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -163,6 +164,16 @@ public String getCatalog() { return ArrowFlightConnectionProperty.CATALOG.getString(properties); } + /** The initial connect timeout. */ + public Duration getConnectTimeout() { + Integer timeout = ArrowFlightConnectionProperty.CONNECT_TIMEOUT_MILLIS.getInteger(properties); + if (timeout == null) { + return Duration.ofMillis( + (int) ArrowFlightConnectionProperty.CONNECT_TIMEOUT_MILLIS.defaultValue()); + } + return Duration.ofMillis(timeout); + } + /** * Gets the {@link CallOption}s from this {@link ConnectionConfig}. * @@ -213,7 +224,9 @@ public enum ArrowFlightConnectionProperty implements ConnectionProperty { TOKEN("token", null, Type.STRING, false), RETAIN_COOKIES("retainCookies", true, Type.BOOLEAN, false), RETAIN_AUTH("retainAuth", true, Type.BOOLEAN, false), - CATALOG("catalog", null, Type.STRING, false); + CATALOG("catalog", null, Type.STRING, false), + CONNECT_TIMEOUT_MILLIS("connectTimeoutMs", 10000, Type.NUMBER, false), + ; private final String camelName; private final Object defaultValue; diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java index a8d04dfc83..cd47408f52 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java @@ -25,12 +25,7 @@ import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assertions.*; import com.google.common.collect.ImmutableSet; import java.nio.charset.StandardCharsets; @@ -645,6 +640,139 @@ public void testFallbackSecondFlightServer() throws Exception { } } + @Test + public void testFallbackUnresolvableFlightServer() throws Exception { + final Schema schema = + new Schema( + Collections.singletonList(Field.nullable("int_column", Types.MinorType.INT.getType()))); + try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + VectorSchemaRoot resultData = VectorSchemaRoot.create(schema, allocator)) { + resultData.setRowCount(1); + ((IntVector) resultData.getVector(0)).set(0, 1); + + try (final FallbackFlightSqlProducer rootProducer = + new FallbackFlightSqlProducer(resultData); + FlightServer rootServer = + FlightServer.builder(allocator, forGrpcInsecure("localhost", 0), rootProducer) + .build() + .start(); + Connection newConnection = + DriverManager.getConnection( + String.format( + "jdbc:arrow-flight-sql://%s:%d/?useEncryption=false", + rootServer.getLocation().getUri().getHost(), rootServer.getPort()))) { + // This first attempt should take a measurable amount of time. + long start = System.nanoTime(); + try (Statement newStatement = newConnection.createStatement()) { + try (ResultSet result = newStatement.executeQuery("fallback with unresolvable")) { + List actualData = new ArrayList<>(); + while (result.next()) { + actualData.add(result.getInt(1)); + } + + // Assert + assertEquals(resultData.getRowCount(), actualData.size()); + assertTrue(actualData.contains(((IntVector) resultData.getVector(0)).get(0))); + } + } + long attempt1 = System.nanoTime(); + double elapsedMs = (attempt1 - start) / 1_000_000.; + assertTrue( + elapsedMs >= 5000., + String.format( + "Expected first attempt to hit the timeout, but only %f ms elapsed", elapsedMs)); + + // Once the client cache is implemented (GH-661), this second attempt should take less time, + // since the failure from before should be cached. + start = System.nanoTime(); + try (Statement newStatement = newConnection.createStatement()) { + try (ResultSet result = newStatement.executeQuery("fallback with unresolvable")) { + List actualData = new ArrayList<>(); + while (result.next()) { + actualData.add(result.getInt(1)); + } + + // Assert + assertEquals(resultData.getRowCount(), actualData.size()); + assertTrue(actualData.contains(((IntVector) resultData.getVector(0)).get(0))); + } + } + attempt1 = System.nanoTime(); + elapsedMs = (attempt1 - start) / 1_000_000.; + // TODO(GH-661): this assertion should be flipped to assertTrue. + assertFalse( + elapsedMs < 5000., + String.format("Expected second attempt to be the same, but %f ms elapsed", elapsedMs)); + } + } + } + + @Test + public void testFallbackUnresolvableFlightServerDisableCache() throws Exception { + final Schema schema = + new Schema( + Collections.singletonList(Field.nullable("int_column", Types.MinorType.INT.getType()))); + try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + VectorSchemaRoot resultData = VectorSchemaRoot.create(schema, allocator)) { + resultData.setRowCount(1); + ((IntVector) resultData.getVector(0)).set(0, 1); + + try (final FallbackFlightSqlProducer rootProducer = + new FallbackFlightSqlProducer(resultData); + FlightServer rootServer = + FlightServer.builder(allocator, forGrpcInsecure("localhost", 0), rootProducer) + .build() + .start(); + Connection newConnection = + DriverManager.getConnection( + String.format( + "jdbc:arrow-flight-sql://%s:%d/?useEncryption=false&useClientCache=false", + rootServer.getLocation().getUri().getHost(), rootServer.getPort()))) { + // This first attempt should take a measurable amount of time. + long start = System.nanoTime(); + try (Statement newStatement = newConnection.createStatement()) { + try (ResultSet result = newStatement.executeQuery("fallback with unresolvable")) { + List actualData = new ArrayList<>(); + while (result.next()) { + actualData.add(result.getInt(1)); + } + + // Assert + assertEquals(resultData.getRowCount(), actualData.size()); + assertTrue(actualData.contains(((IntVector) resultData.getVector(0)).get(0))); + } + } + long attempt1 = System.nanoTime(); + double elapsedMs = (attempt1 - start) / 1_000_000.; + assertTrue( + elapsedMs >= 5000., + String.format( + "Expected first attempt to hit the timeout, but only %f ms elapsed", elapsedMs)); + + // This second attempt should take a long time still, since we disabled the cache. + start = System.nanoTime(); + try (Statement newStatement = newConnection.createStatement()) { + try (ResultSet result = newStatement.executeQuery("fallback with unresolvable")) { + List actualData = new ArrayList<>(); + while (result.next()) { + actualData.add(result.getInt(1)); + } + + // Assert + assertEquals(resultData.getRowCount(), actualData.size()); + assertTrue(actualData.contains(((IntVector) resultData.getVector(0)).get(0))); + } + } + attempt1 = System.nanoTime(); + elapsedMs = (attempt1 - start) / 1_000_000.; + assertTrue( + elapsedMs >= 5000., + String.format( + "Expected second attempt to hit the timeout, but only %f ms elapsed", elapsedMs)); + } + } + } + @Test public void testShouldRunSelectQueryWithEmptyVectorsEmbedded() throws Exception { try (Statement statement = connection.createStatement(); diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerBuilderTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerBuilderTest.java index 6beaba8236..7b416638e1 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerBuilderTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerBuilderTest.java @@ -147,6 +147,7 @@ public void testDefaults() { assertNull(builder.clientCertificatePath); assertNull(builder.clientKeyPath); assertEquals(Optional.empty(), builder.catalog); + assertNull(builder.connectTimeout); } @Test diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImplTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImplTest.java index 4a46b5f5be..c780d53fab 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImplTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImplTest.java @@ -18,6 +18,7 @@ import static java.lang.Runtime.getRuntime; import static org.apache.arrow.driver.jdbc.utils.ArrowFlightConnectionConfigImpl.ArrowFlightConnectionProperty.CATALOG; +import static org.apache.arrow.driver.jdbc.utils.ArrowFlightConnectionConfigImpl.ArrowFlightConnectionProperty.CONNECT_TIMEOUT_MILLIS; import static org.apache.arrow.driver.jdbc.utils.ArrowFlightConnectionConfigImpl.ArrowFlightConnectionProperty.HOST; import static org.apache.arrow.driver.jdbc.utils.ArrowFlightConnectionConfigImpl.ArrowFlightConnectionProperty.PASSWORD; import static org.apache.arrow.driver.jdbc.utils.ArrowFlightConnectionConfigImpl.ArrowFlightConnectionProperty.PORT; @@ -27,6 +28,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +import java.time.Duration; import java.util.Properties; import java.util.Random; import java.util.function.Function; @@ -59,49 +61,67 @@ public void setUp() { public void testGetProperty( ArrowFlightConnectionProperty property, Object value, + Object expected, Function configFunction) { properties.put(property.camelName(), value); arrowFlightConnectionConfigFunction = configFunction; - assertThat(configFunction.apply(arrowFlightConnectionConfig), is(value)); - assertThat(arrowFlightConnectionConfigFunction.apply(arrowFlightConnectionConfig), is(value)); + assertThat(configFunction.apply(arrowFlightConnectionConfig), is(expected)); + assertThat( + arrowFlightConnectionConfigFunction.apply(arrowFlightConnectionConfig), is(expected)); } public static Stream provideParameters() { + int port = RANDOM.nextInt(Short.toUnsignedInt(Short.MAX_VALUE)); + boolean useEncryption = RANDOM.nextBoolean(); + int threadPoolSize = RANDOM.nextInt(getRuntime().availableProcessors()); return Stream.of( Arguments.of( HOST, "host", + "host", (Function) ArrowFlightConnectionConfigImpl::getHost), Arguments.of( PORT, - RANDOM.nextInt(Short.toUnsignedInt(Short.MAX_VALUE)), + port, + port, (Function) ArrowFlightConnectionConfigImpl::getPort), Arguments.of( USER, "user", + "user", (Function) ArrowFlightConnectionConfigImpl::getUser), Arguments.of( PASSWORD, "password", + "password", (Function) ArrowFlightConnectionConfigImpl::getPassword), Arguments.of( USE_ENCRYPTION, - RANDOM.nextBoolean(), + useEncryption, + useEncryption, (Function) ArrowFlightConnectionConfigImpl::useEncryption), Arguments.of( THREAD_POOL_SIZE, - RANDOM.nextInt(getRuntime().availableProcessors()), + threadPoolSize, + threadPoolSize, (Function) ArrowFlightConnectionConfigImpl::threadPoolSize), Arguments.of( CATALOG, "catalog", + "catalog", + (Function) + ArrowFlightConnectionConfigImpl::getCatalog), + Arguments.of( + CONNECT_TIMEOUT_MILLIS, + 5000, + Duration.ofMillis(5000), (Function) - ArrowFlightConnectionConfigImpl::getCatalog)); + ArrowFlightConnectionConfigImpl::getConnectTimeout)); } } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/FallbackFlightSqlProducer.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/FallbackFlightSqlProducer.java index 9aa257172c..670b9e3be0 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/FallbackFlightSqlProducer.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/FallbackFlightSqlProducer.java @@ -109,6 +109,16 @@ private FlightInfo getFlightInfo(FlightDescriptor descriptor, String query) { Location.forGrpcInsecure("localhost", 9999), Location.reuseConnection()) .build()); + } else if (query.equals("fallback with unresolvable")) { + endpoints = + Collections.singletonList( + FlightEndpoint.builder( + ticket, + // Inaccessible IP + // https://stackoverflow.com/questions/10456044/what-is-a-good-invalid-ip-address-to-use-for-unit-tests + Location.forGrpcInsecure("203.0.113.0", 9999), + Location.reuseConnection()) + .build()); } else { throw CallStatus.UNIMPLEMENTED.withDescription(query).toRuntimeException(); } From 437612c07e04fb0283010961a836de22521fa759 Mon Sep 17 00:00:00 2001 From: Ivan Chesnov Date: Wed, 9 Apr 2025 15:19:25 +0300 Subject: [PATCH 008/271] GH-87: [Vector] Add ExtensionWriter (#697) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Based on changes from https://github.com/apache/arrow/pull/41731. ## What's Changed Added writer ExtensionWriter with 3 methods: - write method for writing values from Extension holders; - writeExtensionType method for writing values (arguments is Object because we don't know exact type); - addExtensionTypeFactory method - because the exact vector and value type are unknown, the user should create their own extension type vector, write for it, and ExtensionTypeFactory, which should map the vector and writer. Closes #87. Co-authored-by: Finn Völkel --- .../templates/AbstractFieldWriter.java | 22 ++++ .../AbstractPromotableFieldWriter.java | 10 ++ .../main/codegen/templates/BaseWriter.java | 31 +++++ .../codegen/templates/PromotableWriter.java | 14 +++ .../main/codegen/templates/StructWriters.java | 26 ++++ .../codegen/templates/UnionListWriter.java | 23 ++++ .../codegen/templates/UnionMapWriter.java | 12 ++ .../main/codegen/templates/UnionWriter.java | 20 +++ .../impl/AbstractExtensionTypeWriter.java | 66 ++++++++++ .../impl/ExtensionTypeWriterFactory.java | 38 ++++++ .../complex/impl/UnionExtensionWriter.java | 79 ++++++++++++ .../vector/complex/writer/FieldWriter.java | 4 +- .../arrow/vector/holders/ExtensionHolder.java | 22 ++++ .../apache/arrow/vector/TestStructVector.java | 37 ++++++ .../org/apache/arrow/vector/UuidVector.java | 114 ++++++++++++++++++ .../complex/impl/TestPromotableWriter.java | 29 +++++ .../complex/impl/UuidWriterFactory.java | 31 +++++ .../vector/complex/impl/UuidWriterImpl.java | 47 ++++++++ .../complex/writer/TestSimpleWriter.java | 20 +++ .../arrow/vector/holder/UuidHolder.java | 23 ++++ .../vector/types/pojo/TestExtensionType.java | 70 +---------- .../arrow/vector/types/pojo/UuidType.java | 60 +++++++++ 22 files changed, 728 insertions(+), 70 deletions(-) create mode 100644 vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractExtensionTypeWriter.java create mode 100644 vector/src/main/java/org/apache/arrow/vector/complex/impl/ExtensionTypeWriterFactory.java create mode 100644 vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionExtensionWriter.java create mode 100644 vector/src/main/java/org/apache/arrow/vector/holders/ExtensionHolder.java create mode 100644 vector/src/test/java/org/apache/arrow/vector/UuidVector.java create mode 100644 vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java create mode 100644 vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java create mode 100644 vector/src/test/java/org/apache/arrow/vector/holder/UuidHolder.java create mode 100644 vector/src/test/java/org/apache/arrow/vector/types/pojo/UuidType.java diff --git a/vector/src/main/codegen/templates/AbstractFieldWriter.java b/vector/src/main/codegen/templates/AbstractFieldWriter.java index cc2cc618d8..ae5b97faef 100644 --- a/vector/src/main/codegen/templates/AbstractFieldWriter.java +++ b/vector/src/main/codegen/templates/AbstractFieldWriter.java @@ -107,6 +107,16 @@ public void endEntry() { throw new IllegalStateException(String.format("You tried to end a map entry when you are using a ValueWriter of type %s.", this.getClass().getSimpleName())); } + public void write(ExtensionHolder var1) { + this.fail("ExtensionType"); + } + public void writeExtension(Object var1) { + this.fail("ExtensionType"); + } + public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory var1) { + this.fail("ExtensionType"); + } + <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) /> @@ -241,6 +251,18 @@ public MapWriter map(String name, boolean keysSorted) { fail("Map"); return null; } + + @Override + public ExtensionWriter extension(String name, ArrowType arrowType) { + fail("Extension"); + return null; + } + + @Override + public ExtensionWriter extension(ArrowType arrowType) { + fail("Extension"); + return null; + } <#list vv.types as type><#list type.minor as minor> <#assign lowerName = minor.class?uncap_first /> <#if lowerName == "int" ><#assign lowerName = "integer" /> diff --git a/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java b/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java index 06cb235f7d..951edd5eee 100644 --- a/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java +++ b/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java @@ -293,6 +293,11 @@ public MapWriter map(boolean keysSorted) { return getWriter(MinorType.MAP, new ArrowType.Map(keysSorted)); } + @Override + public ExtensionWriter extension(ArrowType arrowType) { + return getWriter(MinorType.EXTENSIONTYPE).extension(arrowType); + } + @Override public StructWriter struct(String name) { return getWriter(MinorType.STRUCT).struct(name); @@ -318,6 +323,11 @@ public MapWriter map(String name, boolean keysSorted) { return getWriter(MinorType.STRUCT).map(name, keysSorted); } + @Override + public ExtensionWriter extension(String name, ArrowType arrowType) { + return getWriter(MinorType.EXTENSIONTYPE).extension(name, arrowType); + } + <#list vv.types as type><#list type.minor as minor> <#assign lowerName = minor.class?uncap_first /> <#if lowerName == "int" ><#assign lowerName = "integer" /> diff --git a/vector/src/main/codegen/templates/BaseWriter.java b/vector/src/main/codegen/templates/BaseWriter.java index e952d46f1f..78da7fddc3 100644 --- a/vector/src/main/codegen/templates/BaseWriter.java +++ b/vector/src/main/codegen/templates/BaseWriter.java @@ -61,6 +61,7 @@ public interface StructWriter extends BaseWriter { void copyReaderToField(String name, FieldReader reader); StructWriter struct(String name); + ExtensionWriter extension(String name, ArrowType arrowType); ListWriter list(String name); ListWriter listView(String name); MapWriter map(String name); @@ -79,6 +80,7 @@ public interface ListWriter extends BaseWriter { ListWriter listView(); MapWriter map(); MapWriter map(boolean keysSorted); + ExtensionWriter extension(ArrowType arrowType); void copyReader(FieldReader reader); <#list vv.types as type><#list type.minor as minor> @@ -101,6 +103,35 @@ public interface MapWriter extends ListWriter { MapWriter value(); } + public interface ExtensionWriter extends BaseWriter { + + /** + * Writes a null value. + */ + void writeNull(); + + /** + * Writes value from the given extension holder. + * + * @param holder the extension holder to write + */ + void write(ExtensionHolder holder); + + /** + * Writes the given extension type value. + * + * @param value the extension type value to write + */ + void writeExtension(Object value); + + /** + * Adds the given extension type factory. This factory allows configuring writer implementations for specific ExtensionTypeVector. + * + * @param factory the extension type factory to add + */ + void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory factory); + } + public interface ScalarWriter extends <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> ${name}Writer, BaseWriter {} diff --git a/vector/src/main/codegen/templates/PromotableWriter.java b/vector/src/main/codegen/templates/PromotableWriter.java index c0e686f317..8d7d57bb9d 100644 --- a/vector/src/main/codegen/templates/PromotableWriter.java +++ b/vector/src/main/codegen/templates/PromotableWriter.java @@ -285,6 +285,9 @@ protected void setWriter(ValueVector v) { case UNION: writer = new UnionWriter((UnionVector) vector, nullableStructWriterFactory); break; + case EXTENSIONTYPE: + writer = new UnionExtensionWriter((ExtensionTypeVector) vector); + break; default: writer = type.getNewFieldWriter(vector); break; @@ -316,6 +319,7 @@ protected boolean requiresArrowType(MinorType type) { || type == MinorType.MAP || type == MinorType.DURATION || type == MinorType.FIXEDSIZEBINARY + || type == MinorType.EXTENSIONTYPE || (type.name().startsWith("TIMESTAMP") && type.name().endsWith("TZ")); } @@ -536,6 +540,16 @@ public void writeLargeVarChar(String value) { getWriter(MinorType.LARGEVARCHAR).writeLargeVarChar(value); } + @Override + public void writeExtension(Object value) { + getWriter(MinorType.EXTENSIONTYPE).writeExtension(value); + } + + @Override + public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory factory) { + getWriter(MinorType.EXTENSIONTYPE).addExtensionTypeWriterFactory(factory); + } + @Override public void allocate() { getWriter().allocate(); diff --git a/vector/src/main/codegen/templates/StructWriters.java b/vector/src/main/codegen/templates/StructWriters.java index 3e6b9fd773..413f707c70 100644 --- a/vector/src/main/codegen/templates/StructWriters.java +++ b/vector/src/main/codegen/templates/StructWriters.java @@ -83,6 +83,9 @@ public class ${mode}StructWriter extends AbstractFieldWriter { fields.put(handleCase(child.getName()), writer); break; } + case EXTENSIONTYPE: + extension(child.getName(), child.getType()); + break; case UNION: FieldType fieldType = new FieldType(addVectorAsNullable, MinorType.UNION.getType(), null, null); UnionWriter writer = new UnionWriter(container.addOrGet(child.getName(), fieldType, UnionVector.class), getNullableStructWriterFactory()); @@ -159,6 +162,29 @@ public StructWriter struct(String name) { return writer; } + @Override + public ExtensionWriter extension(String name, ArrowType arrowType) { + String finalName = handleCase(name); + FieldWriter writer = fields.get(finalName); + if(writer == null){ + int vectorCount=container.size(); + FieldType fieldType = new FieldType(addVectorAsNullable, arrowType, null, null); + ExtensionTypeVector vector = container.addOrGet(name, fieldType, ExtensionTypeVector.class); + writer = new PromotableWriter(vector, container, getNullableStructWriterFactory()); + if(vectorCount != container.size()) { + writer.allocate(); + } + writer.setPosition(idx()); + fields.put(finalName, writer); + } else { + if (writer instanceof PromotableWriter) { + // ensure writers are initialized + ((PromotableWriter)writer).getWriter(MinorType.EXTENSIONTYPE, arrowType); + } + } + return (ExtensionWriter) writer; + } + @Override public void close() throws Exception { clear(); diff --git a/vector/src/main/codegen/templates/UnionListWriter.java b/vector/src/main/codegen/templates/UnionListWriter.java index 3962e1d073..9424533f29 100644 --- a/vector/src/main/codegen/templates/UnionListWriter.java +++ b/vector/src/main/codegen/templates/UnionListWriter.java @@ -201,6 +201,17 @@ public MapWriter map(String name, boolean keysSorted) { return mapWriter; } + @Override + public ExtensionWriter extension(ArrowType arrowType) { + writer.extension(arrowType); + return writer; + } + @Override + public ExtensionWriter extension(String name, ArrowType arrowType) { + ExtensionWriter extensionWriter = writer.extension(name, arrowType); + return extensionWriter; + } + <#if listName == "LargeList"> @Override public void startList() { @@ -323,6 +334,18 @@ public void writeNull() { } } + @Override + public void writeExtension(Object value) { + writer.writeExtension(value); + } + @Override + public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory var1) { + writer.addExtensionTypeWriterFactory(var1); + } + public void write(ExtensionHolder var1) { + writer.write(var1); + } + <#list vv.types as type> <#list type.minor as minor> <#assign name = minor.class?cap_first /> diff --git a/vector/src/main/codegen/templates/UnionMapWriter.java b/vector/src/main/codegen/templates/UnionMapWriter.java index 90b55cb65e..8b2f091215 100644 --- a/vector/src/main/codegen/templates/UnionMapWriter.java +++ b/vector/src/main/codegen/templates/UnionMapWriter.java @@ -231,4 +231,16 @@ public MapWriter map() { return super.map(); } } + + @Override + public ExtensionWriter extension(ArrowType type) { + switch (mode) { + case KEY: + return entryWriter.extension(MapVector.KEY_NAME, type); + case VALUE: + return entryWriter.extension(MapVector.VALUE_NAME, type); + default: + return super.extension(type); + } + } } diff --git a/vector/src/main/codegen/templates/UnionWriter.java b/vector/src/main/codegen/templates/UnionWriter.java index bfe97e2770..272edab17c 100644 --- a/vector/src/main/codegen/templates/UnionWriter.java +++ b/vector/src/main/codegen/templates/UnionWriter.java @@ -213,6 +213,10 @@ public MapWriter asMap(ArrowType arrowType) { return getMapWriter(arrowType); } + private ExtensionWriter getExtensionWriter(ArrowType arrowType) { + throw new UnsupportedOperationException("ExtensionTypes are not supported yet."); + } + BaseWriter getWriter(MinorType minorType) { return getWriter(minorType, null); } @@ -227,6 +231,8 @@ BaseWriter getWriter(MinorType minorType, ArrowType arrowType) { return getListViewWriter(); case MAP: return getMapWriter(arrowType); + case EXTENSIONTYPE: + return getExtensionWriter(arrowType); <#list vv.types as type> <#list type.minor as minor> <#assign name = minor.class?cap_first /> @@ -460,6 +466,20 @@ public MapWriter map(String name, boolean keysSorted) { return getStructWriter().map(name, keysSorted); } + @Override + public ExtensionWriter extension(ArrowType arrowType) { + data.setType(idx(), MinorType.EXTENSIONTYPE); + getListWriter().setPosition(idx()); + return getListWriter().extension(arrowType); + } + + @Override + public ExtensionWriter extension(String name, ArrowType arrowType) { + data.setType(idx(), MinorType.EXTENSIONTYPE); + getStructWriter().setPosition(idx()); + return getStructWriter().extension(name, arrowType); + } + <#list vv.types as type><#list type.minor as minor> <#assign lowerName = minor.class?uncap_first /> <#if lowerName == "int" ><#assign lowerName = "integer" /> diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractExtensionTypeWriter.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractExtensionTypeWriter.java new file mode 100644 index 0000000000..fccff6c21f --- /dev/null +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractExtensionTypeWriter.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.complex.impl; + +import org.apache.arrow.vector.ExtensionTypeVector; +import org.apache.arrow.vector.types.pojo.Field; + +/** + * Base {@link AbstractFieldWriter} class for an {@link + * org.apache.arrow.vector.ExtensionTypeVector}. + * + * @param a specific {@link ExtensionTypeVector}. + */ +public class AbstractExtensionTypeWriter + extends AbstractFieldWriter { + protected final T vector; + + public AbstractExtensionTypeWriter(T vector) { + this.vector = vector; + } + + @Override + public Field getField() { + return this.vector.getField(); + } + + @Override + public int getValueCapacity() { + return this.vector.getValueCapacity(); + } + + @Override + public void allocate() { + this.vector.allocateNew(); + } + + @Override + public void close() { + this.vector.close(); + } + + @Override + public void clear() { + this.vector.clear(); + } + + @Override + public void writeNull() { + this.vector.setNull(getPosition()); + this.vector.setValueCount(getPosition() + 1); + } +} diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/ExtensionTypeWriterFactory.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/ExtensionTypeWriterFactory.java new file mode 100644 index 0000000000..09f0314c5f --- /dev/null +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/ExtensionTypeWriterFactory.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.complex.impl; + +import org.apache.arrow.vector.ExtensionTypeVector; +import org.apache.arrow.vector.complex.writer.FieldWriter; + +/** + * A factory interface for creating instances of {@link ExtensionTypeWriter}. This factory allows + * configuring writer implementations for specific {@link ExtensionTypeVector}. + * + * @param the type of writer implementation for a specific {@link ExtensionTypeVector}. + */ +public interface ExtensionTypeWriterFactory { + + /** + * Returns an instance of the writer implementation for the given {@link ExtensionTypeVector}. + * + * @param vector the {@link ExtensionTypeVector} for which the writer implementation is to be + * returned. + * @return an instance of the writer implementation for the given {@link ExtensionTypeVector}. + */ + T getWriterImpl(ExtensionTypeVector vector); +} diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionExtensionWriter.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionExtensionWriter.java new file mode 100644 index 0000000000..d341384bd9 --- /dev/null +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionExtensionWriter.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.complex.impl; + +import org.apache.arrow.vector.ExtensionTypeVector; +import org.apache.arrow.vector.complex.writer.FieldWriter; +import org.apache.arrow.vector.holders.ExtensionHolder; +import org.apache.arrow.vector.types.pojo.Field; + +public class UnionExtensionWriter extends AbstractFieldWriter { + protected ExtensionTypeVector vector; + protected FieldWriter writer; + + public UnionExtensionWriter(ExtensionTypeVector vector) { + this.vector = vector; + } + + @Override + public void allocate() { + vector.allocateNew(); + } + + @Override + public void clear() { + vector.clear(); + } + + @Override + public int getValueCapacity() { + return vector.getValueCapacity(); + } + + @Override + public Field getField() { + return vector.getField(); + } + + @Override + public void close() throws Exception { + vector.close(); + } + + @Override + public void writeExtension(Object var1) { + this.writer.writeExtension(var1); + } + + @Override + public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory factory) { + this.writer = factory.getWriterImpl(vector); + this.writer.setPosition(idx()); + } + + public void write(ExtensionHolder holder) { + this.writer.write(holder); + } + + @Override + public void setPosition(int index) { + super.setPosition(index); + if (this.writer != null) { + this.writer.setPosition(index); + } + } +} diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/writer/FieldWriter.java b/vector/src/main/java/org/apache/arrow/vector/complex/writer/FieldWriter.java index 949eb35d8e..51bf106685 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/writer/FieldWriter.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/writer/FieldWriter.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.vector.complex.writer; +import org.apache.arrow.vector.complex.writer.BaseWriter.ExtensionWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.ScalarWriter; @@ -25,7 +26,8 @@ * Composite of all writer types. Writers are convenience classes for incrementally adding values to * {@linkplain org.apache.arrow.vector.ValueVector}s. */ -public interface FieldWriter extends StructWriter, ListWriter, MapWriter, ScalarWriter { +public interface FieldWriter + extends StructWriter, ListWriter, MapWriter, ScalarWriter, ExtensionWriter { void allocate(); void clear(); diff --git a/vector/src/main/java/org/apache/arrow/vector/holders/ExtensionHolder.java b/vector/src/main/java/org/apache/arrow/vector/holders/ExtensionHolder.java new file mode 100644 index 0000000000..fc7ed85878 --- /dev/null +++ b/vector/src/main/java/org/apache/arrow/vector/holders/ExtensionHolder.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.holders; + +/** Base {@link ValueHolder} class for a {@link org.apache.arrow.vector.ExtensionTypeVector}. */ +public abstract class ExtensionHolder implements ValueHolder { + public int isSet; +} diff --git a/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java b/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java index 4ef0fbe2d9..d40af9ae89 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java @@ -26,6 +26,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.complex.AbstractStructVector; import org.apache.arrow.vector.complex.ListVector; @@ -37,9 +38,11 @@ import org.apache.arrow.vector.holders.ComplexHolder; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.Types.MinorType; +import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.ArrowType.Struct; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.UuidType; import org.apache.arrow.vector.util.TransferPair; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -336,6 +339,40 @@ public void testGetTransferPairWithFieldAndCallBack() { } } + @Test + public void testStructVectorWithExtensionTypes() { + UuidType uuidType = new UuidType(); + Field uuidField = new Field("struct_child", FieldType.nullable(uuidType), null); + Field structField = + new Field("struct", FieldType.nullable(new ArrowType.Struct()), List.of(uuidField)); + StructVector s1 = new StructVector(structField, allocator, null); + StructVector s2 = (StructVector) structField.createVector(allocator); + s1.close(); + s2.close(); + } + + @Test + public void testStructVectorTransferPairWithExtensionType() { + UuidType uuidType = new UuidType(); + Field uuidField = new Field("uuid_child", FieldType.nullable(uuidType), null); + Field structField = + new Field("struct", FieldType.nullable(new ArrowType.Struct()), List.of(uuidField)); + + StructVector s1 = (StructVector) structField.createVector(allocator); + UuidVector uuidVector = + s1.addOrGet("uuid_child", FieldType.nullable(uuidType), UuidVector.class); + s1.setValueCount(1); + uuidVector.set(0, new UUID(1, 2)); + s1.setIndexDefined(0); + + TransferPair tp = s1.getTransferPair(structField, allocator); + final StructVector toVector = (StructVector) tp.getTo(); + assertEquals(s1.getField(), toVector.getField()); + + s1.close(); + toVector.close(); + } + private StructVector simpleStructVector(String name, BufferAllocator allocator) { final String INT_COL = "struct_int_child"; final String FLT_COL = "struct_flt_child"; diff --git a/vector/src/test/java/org/apache/arrow/vector/UuidVector.java b/vector/src/test/java/org/apache/arrow/vector/UuidVector.java new file mode 100644 index 0000000000..5c90d45f60 --- /dev/null +++ b/vector/src/test/java/org/apache/arrow/vector/UuidVector.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector; + +import java.nio.ByteBuffer; +import java.util.UUID; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.util.hash.ArrowBufHasher; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.UuidType; +import org.apache.arrow.vector.util.TransferPair; + +public class UuidVector extends ExtensionTypeVector + implements ValueIterableVector { + private final Field field; + + public UuidVector( + String name, BufferAllocator allocator, FixedSizeBinaryVector underlyingVector) { + super(name, allocator, underlyingVector); + this.field = new Field(name, FieldType.nullable(new UuidType()), null); + } + + public UuidVector(String name, BufferAllocator allocator) { + super(name, allocator, new FixedSizeBinaryVector(name, allocator, 16)); + this.field = new Field(name, FieldType.nullable(new UuidType()), null); + } + + @Override + public UUID getObject(int index) { + final ByteBuffer bb = ByteBuffer.wrap(getUnderlyingVector().getObject(index)); + return new UUID(bb.getLong(), bb.getLong()); + } + + @Override + public int hashCode(int index) { + return hashCode(index, null); + } + + @Override + public int hashCode(int index, ArrowBufHasher hasher) { + return getUnderlyingVector().hashCode(index, hasher); + } + + public void set(int index, UUID uuid) { + ByteBuffer bb = ByteBuffer.allocate(16); + bb.putLong(uuid.getMostSignificantBits()); + bb.putLong(uuid.getLeastSignificantBits()); + getUnderlyingVector().set(index, bb.array()); + } + + @Override + public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { + getUnderlyingVector() + .copyFromSafe(fromIndex, thisIndex, ((UuidVector) from).getUnderlyingVector()); + } + + @Override + public Field getField() { + return field; + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((UuidVector) to); + } + + public void setSafe(int index, byte[] value) { + getUnderlyingVector().setIndexDefined(index); + getUnderlyingVector().setSafe(index, value); + } + + public class TransferImpl implements TransferPair { + UuidVector to; + ValueVector targetUnderlyingVector; + TransferPair tp; + + public TransferImpl(UuidVector to) { + this.to = to; + targetUnderlyingVector = this.to.getUnderlyingVector(); + tp = getUnderlyingVector().makeTransferPair(targetUnderlyingVector); + } + + public UuidVector getTo() { + return this.to; + } + + public void transfer() { + tp.transfer(); + } + + public void splitAndTransfer(int startIndex, int length) { + tp.splitAndTransfer(startIndex, length); + } + + public void copyValueSafe(int fromIndex, int toIndex) { + tp.copyValueSafe(fromIndex, toIndex); + } + } +} diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java index a791e55135..1556852c5a 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java @@ -26,12 +26,14 @@ import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; import java.util.Objects; +import java.util.UUID; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.DecimalVector; import org.apache.arrow.vector.DirtyRootAllocator; import org.apache.arrow.vector.LargeVarBinaryVector; import org.apache.arrow.vector.LargeVarCharVector; +import org.apache.arrow.vector.UuidVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.complex.ListVector; @@ -52,6 +54,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.UuidType; import org.apache.arrow.vector.util.DecimalUtility; import org.apache.arrow.vector.util.Text; import org.junit.jupiter.api.AfterEach; @@ -776,4 +779,30 @@ public void testPromoteToUnionFromDecimal() throws Exception { assertEquals(1, intHolder.value); } } + + @Test + public void testExtensionType() throws Exception { + try (final NonNullableStructVector container = + NonNullableStructVector.empty(EMPTY_SCHEMA_PATH, allocator); + final UuidVector v = + container.addOrGet("uuid", FieldType.nullable(new UuidType()), UuidVector.class); + final PromotableWriter writer = new PromotableWriter(v, container)) { + UUID u1 = UUID.randomUUID(); + UUID u2 = UUID.randomUUID(); + container.allocateNew(); + container.setValueCount(1); + writer.addExtensionTypeWriterFactory(new UuidWriterFactory()); + + writer.setPosition(0); + writer.writeExtension(u1); + writer.setPosition(1); + writer.writeExtension(u2); + + container.setValueCount(2); + + UuidVector uuidVector = (UuidVector) container.getChild("uuid"); + assertEquals(u1, uuidVector.getObject(0)); + assertEquals(u2, uuidVector.getObject(1)); + } + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java new file mode 100644 index 0000000000..1b1bf4e6e4 --- /dev/null +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.complex.impl; + +import org.apache.arrow.vector.ExtensionTypeVector; +import org.apache.arrow.vector.UuidVector; + +public class UuidWriterFactory implements ExtensionTypeWriterFactory { + + @Override + public AbstractFieldWriter getWriterImpl(ExtensionTypeVector extensionTypeVector) { + if (extensionTypeVector instanceof UuidVector) { + return new UuidWriterImpl((UuidVector) extensionTypeVector); + } + return null; + } +} diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java new file mode 100644 index 0000000000..68029b1df5 --- /dev/null +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.complex.impl; + +import java.nio.ByteBuffer; +import java.util.UUID; +import org.apache.arrow.vector.UuidVector; +import org.apache.arrow.vector.holder.UuidHolder; +import org.apache.arrow.vector.holders.ExtensionHolder; + +public class UuidWriterImpl extends AbstractExtensionTypeWriter { + + public UuidWriterImpl(UuidVector vector) { + super(vector); + } + + @Override + public void writeExtension(Object value) { + UUID uuid = (UUID) value; + ByteBuffer bb = ByteBuffer.allocate(16); + bb.putLong(uuid.getMostSignificantBits()); + bb.putLong(uuid.getLeastSignificantBits()); + vector.setSafe(getPosition(), bb.array()); + vector.setValueCount(getPosition() + 1); + } + + @Override + public void write(ExtensionHolder holder) { + UuidHolder uuidHolder = (UuidHolder) holder; + vector.setSafe(getPosition(), uuidHolder.value); + vector.setValueCount(getPosition() + 1); + } +} diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java index 5bb5962704..bf1b9b0dfa 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java @@ -20,16 +20,20 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import java.nio.ByteBuffer; +import java.util.UUID; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.LargeVarBinaryVector; import org.apache.arrow.vector.LargeVarCharVector; +import org.apache.arrow.vector.UuidVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.complex.impl.LargeVarBinaryWriterImpl; import org.apache.arrow.vector.complex.impl.LargeVarCharWriterImpl; +import org.apache.arrow.vector.complex.impl.UuidWriterImpl; import org.apache.arrow.vector.complex.impl.VarBinaryWriterImpl; import org.apache.arrow.vector.complex.impl.VarCharWriterImpl; +import org.apache.arrow.vector.holder.UuidHolder; import org.apache.arrow.vector.util.Text; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -184,4 +188,20 @@ public void testWriteTextToLargeVarChar() throws Exception { assertEquals(input, result); } } + + @Test + public void testWriteToExtensionVector() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator); + UuidWriterImpl writer = new UuidWriterImpl(vector)) { + UUID uuid = UUID.randomUUID(); + ByteBuffer bb = ByteBuffer.allocate(16); + bb.putLong(uuid.getMostSignificantBits()); + bb.putLong(uuid.getLeastSignificantBits()); + UuidHolder holder = new UuidHolder(); + holder.value = bb.array(); + writer.write(holder); + UUID result = vector.getObject(0); + assertEquals(uuid, result); + } + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/holder/UuidHolder.java b/vector/src/test/java/org/apache/arrow/vector/holder/UuidHolder.java new file mode 100644 index 0000000000..207b0951a7 --- /dev/null +++ b/vector/src/test/java/org/apache/arrow/vector/holder/UuidHolder.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.holder; + +import org.apache.arrow.vector.holders.ExtensionHolder; + +public class UuidHolder extends ExtensionHolder { + public byte[] value; +} diff --git a/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java b/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java index 8f54a6e5d7..d24708d66c 100644 --- a/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java +++ b/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java @@ -41,6 +41,7 @@ import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.FixedSizeBinaryVector; import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.UuidVector; import org.apache.arrow.vector.ValueIterableVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.compare.Range; @@ -295,75 +296,6 @@ public void testVectorCompare() { } } - static class UuidType extends ExtensionType { - - @Override - public ArrowType storageType() { - return new ArrowType.FixedSizeBinary(16); - } - - @Override - public String extensionName() { - return "uuid"; - } - - @Override - public boolean extensionEquals(ExtensionType other) { - return other instanceof UuidType; - } - - @Override - public ArrowType deserialize(ArrowType storageType, String serializedData) { - if (!storageType.equals(storageType())) { - throw new UnsupportedOperationException( - "Cannot construct UuidType from underlying type " + storageType); - } - return new UuidType(); - } - - @Override - public String serialize() { - return ""; - } - - @Override - public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator) { - return new UuidVector(name, allocator, new FixedSizeBinaryVector(name, allocator, 16)); - } - } - - static class UuidVector extends ExtensionTypeVector - implements ValueIterableVector { - - public UuidVector( - String name, BufferAllocator allocator, FixedSizeBinaryVector underlyingVector) { - super(name, allocator, underlyingVector); - } - - @Override - public UUID getObject(int index) { - final ByteBuffer bb = ByteBuffer.wrap(getUnderlyingVector().getObject(index)); - return new UUID(bb.getLong(), bb.getLong()); - } - - @Override - public int hashCode(int index) { - return hashCode(index, null); - } - - @Override - public int hashCode(int index, ArrowBufHasher hasher) { - return getUnderlyingVector().hashCode(index, hasher); - } - - public void set(int index, UUID uuid) { - ByteBuffer bb = ByteBuffer.allocate(16); - bb.putLong(uuid.getMostSignificantBits()); - bb.putLong(uuid.getLeastSignificantBits()); - getUnderlyingVector().set(index, bb.array()); - } - } - static class LocationType extends ExtensionType { @Override diff --git a/vector/src/test/java/org/apache/arrow/vector/types/pojo/UuidType.java b/vector/src/test/java/org/apache/arrow/vector/types/pojo/UuidType.java new file mode 100644 index 0000000000..5e2bd8881b --- /dev/null +++ b/vector/src/test/java/org/apache/arrow/vector/types/pojo/UuidType.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.types.pojo; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.FixedSizeBinaryVector; +import org.apache.arrow.vector.UuidVector; +import org.apache.arrow.vector.types.pojo.ArrowType.ExtensionType; + +public class UuidType extends ExtensionType { + + @Override + public ArrowType storageType() { + return new ArrowType.FixedSizeBinary(16); + } + + @Override + public String extensionName() { + return "uuid"; + } + + @Override + public boolean extensionEquals(ExtensionType other) { + return other instanceof UuidType; + } + + @Override + public ArrowType deserialize(ArrowType storageType, String serializedData) { + if (!storageType.equals(storageType())) { + throw new UnsupportedOperationException( + "Cannot construct UuidType from underlying type " + storageType); + } + return new UuidType(); + } + + @Override + public String serialize() { + return ""; + } + + @Override + public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator) { + return new UuidVector(name, allocator, new FixedSizeBinaryVector(name, allocator, 16)); + } +} From 3e135f4f24551cd932c7ac0646ca58aa99c07a90 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 14 Apr 2025 11:37:26 +0900 Subject: [PATCH 009/271] GH-708: Enable GitHub discussions (#710) ## What's Changed Enable GitHub Discussions like other Arrow repositories. Fixes #708. --------- Co-authored-by: Sutou Kouhei --- .asf.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.asf.yaml b/.asf.yaml index ead2581149..13bd59c224 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -29,12 +29,14 @@ github: rebase: false squash: true features: + discussions: true issues: true protected_branches: main: required_linear_history: true notifications: commits: commits@arrow.apache.org + discussions: user@arrow.apache.org issues_status: issues@arrow.apache.org issues_comment: github@arrow.apache.org pullrequests: github@arrow.apache.org From 74e8981d5ba0646f2ee1dbc99364766650ad084f Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Mon, 14 Apr 2025 07:46:56 +0200 Subject: [PATCH 010/271] GH-709: Correct length calculation of value buffers of variable-sized arrays (#707) ## What's Changed For variable-size binary layout arrays, BufferImportTypeVisitor currently derives the length of the value buffer by calculating the difference between the last and first offset. When the first offset is not zero, this is actually incorrect and leads to out of bounds errors when attempting to read values from the imported array. Instead, BufferImportTypeVisitor should simply use the last offset value as the length of the value buffer. This PR makes that change. Just FYI, I bumped into this issue when attempting to import an array originating from DataFusion. A test query of the form `SELECT column1 FROM VALUES ('a'), ('b'), ('c'), ('d') LIMIT 2 OFFSET 1;` returns a slice of the full set of values. The values buffer contains all the original values, and the offsets buffer contains 1 and 2 as values to handle the offset from the query. Closes #709. --- .../org/apache/arrow/c/BufferImportTypeVisitor.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/c/src/main/java/org/apache/arrow/c/BufferImportTypeVisitor.java b/c/src/main/java/org/apache/arrow/c/BufferImportTypeVisitor.java index 5ca398c9f9..10f690fc87 100644 --- a/c/src/main/java/org/apache/arrow/c/BufferImportTypeVisitor.java +++ b/c/src/main/java/org/apache/arrow/c/BufferImportTypeVisitor.java @@ -228,9 +228,8 @@ public List visit(ArrowType.Utf8 type) { type, start, end); - final int len = end - start; offsets.getReferenceManager().retain(); - return Arrays.asList(maybeImportBitmap(type), offsets, importData(type, len)); + return Arrays.asList(maybeImportBitmap(type), offsets, importData(type, end)); } } @@ -279,9 +278,8 @@ public List visit(ArrowType.LargeUtf8 type) { type, start, end); - final long len = end - start; offsets.getReferenceManager().retain(); - return Arrays.asList(maybeImportBitmap(type), offsets, importData(type, len)); + return Arrays.asList(maybeImportBitmap(type), offsets, importData(type, end)); } } @@ -296,9 +294,8 @@ public List visit(ArrowType.Binary type) { type, start, end); - final int len = end - start; offsets.getReferenceManager().retain(); - return Arrays.asList(maybeImportBitmap(type), offsets, importData(type, len)); + return Arrays.asList(maybeImportBitmap(type), offsets, importData(type, end)); } } @@ -320,9 +317,8 @@ public List visit(ArrowType.LargeBinary type) { type, start, end); - final long len = end - start; offsets.getReferenceManager().retain(); - return Arrays.asList(maybeImportBitmap(type), offsets, importData(type, len)); + return Arrays.asList(maybeImportBitmap(type), offsets, importData(type, end)); } } From 4b4d92851c8ecba1876a8f452279122e88e5ff8e Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 14 Apr 2025 21:47:30 +0900 Subject: [PATCH 011/271] MINOR: Don't uninstall Flatbuffers in CI (#711) ## What's Changed There is no more need after https://github.com/apache/arrow/commit/32641ecfd97e302ea8119b1147d77aced96d097b Also, disable building tests to avoid issues with Boost/Homebrew. They aren't executed anyways. --- .github/workflows/rc.yml | 5 ----- ci/scripts/jni_macos_build.sh | 2 +- ci/scripts/jni_manylinux_build.sh | 2 +- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index e039306ec3..5b78cc9395 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -261,11 +261,6 @@ jobs: # bundled Protobuf. brew uninstall protobuf - # We need Flatbuffers 24, not the latest version - # Homebrew does not offer older versions, so remove the Homebrew - # package and rely on Arrow using a bundled version instead - brew uninstall flatbuffers - brew bundle --file=Brewfile - name: Prepare ccache run: | diff --git a/ci/scripts/jni_macos_build.sh b/ci/scripts/jni_macos_build.sh index 65b255e2dc..f7543b6f7a 100755 --- a/ci/scripts/jni_macos_build.sh +++ b/ci/scripts/jni_macos_build.sh @@ -61,7 +61,7 @@ github_actions_group_begin "Building Arrow C++ libraries" install_dir="${build_dir}/cpp-install" : "${ARROW_ACERO:=ON}" export ARROW_ACERO -: "${ARROW_BUILD_TESTS:=ON}" +: "${ARROW_BUILD_TESTS:=OFF}" export ARROW_BUILD_TESTS : "${ARROW_DATASET:=ON}" export ARROW_DATASET diff --git a/ci/scripts/jni_manylinux_build.sh b/ci/scripts/jni_manylinux_build.sh index 148d2e02f6..a34ec0f420 100755 --- a/ci/scripts/jni_manylinux_build.sh +++ b/ci/scripts/jni_manylinux_build.sh @@ -57,7 +57,7 @@ devtoolset_version="$(rpm -qa "devtoolset-*-gcc" --queryformat '%{VERSION}' | gr devtoolset_include_cpp="/opt/rh/devtoolset-${devtoolset_version}/root/usr/include/c++/${devtoolset_version}" : "${ARROW_ACERO:=ON}" export ARROW_ACERO -: "${ARROW_BUILD_TESTS:=ON}" +: "${ARROW_BUILD_TESTS:=OFF}" export ARROW_BUILD_TESTS : "${ARROW_DATASET:=ON}" export ARROW_DATASET From 4af464cc80361fdbfcb2024d3ff848b76c27524f Mon Sep 17 00:00:00 2001 From: Gabor Szadovszky Date: Wed, 23 Apr 2025 01:38:35 +0200 Subject: [PATCH 012/271] GH-721: Allow using 1GB+ data buffers in variable width vectors (#722) ## What's Changed Allow actually reaching MAX_BUFFER_SIZE at reallocating variable width vectors instead of exceeding it calculating the next power of 2. For unit testing the maximum allocation size has be increased to 2MB - 1byte to simulate the default maximum behavior. Due to this change needed some updates in existing unit tests because of the round ups used at calculating the required buffer sizes. Closes #721. --- pom.xml | 4 ++-- .../arrow/vector/BaseVariableWidthVector.java | 7 +++++-- .../arrow/vector/BaseVariableWidthViewVector.java | 14 ++++++++++---- .../org/apache/arrow/vector/TestValueVector.java | 2 +- .../org/apache/arrow/vector/TestVectorReAlloc.java | 12 ++++++++++++ .../arrow/vector/util/TestVectorAppender.java | 11 ++++++++++- 6 files changed, 40 insertions(+), 10 deletions(-) diff --git a/pom.xml b/pom.xml index cb189dc8f9..ab988c7d05 100644 --- a/pom.xml +++ b/pom.xml @@ -327,8 +327,8 @@ under the License. true UTC - 1048576 + which in turn can cause OOM. Using 2MB - 1byte to simulate the defaul limit of 2^31 - 1 bytes. --> + 2097151 false diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java index 7b8d2cdfda..1609e64ca5 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java @@ -571,10 +571,13 @@ public void reallocDataBuffer(long desiredAllocSize) { return; } - final long newAllocationSize = CommonUtil.nextPowerOfTwo(desiredAllocSize); + final long newAllocationSize = + Math.min(CommonUtil.nextPowerOfTwo(desiredAllocSize), MAX_BUFFER_SIZE); assert newAllocationSize >= 1; - checkDataBufferSize(newAllocationSize); + if (newAllocationSize < desiredAllocSize) { + checkDataBufferSize(desiredAllocSize); + } final ArrowBuf newBuf = allocator.buffer(newAllocationSize); newBuf.setBytes(0, valueBuffer, 0, valueBuffer.capacity()); diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java index e0e16762f2..beda91dc3f 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java @@ -550,15 +550,18 @@ public void reallocViewBuffer(long desiredAllocSize) { if (desiredAllocSize == 0) { return; } - long newAllocationSize = CommonUtil.nextPowerOfTwo(desiredAllocSize); + long newAllocationSize = Math.min(CommonUtil.nextPowerOfTwo(desiredAllocSize), MAX_BUFFER_SIZE); assert newAllocationSize >= 1; - checkDataBufferSize(newAllocationSize); // for each set operation, we have to allocate 16 bytes // here we are adjusting the desired allocation-based allocation size // to align with the 16bytes requirement. newAllocationSize = roundUpToMultipleOf16(newAllocationSize); + if (newAllocationSize < desiredAllocSize) { + checkDataBufferSize(desiredAllocSize); + } + final ArrowBuf newBuf = allocator.buffer(newAllocationSize); newBuf.setBytes(0, viewBuffer, 0, viewBuffer.capacity()); @@ -587,10 +590,13 @@ public void reallocViewDataBuffer(long desiredAllocSize) { return; } - final long newAllocationSize = CommonUtil.nextPowerOfTwo(desiredAllocSize); + final long newAllocationSize = + Math.min(CommonUtil.nextPowerOfTwo(desiredAllocSize), MAX_BUFFER_SIZE); assert newAllocationSize >= 1; - checkDataBufferSize(newAllocationSize); + if (newAllocationSize < desiredAllocSize) { + checkDataBufferSize(desiredAllocSize); + } final ArrowBuf newBuf = allocator.buffer(newAllocationSize); dataBuffers.add(newBuf); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index 83e470ae25..daec331831 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -95,7 +95,7 @@ public void init() { private static final byte[] STR5 = "EEE5".getBytes(utf8Charset); private static final byte[] STR6 = "FFFFF6".getBytes(utf8Charset); private static final int MAX_VALUE_COUNT = - (int) (Integer.getInteger("arrow.vector.max_allocation_bytes", Integer.MAX_VALUE) / 7); + (int) (Integer.getInteger("arrow.vector.max_allocation_bytes", Integer.MAX_VALUE) / 9); private static final int MAX_VALUE_COUNT_8BYTE = (int) (MAX_VALUE_COUNT / 2); @AfterEach diff --git a/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java index f5ec42c71c..bc47150376 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java @@ -24,6 +24,7 @@ import java.nio.charset.StandardCharsets; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.memory.util.CommonUtil; import org.apache.arrow.vector.complex.DenseUnionVector; import org.apache.arrow.vector.complex.FixedSizeListVector; import org.apache.arrow.vector.complex.ListVector; @@ -222,6 +223,17 @@ public void testVariableAllocateAfterReAlloc() throws Exception { } } + @Test + public void testVariableReAllocAbove1GB() throws Exception { + try (final VarCharVector vector = new VarCharVector("", allocator)) { + long desiredSizeAboveLastPowerOf2 = + CommonUtil.nextPowerOfTwo(BaseVariableWidthVector.MAX_ALLOCATION_SIZE) / 2 + 1; + vector.reallocDataBuffer(desiredSizeAboveLastPowerOf2); + + assertTrue(vector.getDataBuffer().capacity() >= desiredSizeAboveLastPowerOf2); + } + } + @Test public void testLargeVariableAllocateAfterReAlloc() throws Exception { try (final LargeVarCharVector vector = new LargeVarCharVector("", allocator)) { diff --git a/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java b/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java index e1b3889d85..4ee9630a4d 100644 --- a/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java +++ b/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java @@ -28,6 +28,7 @@ import java.util.stream.Stream; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.memory.util.CommonUtil; import org.apache.arrow.vector.BaseValueVector; import org.apache.arrow.vector.BaseVariableWidthViewVector; import org.apache.arrow.vector.BigIntVector; @@ -309,7 +310,15 @@ public void testAppendEmptyVariableWidthVector() { @Test public void testAppendLargeAndSmallVariableVectorsWithinLimit() { - int sixteenthOfMaxAllocation = Math.toIntExact(BaseValueVector.MAX_ALLOCATION_SIZE / 16); + // Using the max power of 2 allocation size to avoid hitting the max limit at round ups + long maxPowerOfTwoAllocationSize = + CommonUtil.nextPowerOfTwo(BaseValueVector.MAX_ALLOCATION_SIZE); + if (maxPowerOfTwoAllocationSize > BaseValueVector.MAX_ALLOCATION_SIZE) { + maxPowerOfTwoAllocationSize = + CommonUtil.nextPowerOfTwo(BaseValueVector.MAX_ALLOCATION_SIZE / 2); + } + + int sixteenthOfMaxAllocation = Math.toIntExact(maxPowerOfTwoAllocationSize / 16); try (VarCharVector target = makeVarCharVec(1, sixteenthOfMaxAllocation); VarCharVector delta = makeVarCharVec(sixteenthOfMaxAllocation, 1)) { new VectorAppender(delta).visit(target, null); From d2465c3fec94ff25a966490827b57b5eafb1b637 Mon Sep 17 00:00:00 2001 From: Martin Traverse Date: Wed, 23 Apr 2025 02:19:24 +0100 Subject: [PATCH 013/271] GH-698: Improve and fix Avro read consumers (#718) ## What's Changed This PR relates to #698 and is the second in a series intended to provide full Avro read / write support in native Java. It adds round-trip tests for both schemas (Arrow schema -> Avro -> Arrow) and data (Arrow VSR -> Avro block -> Arrow VSR). It also adds a number of fixes and improvements to the Avro Consumers so that data arrives back in its original form after a round trip. The main changes are: * Added a top level method in AvroToArrow to convert Avro schema directly to Arrow schema (this may exist elsewhere, but is needed to provide an API that matches the logic of this implementation) * Avro unions of [ type, null ] or [ null, type ] now have special handling, these are interpreted as a single nullable type rather than a union. Setting legacyMode = false in the AvroToArrowConfig object is required to enable this behaviour, otherwise unions are interpreted literally. Unions with more than 2 elements are always interpreted literally (but, per #108, in practice Java's current Union implementation is probably not usable with Avro atm). * Added support for new logical types (decimal 256, timestamp nano and 3 local timestamp types) * Existing timestamp-mills and timestamp-micros times now interpreted as zone-aware (previously they were interpreted as local, but now the local timestamp types are interpreted as local - I think this is correct per the [Avro spec](https://avro.apache.org/docs/1.12.0/specification/#timestamps)). Requires setting legacyMode = false. * Removed namespaces from generated Arrow field names in complex types. E.g. the Avro field myNamepsace.outerRecord.structField.intField should be called just "intField" inside the Arrow struct. This doesn't affect the skip field logic, which still works using the qualified names. This requires setting legacyMode = false. * Remove unexpected metadata in generated Arrow fields (empty alias lists and attributes interpreted as part of the field schema). This requires setting legacyMode = false. * Use the expected child vector names for Arrow LIST and MAP types when reading. For LIST, the default child vector is called "$data$" which is illegal in Avro, so the child field name is also changed to "item" in the producers. This requires setting legacyMode = false. Breaking changes have been removed from this PR. Per discussion below, all breaking changes are now behind a "legacyMode" flag in the AvroToArrowConfig object, which is enabled by default in all the original code paths. Closes #698 . This change is meant to allow for round trip of schemas and individual Avro data blocks (one Avro data block -> one VSR). File-level capabilities are not included. I have not included anything to recycle the VSR as part of the read API, this feels like it belongs with the file-level piece. Also I have not done anything specific for enums / dict encoding as of yet. --- .../arrow/adapter/avro/ArrowToAvroUtils.java | 12 +- .../arrow/adapter/avro/AvroToArrow.java | 19 + .../arrow/adapter/avro/AvroToArrowConfig.java | 41 + .../arrow/adapter/avro/AvroToArrowUtils.java | 414 ++++- .../avro/consumers/AvroNullableConsumer.java | 82 + .../logical/AvroDecimal256Consumer.java | 74 + .../logical/AvroTimestampMicrosConsumer.java | 2 +- .../AvroTimestampMicrosTzConsumer.java | 39 + .../logical/AvroTimestampMillisConsumer.java | 2 +- .../AvroTimestampMillisTzConsumer.java | 39 + .../logical/AvroTimestampNanosConsumer.java | 39 + .../logical/AvroTimestampNanosTzConsumer.java | 39 + .../avro/producers/AvroNullableProducer.java | 4 +- .../adapter/avro/ArrowToAvroSchemaTest.java | 24 +- .../adapter/avro/AvroLogicalTypesTest.java | 2 +- .../arrow/adapter/avro/RoundTripDataTest.java | 1606 +++++++++++++++++ .../adapter/avro/RoundTripSchemaTest.java | 443 +++++ .../schema/logical/test_decimal_invalid1.avsc | 2 +- .../logical/test_local_timestamp_micros.avsc | 23 + .../logical/test_local_timestamp_millis.avsc | 23 + .../logical/test_local_timestamp_nanos.avsc | 23 + .../schema/logical/test_timestamp_nanos.avsc | 23 + 22 files changed, 2870 insertions(+), 105 deletions(-) create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroNullableConsumer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDecimal256Consumer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMicrosTzConsumer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMillisTzConsumer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampNanosConsumer.java create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampNanosTzConsumer.java create mode 100644 adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripDataTest.java create mode 100644 adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripSchemaTest.java create mode 100644 adapter/avro/src/test/resources/schema/logical/test_local_timestamp_micros.avsc create mode 100644 adapter/avro/src/test/resources/schema/logical/test_local_timestamp_millis.avsc create mode 100644 adapter/avro/src/test/resources/schema/logical/test_local_timestamp_nanos.avsc create mode 100644 adapter/avro/src/test/resources/schema/logical/test_timestamp_nanos.avsc diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/ArrowToAvroUtils.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/ArrowToAvroUtils.java index 6f0cb5cffc..87b594af9e 100644 --- a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/ArrowToAvroUtils.java +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/ArrowToAvroUtils.java @@ -332,7 +332,17 @@ private static T buildBaseTypeSchema( case List: case FixedSizeList: - return buildArraySchema(builder.array(), field, namespace); + // Arrow uses "$data$" as the field name for list items, that is not a valid Avro name + Field itemField = field.getChildren().get(0); + if (ListVector.DATA_VECTOR_NAME.equals(itemField.getName())) { + Field safeItemField = + new Field("item", itemField.getFieldType(), itemField.getChildren()); + Field safeListField = + new Field(field.getName(), field.getFieldType(), List.of(safeItemField)); + return buildArraySchema(builder.array(), safeListField, namespace); + } else { + return buildArraySchema(builder.array(), field, namespace); + } case Map: return buildMapSchema(builder.map(), field, namespace); diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrow.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrow.java index 2392c36f94..2a28ad393b 100644 --- a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrow.java +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrow.java @@ -59,4 +59,23 @@ public static AvroToArrowVectorIterator avroToArrowIterator( return AvroToArrowVectorIterator.create(decoder, schema, config); } + + /** + * Convert an Avro schema to its Arrow equivalent. + * + *

The resulting set of Arrow fields matches what would be set in the VSR after calling + * avroToArrow() or avroToArrowIterator(), respecting the configuration in the config parameter. + * + * @param schema The Avro schema to convert + * @param config Configuration options for conversion + * @return The equivalent Arrow schema + */ + public static org.apache.arrow.vector.types.pojo.Schema avroToAvroSchema( + Schema schema, AvroToArrowConfig config) { + + Preconditions.checkNotNull(schema, "Avro schema object cannot be null"); + Preconditions.checkNotNull(config, "config cannot be null"); + + return AvroToArrowUtils.createArrowSchema(schema, config); + } } diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowConfig.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowConfig.java index bd70c2b8ba..5596138586 100644 --- a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowConfig.java +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowConfig.java @@ -41,6 +41,12 @@ public class AvroToArrowConfig { /** The field names which to skip when reading decoder values. */ private final Set skipFieldNames; + /** + * Use legacy-mode to keep compatibility with old behavior (pre-2025), enabled by default. This + * affects how the AvroToArrow code interprets the Avro schema. + */ + private final boolean legacyMode; + /** * Instantiate an instance. * @@ -64,6 +70,37 @@ public class AvroToArrowConfig { this.targetBatchSize = targetBatchSize; this.provider = provider; this.skipFieldNames = skipFieldNames; + + // Default values for optional parameters + legacyMode = true; // Keep compatibility with old behavior by default + } + + /** + * Instantiate an instance. + * + * @param allocator The memory allocator to construct the Arrow vectors with. + * @param targetBatchSize The maximum rowCount to read each time when partially convert data. + * @param provider The dictionary provider used for enum type, adapter will update this provider. + * @param skipFieldNames Field names which to skip. + * @param legacyMode Keep compatibility with old behavior (pre-2025) + */ + AvroToArrowConfig( + BufferAllocator allocator, + int targetBatchSize, + DictionaryProvider.MapDictionaryProvider provider, + Set skipFieldNames, + boolean legacyMode) { + + Preconditions.checkArgument( + targetBatchSize == AvroToArrowVectorIterator.NO_LIMIT_BATCH_SIZE || targetBatchSize > 0, + "invalid targetBatchSize: %s", + targetBatchSize); + + this.allocator = allocator; + this.targetBatchSize = targetBatchSize; + this.provider = provider; + this.skipFieldNames = skipFieldNames; + this.legacyMode = legacyMode; } public BufferAllocator getAllocator() { @@ -81,4 +118,8 @@ public DictionaryProvider.MapDictionaryProvider getProvider() { public Set getSkipFieldNames() { return skipFieldNames; } + + public boolean isLegacyMode() { + return legacyMode; + } } diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java index ed7642aabd..aedef7732e 100644 --- a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java @@ -41,6 +41,7 @@ import org.apache.arrow.adapter.avro.consumers.AvroLongConsumer; import org.apache.arrow.adapter.avro.consumers.AvroMapConsumer; import org.apache.arrow.adapter.avro.consumers.AvroNullConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroNullableConsumer; import org.apache.arrow.adapter.avro.consumers.AvroStringConsumer; import org.apache.arrow.adapter.avro.consumers.AvroStructConsumer; import org.apache.arrow.adapter.avro.consumers.AvroUnionsConsumer; @@ -49,17 +50,23 @@ import org.apache.arrow.adapter.avro.consumers.SkipConsumer; import org.apache.arrow.adapter.avro.consumers.SkipFunction; import org.apache.arrow.adapter.avro.consumers.logical.AvroDateConsumer; +import org.apache.arrow.adapter.avro.consumers.logical.AvroDecimal256Consumer; import org.apache.arrow.adapter.avro.consumers.logical.AvroDecimalConsumer; import org.apache.arrow.adapter.avro.consumers.logical.AvroTimeMicroConsumer; import org.apache.arrow.adapter.avro.consumers.logical.AvroTimeMillisConsumer; import org.apache.arrow.adapter.avro.consumers.logical.AvroTimestampMicrosConsumer; +import org.apache.arrow.adapter.avro.consumers.logical.AvroTimestampMicrosTzConsumer; import org.apache.arrow.adapter.avro.consumers.logical.AvroTimestampMillisConsumer; +import org.apache.arrow.adapter.avro.consumers.logical.AvroTimestampMillisTzConsumer; +import org.apache.arrow.adapter.avro.consumers.logical.AvroTimestampNanosConsumer; +import org.apache.arrow.adapter.avro.consumers.logical.AvroTimestampNanosTzConsumer; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.BaseIntVector; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.DateDayVector; +import org.apache.arrow.vector.Decimal256Vector; import org.apache.arrow.vector.DecimalVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.FixedSizeBinaryVector; @@ -69,8 +76,12 @@ import org.apache.arrow.vector.NullVector; import org.apache.arrow.vector.TimeMicroVector; import org.apache.arrow.vector.TimeMilliVector; +import org.apache.arrow.vector.TimeStampMicroTZVector; import org.apache.arrow.vector.TimeStampMicroVector; +import org.apache.arrow.vector.TimeStampMilliTZVector; import org.apache.arrow.vector.TimeStampMilliVector; +import org.apache.arrow.vector.TimeStampNanoTZVector; +import org.apache.arrow.vector.TimeStampNanoVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; @@ -169,42 +180,69 @@ private static Consumer createConsumer( switch (type) { case UNION: - consumer = createUnionConsumer(schema, name, config, consumerVector); + boolean nullableUnion = + schema.getTypes().stream().anyMatch(t -> t.getType() == Schema.Type.NULL); + if (schema.getTypes().size() == 2 && nullableUnion && !config.isLegacyMode()) { + // For a simple nullable (null | type), interpret the union as a single nullable field. + // Not available in legacy mode, which uses the literal interpretation instead + int nullIndex = schema.getTypes().get(0).getType() == Schema.Type.NULL ? 0 : 1; + int childIndex = nullIndex == 0 ? 1 : 0; + Schema childSchema = schema.getTypes().get(childIndex); + Consumer childConsumer = + createConsumer(childSchema, name, true, config, consumerVector); + consumer = new AvroNullableConsumer<>(childConsumer, nullIndex); + } else { + // Literal interpretation of a union, which may or may not include a null element. + consumer = createUnionConsumer(schema, name, nullableUnion, config, consumerVector); + } break; case ARRAY: - consumer = createArrayConsumer(schema, name, config, consumerVector); + consumer = createArrayConsumer(schema, name, nullable, config, consumerVector); break; case MAP: - consumer = createMapConsumer(schema, name, config, consumerVector); + consumer = createMapConsumer(schema, name, nullable, config, consumerVector); break; case RECORD: - consumer = createStructConsumer(schema, name, config, consumerVector); + consumer = createStructConsumer(schema, name, nullable, config, consumerVector); break; case ENUM: - consumer = createEnumConsumer(schema, name, config, consumerVector); + consumer = createEnumConsumer(schema, name, nullable, config, consumerVector); break; case STRING: arrowType = new ArrowType.Utf8(); - fieldType = new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); + fieldType = + new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroStringConsumer((VarCharVector) vector); break; case FIXED: - Map extProps = createExternalProps(schema); + Map extProps = createExternalProps(schema, config); if (logicalType instanceof LogicalTypes.Decimal) { - arrowType = createDecimalArrowType((LogicalTypes.Decimal) logicalType); + arrowType = createDecimalArrowType((LogicalTypes.Decimal) logicalType, schema); fieldType = new FieldType( - nullable, arrowType, /* dictionary= */ null, getMetaData(schema, extProps)); + nullable, + arrowType, + /* dictionary= */ null, + getMetaData(schema, extProps, config)); vector = createVector(consumerVector, fieldType, name, allocator); - consumer = - new AvroDecimalConsumer.FixedDecimalConsumer( - (DecimalVector) vector, schema.getFixedSize()); + if (schema.getFixedSize() <= 16) { + consumer = + new AvroDecimalConsumer.FixedDecimalConsumer( + (DecimalVector) vector, schema.getFixedSize()); + } else { + consumer = + new AvroDecimal256Consumer.FixedDecimal256Consumer( + (Decimal256Vector) vector, schema.getFixedSize()); + } } else { arrowType = new ArrowType.FixedSizeBinary(schema.getFixedSize()); fieldType = new FieldType( - nullable, arrowType, /* dictionary= */ null, getMetaData(schema, extProps)); + nullable, + arrowType, + /* dictionary= */ null, + getMetaData(schema, extProps, config)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroFixedConsumer((FixedSizeBinaryVector) vector, schema.getFixedSize()); } @@ -213,26 +251,30 @@ private static Consumer createConsumer( if (logicalType instanceof LogicalTypes.Date) { arrowType = new ArrowType.Date(DateUnit.DAY); fieldType = - new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); + new FieldType( + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroDateConsumer((DateDayVector) vector); } else if (logicalType instanceof LogicalTypes.TimeMillis) { arrowType = new ArrowType.Time(TimeUnit.MILLISECOND, 32); fieldType = - new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); + new FieldType( + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroTimeMillisConsumer((TimeMilliVector) vector); } else { arrowType = new ArrowType.Int(32, /* isSigned= */ true); fieldType = - new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); + new FieldType( + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroIntConsumer((IntVector) vector); } break; case BOOLEAN: arrowType = new ArrowType.Bool(); - fieldType = new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); + fieldType = + new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroBooleanConsumer((BitVector) vector); break; @@ -240,60 +282,109 @@ private static Consumer createConsumer( if (logicalType instanceof LogicalTypes.TimeMicros) { arrowType = new ArrowType.Time(TimeUnit.MICROSECOND, 64); fieldType = - new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); + new FieldType( + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroTimeMicroConsumer((TimeMicroVector) vector); - } else if (logicalType instanceof LogicalTypes.TimestampMillis) { + } else if (logicalType instanceof LogicalTypes.TimestampMillis && !config.isLegacyMode()) { + // In legacy mode the timestamp-xxx types are treated as local, there is no zone aware + // type + arrowType = new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC"); + fieldType = + new FieldType( + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); + vector = createVector(consumerVector, fieldType, name, allocator); + consumer = new AvroTimestampMillisTzConsumer((TimeStampMilliTZVector) vector); + } else if (logicalType instanceof LogicalTypes.TimestampMicros && !config.isLegacyMode()) { + arrowType = new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC"); + fieldType = + new FieldType( + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); + vector = createVector(consumerVector, fieldType, name, allocator); + consumer = new AvroTimestampMicrosTzConsumer((TimeStampMicroTZVector) vector); + } else if (logicalType instanceof LogicalTypes.TimestampNanos && !config.isLegacyMode()) { + arrowType = new ArrowType.Timestamp(TimeUnit.NANOSECOND, "UTC"); + fieldType = + new FieldType( + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); + vector = createVector(consumerVector, fieldType, name, allocator); + consumer = new AvroTimestampNanosTzConsumer((TimeStampNanoTZVector) vector); + } else if (logicalType instanceof LogicalTypes.LocalTimestampMillis + || (logicalType instanceof LogicalTypes.TimestampMillis && config.isLegacyMode())) { arrowType = new ArrowType.Timestamp(TimeUnit.MILLISECOND, null); fieldType = - new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); + new FieldType( + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroTimestampMillisConsumer((TimeStampMilliVector) vector); - } else if (logicalType instanceof LogicalTypes.TimestampMicros) { + } else if (logicalType instanceof LogicalTypes.LocalTimestampMicros + || (logicalType instanceof LogicalTypes.TimestampMicros && config.isLegacyMode())) { + // In legacy mode the timestamp-xxx types are treated as local arrowType = new ArrowType.Timestamp(TimeUnit.MICROSECOND, null); fieldType = - new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); + new FieldType( + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroTimestampMicrosConsumer((TimeStampMicroVector) vector); + } else if (logicalType instanceof LogicalTypes.LocalTimestampNanos + || (logicalType instanceof LogicalTypes.TimestampNanos && config.isLegacyMode())) { + arrowType = new ArrowType.Timestamp(TimeUnit.NANOSECOND, null); + fieldType = + new FieldType( + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); + vector = createVector(consumerVector, fieldType, name, allocator); + consumer = new AvroTimestampNanosConsumer((TimeStampNanoVector) vector); } else { arrowType = new ArrowType.Int(64, /* isSigned= */ true); fieldType = - new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); + new FieldType( + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroLongConsumer((BigIntVector) vector); } break; case FLOAT: arrowType = new ArrowType.FloatingPoint(SINGLE); - fieldType = new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); + fieldType = + new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroFloatConsumer((Float4Vector) vector); break; case DOUBLE: arrowType = new ArrowType.FloatingPoint(DOUBLE); - fieldType = new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); + fieldType = + new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroDoubleConsumer((Float8Vector) vector); break; case BYTES: if (logicalType instanceof LogicalTypes.Decimal) { - arrowType = createDecimalArrowType((LogicalTypes.Decimal) logicalType); + LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType; + arrowType = createDecimalArrowType(decimalType, schema); fieldType = - new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); + new FieldType( + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); vector = createVector(consumerVector, fieldType, name, allocator); - consumer = new AvroDecimalConsumer.BytesDecimalConsumer((DecimalVector) vector); + if (decimalType.getPrecision() <= 38) { + consumer = new AvroDecimalConsumer.BytesDecimalConsumer((DecimalVector) vector); + } else { + consumer = + new AvroDecimal256Consumer.BytesDecimal256Consumer((Decimal256Vector) vector); + } } else { arrowType = new ArrowType.Binary(); fieldType = - new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); + new FieldType( + nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroBytesConsumer((VarBinaryVector) vector); } break; case NULL: arrowType = new ArrowType.Null(); - fieldType = new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema)); - vector = fieldType.createNewSingleVector(name, allocator, /* schemaCallBack= */ null); + fieldType = + new FieldType(nullable, arrowType, /* dictionary= */ null, getMetaData(schema, config)); + vector = new NullVector(name, fieldType); // Respect nullability defined in fieldType consumer = new AvroNullConsumer((NullVector) vector); break; default: @@ -304,19 +395,31 @@ private static Consumer createConsumer( return consumer; } - private static ArrowType createDecimalArrowType(LogicalTypes.Decimal logicalType) { + private static ArrowType createDecimalArrowType(LogicalTypes.Decimal logicalType, Schema schema) { final int scale = logicalType.getScale(); final int precision = logicalType.getPrecision(); Preconditions.checkArgument( - precision > 0 && precision <= 38, "Precision must be in range of 1 to 38"); - Preconditions.checkArgument(scale >= 0 && scale <= 38, "Scale must be in range of 0 to 38."); + precision > 0 && precision <= 76, "Precision must be in range of 1 to 76"); + Preconditions.checkArgument(scale >= 0 && scale <= 76, "Scale must be in range of 0 to 76."); Preconditions.checkArgument( scale <= precision, "Invalid decimal scale: %s (greater than precision: %s)", scale, precision); - return new ArrowType.Decimal(precision, scale, 128); + if (schema.getType() == Schema.Type.FIXED) { + if (schema.getFixedSize() <= 16) { + return new ArrowType.Decimal(precision, scale, 128); + } else { + return new ArrowType.Decimal(precision, scale, 256); + } + } else { + if (precision <= 38) { + return new ArrowType.Decimal(precision, scale, 128); + } else { + return new ArrowType.Decimal(precision, scale, 256); + } + } } private static Consumer createSkipConsumer(Schema schema) { @@ -406,6 +509,30 @@ private static Consumer createSkipConsumer(Schema schema) { return new SkipConsumer(skipFunction); } + static org.apache.arrow.vector.types.pojo.Schema createArrowSchema( + Schema schema, AvroToArrowConfig config) { + + // Create an Arrow schema matching the structure of vectors built by createCompositeConsumer() + + Set skipFieldNames = config.getSkipFieldNames(); + List arrowFields = new ArrayList<>(schema.getFields().size()); + + Schema.Type type = schema.getType(); + if (type == Schema.Type.RECORD) { + for (Schema.Field field : schema.getFields()) { + if (!skipFieldNames.contains(field.name())) { + Field arrowField = avroSchemaToField(field.schema(), field.name(), config); + arrowFields.add(arrowField); + } + } + } else { + Field arrowField = avroSchemaToField(schema, schema.getName(), config); + arrowFields.add(arrowField); + } + + return new org.apache.arrow.vector.types.pojo.Schema(arrowFields); + } + static CompositeAvroConsumer createCompositeConsumer(Schema schema, AvroToArrowConfig config) { List consumers = new ArrayList<>(); @@ -442,11 +569,20 @@ private static String getDefaultFieldName(ArrowType type) { } private static Field avroSchemaToField(Schema schema, String name, AvroToArrowConfig config) { - return avroSchemaToField(schema, name, config, null); + return avroSchemaToField(schema, name, false, config, null); } private static Field avroSchemaToField( Schema schema, String name, AvroToArrowConfig config, Map externalProps) { + return avroSchemaToField(schema, name, false, config, externalProps); + } + + private static Field avroSchemaToField( + Schema schema, + String name, + boolean nullable, + AvroToArrowConfig config, + Map externalProps) { final Schema.Type type = schema.getType(); final LogicalType logicalType = schema.getLogicalType(); @@ -455,33 +591,53 @@ private static Field avroSchemaToField( switch (type) { case UNION: - for (int i = 0; i < schema.getTypes().size(); i++) { - Schema childSchema = schema.getTypes().get(i); - // Union child vector should use default name - children.add(avroSchemaToField(childSchema, null, config)); + boolean nullableUnion = + schema.getTypes().stream().anyMatch(t -> t.getType() == Schema.Type.NULL); + if (nullableUnion && schema.getTypes().size() == 2 && !config.isLegacyMode()) { + // For a simple nullable (null | type), interpret the union as a single nullable field. + // Not available in legacy mode, which uses the literal interpretation instead + Schema childSchema = + schema.getTypes().get(0).getType() == Schema.Type.NULL + ? schema.getTypes().get(1) + : schema.getTypes().get(0); + return avroSchemaToField(childSchema, name, true, config, externalProps); + } else { + // Literal interpretation of a union, which may or may not include a null element. + for (int i = 0; i < schema.getTypes().size(); i++) { + Schema childSchema = schema.getTypes().get(i); + // Union child vector should use default name + children.add(avroSchemaToField(childSchema, null, nullableUnion, config, null)); + } + fieldType = + createFieldType( + new ArrowType.Union(UnionMode.Sparse, null), schema, externalProps, config); } - fieldType = - createFieldType(new ArrowType.Union(UnionMode.Sparse, null), schema, externalProps); break; case ARRAY: Schema elementSchema = schema.getElementType(); - children.add(avroSchemaToField(elementSchema, elementSchema.getName(), config)); - fieldType = createFieldType(new ArrowType.List(), schema, externalProps); + children.add(avroSchemaToField(elementSchema, ListVector.DATA_VECTOR_NAME, config)); + fieldType = createFieldType(nullable, new ArrowType.List(), schema, externalProps, config); break; case MAP: // MapVector internal struct field and key field should be non-nullable FieldType keyFieldType = new FieldType(/* nullable= */ false, new ArrowType.Utf8(), /* dictionary= */ null); - Field keyField = new Field("key", keyFieldType, /* children= */ null); - Field valueField = avroSchemaToField(schema.getValueType(), "value", config); + Field keyField = new Field(MapVector.KEY_NAME, keyFieldType, /* children= */ null); + Field valueField = avroSchemaToField(schema.getValueType(), MapVector.VALUE_NAME, config); FieldType structFieldType = new FieldType(false, new ArrowType.Struct(), /* dictionary= */ null); Field structField = - new Field("internal", structFieldType, Arrays.asList(keyField, valueField)); + new Field( + MapVector.DATA_VECTOR_NAME, structFieldType, Arrays.asList(keyField, valueField)); children.add(structField); fieldType = - createFieldType(new ArrowType.Map(/* keysSorted= */ false), schema, externalProps); + createFieldType( + nullable, + new ArrowType.Map(/* keysSorted= */ false), + schema, + externalProps, + config); break; case RECORD: final Set skipFieldNames = config.getSkipFieldNames(); @@ -496,13 +652,14 @@ private static Field avroSchemaToField( if (doc != null) { extProps.put("doc", doc); } - if (aliases != null) { + if (aliases != null && (!aliases.isEmpty() || config.isLegacyMode())) { extProps.put("aliases", convertAliases(aliases)); } children.add(avroSchemaToField(childSchema, fullChildName, config, extProps)); } } - fieldType = createFieldType(new ArrowType.Struct(), schema, externalProps); + fieldType = + createFieldType(nullable, new ArrowType.Struct(), schema, externalProps, config); break; case ENUM: DictionaryProvider.MapDictionaryProvider provider = config.getProvider(); @@ -512,23 +669,25 @@ private static Field avroSchemaToField( fieldType = createFieldType( + nullable, indexType, schema, externalProps, - new DictionaryEncoding(current, /* ordered= */ false, /* indexType= */ indexType)); + new DictionaryEncoding(current, /* ordered= */ false, /* indexType= */ indexType), + config); break; case STRING: - fieldType = createFieldType(new ArrowType.Utf8(), schema, externalProps); + fieldType = createFieldType(nullable, new ArrowType.Utf8(), schema, externalProps, config); break; case FIXED: final ArrowType fixedArrowType; if (logicalType instanceof LogicalTypes.Decimal) { - fixedArrowType = createDecimalArrowType((LogicalTypes.Decimal) logicalType); + fixedArrowType = createDecimalArrowType((LogicalTypes.Decimal) logicalType, schema); } else { fixedArrowType = new ArrowType.FixedSizeBinary(schema.getFixedSize()); } - fieldType = createFieldType(fixedArrowType, schema, externalProps); + fieldType = createFieldType(nullable, fixedArrowType, schema, externalProps, config); break; case INT: final ArrowType intArrowType; @@ -539,41 +698,62 @@ private static Field avroSchemaToField( } else { intArrowType = new ArrowType.Int(32, /* isSigned= */ true); } - fieldType = createFieldType(intArrowType, schema, externalProps); + fieldType = createFieldType(nullable, intArrowType, schema, externalProps, config); break; case BOOLEAN: - fieldType = createFieldType(new ArrowType.Bool(), schema, externalProps); + fieldType = createFieldType(nullable, new ArrowType.Bool(), schema, externalProps, config); break; case LONG: final ArrowType longArrowType; if (logicalType instanceof LogicalTypes.TimeMicros) { longArrowType = new ArrowType.Time(TimeUnit.MICROSECOND, 64); } else if (logicalType instanceof LogicalTypes.TimestampMillis) { - longArrowType = new ArrowType.Timestamp(TimeUnit.MILLISECOND, null); + // In legacy mode the timestamp-xxx types are treated as local + String tz = config.isLegacyMode() ? null : "UTC"; + longArrowType = new ArrowType.Timestamp(TimeUnit.MILLISECOND, tz); } else if (logicalType instanceof LogicalTypes.TimestampMicros) { + String tz = config.isLegacyMode() ? null : "UTC"; + longArrowType = new ArrowType.Timestamp(TimeUnit.MICROSECOND, tz); + } else if (logicalType instanceof LogicalTypes.TimestampNanos) { + String tz = config.isLegacyMode() ? null : "UTC"; + longArrowType = new ArrowType.Timestamp(TimeUnit.NANOSECOND, tz); + } else if (logicalType instanceof LogicalTypes.LocalTimestampMillis + && !config.isLegacyMode()) { + // In legacy mode the local-timestamp-xxx types are not recognized (result is just type = + // long) + longArrowType = new ArrowType.Timestamp(TimeUnit.MILLISECOND, null); + } else if (logicalType instanceof LogicalTypes.LocalTimestampMicros + && !config.isLegacyMode()) { longArrowType = new ArrowType.Timestamp(TimeUnit.MICROSECOND, null); + } else if (logicalType instanceof LogicalTypes.LocalTimestampNanos + && !config.isLegacyMode()) { + longArrowType = new ArrowType.Timestamp(TimeUnit.NANOSECOND, null); } else { longArrowType = new ArrowType.Int(64, /* isSigned= */ true); } - fieldType = createFieldType(longArrowType, schema, externalProps); + fieldType = createFieldType(nullable, longArrowType, schema, externalProps, config); break; case FLOAT: - fieldType = createFieldType(new ArrowType.FloatingPoint(SINGLE), schema, externalProps); + fieldType = + createFieldType( + nullable, new ArrowType.FloatingPoint(SINGLE), schema, externalProps, config); break; case DOUBLE: - fieldType = createFieldType(new ArrowType.FloatingPoint(DOUBLE), schema, externalProps); + fieldType = + createFieldType( + nullable, new ArrowType.FloatingPoint(DOUBLE), schema, externalProps, config); break; case BYTES: final ArrowType bytesArrowType; if (logicalType instanceof LogicalTypes.Decimal) { - bytesArrowType = createDecimalArrowType((LogicalTypes.Decimal) logicalType); + bytesArrowType = createDecimalArrowType((LogicalTypes.Decimal) logicalType, schema); } else { bytesArrowType = new ArrowType.Binary(); } - fieldType = createFieldType(bytesArrowType, schema, externalProps); + fieldType = createFieldType(nullable, bytesArrowType, schema, externalProps, config); break; case NULL: - fieldType = createFieldType(ArrowType.Null.INSTANCE, schema, externalProps); + fieldType = createFieldType(ArrowType.Null.INSTANCE, schema, externalProps, config); break; default: // no-op, shouldn't get here @@ -583,15 +763,24 @@ private static Field avroSchemaToField( if (name == null) { name = getDefaultFieldName(fieldType.getType()); } + if (name.contains(".") && !config.isLegacyMode()) { + // Do not include namespace as part of the field name + name = name.substring(name.lastIndexOf(".") + 1); + } return new Field(name, fieldType, children.size() == 0 ? null : children); } private static Consumer createArrayConsumer( - Schema schema, String name, AvroToArrowConfig config, FieldVector consumerVector) { + Schema schema, + String name, + boolean nullable, + AvroToArrowConfig config, + FieldVector consumerVector) { ListVector listVector; if (consumerVector == null) { - final Field field = avroSchemaToField(schema, name, config); + final Field field = + avroSchemaToField(schema, name, nullable, config, /* externalProps= */ null); listVector = (ListVector) field.createVector(config.getAllocator()); } else { listVector = (ListVector) consumerVector; @@ -607,13 +796,18 @@ private static Consumer createArrayConsumer( } private static Consumer createStructConsumer( - Schema schema, String name, AvroToArrowConfig config, FieldVector consumerVector) { + Schema schema, + String name, + boolean nullable, + AvroToArrowConfig config, + FieldVector consumerVector) { final Set skipFieldNames = config.getSkipFieldNames(); StructVector structVector; if (consumerVector == null) { - final Field field = avroSchemaToField(schema, name, config, createExternalProps(schema)); + final Field field = + avroSchemaToField(schema, name, nullable, config, createExternalProps(schema, config)); structVector = (StructVector) field.createVector(config.getAllocator()); } else { structVector = (StructVector) consumerVector; @@ -644,11 +838,16 @@ private static Consumer createStructConsumer( } private static Consumer createEnumConsumer( - Schema schema, String name, AvroToArrowConfig config, FieldVector consumerVector) { + Schema schema, + String name, + boolean nullable, + AvroToArrowConfig config, + FieldVector consumerVector) { BaseIntVector indexVector; if (consumerVector == null) { - final Field field = avroSchemaToField(schema, name, config, createExternalProps(schema)); + final Field field = + avroSchemaToField(schema, name, nullable, config, createExternalProps(schema, config)); indexVector = (BaseIntVector) field.createVector(config.getAllocator()); } else { indexVector = (BaseIntVector) consumerVector; @@ -668,11 +867,16 @@ private static Consumer createEnumConsumer( } private static Consumer createMapConsumer( - Schema schema, String name, AvroToArrowConfig config, FieldVector consumerVector) { + Schema schema, + String name, + boolean nullable, + AvroToArrowConfig config, + FieldVector consumerVector) { MapVector mapVector; if (consumerVector == null) { - final Field field = avroSchemaToField(schema, name, config); + final Field field = + avroSchemaToField(schema, name, nullable, config, /* externalProps= */ null); mapVector = (MapVector) field.createVector(config.getAllocator()); } else { mapVector = (MapVector) consumerVector; @@ -698,12 +902,13 @@ private static Consumer createMapConsumer( } private static Consumer createUnionConsumer( - Schema schema, String name, AvroToArrowConfig config, FieldVector consumerVector) { + Schema schema, + String name, + boolean nullableUnion, + AvroToArrowConfig config, + FieldVector consumerVector) { final int size = schema.getTypes().size(); - final boolean nullable = - schema.getTypes().stream().anyMatch(t -> t.getType() == Schema.Type.NULL); - UnionVector unionVector; if (consumerVector == null) { final Field field = avroSchemaToField(schema, name, config); @@ -720,7 +925,8 @@ private static Consumer createUnionConsumer( for (int i = 0; i < size; i++) { FieldVector child = childVectors.get(i); Schema subSchema = schema.getTypes().get(i); - Consumer delegate = createConsumer(subSchema, subSchema.getName(), nullable, config, child); + Consumer delegate = + createConsumer(subSchema, subSchema.getName(), nullableUnion, config, child); delegates[i] = delegate; types[i] = child.getMinorType(); } @@ -785,14 +991,24 @@ static VectorSchemaRoot avroToArrowVectors( return root; } - private static Map getMetaData(Schema schema) { + // Do not include props that are part of the Avro format itself as field metadata + // These are already represented in the field / type structure and are not custom attributes + private static final List AVRO_FORMAT_METADATA = + Arrays.asList("logicalType", "precision", "scale"); + + private static Map getMetaData(Schema schema, AvroToArrowConfig config) { Map metadata = new HashMap<>(); - schema.getObjectProps().forEach((k, v) -> metadata.put(k, v.toString())); + for (Map.Entry prop : schema.getObjectProps().entrySet()) { + if (!AVRO_FORMAT_METADATA.contains(prop.getKey()) || config.isLegacyMode()) { + metadata.put(prop.getKey(), prop.getValue().toString()); + } + } return metadata; } - private static Map getMetaData(Schema schema, Map externalProps) { - Map metadata = getMetaData(schema); + private static Map getMetaData( + Schema schema, Map externalProps, AvroToArrowConfig config) { + Map metadata = getMetaData(schema, config); if (externalProps != null) { metadata.putAll(externalProps); } @@ -800,32 +1016,58 @@ private static Map getMetaData(Schema schema, Map createExternalProps(Schema schema) { + private static Map createExternalProps(Schema schema, AvroToArrowConfig config) { final Map extProps = new HashMap<>(); String doc = schema.getDoc(); Set aliases = schema.getAliases(); if (doc != null) { extProps.put("doc", doc); } - if (aliases != null) { + if (aliases != null && (!aliases.isEmpty() || config.isLegacyMode())) { extProps.put("aliases", convertAliases(aliases)); } return extProps; } private static FieldType createFieldType( - ArrowType arrowType, Schema schema, Map externalProps) { - return createFieldType(arrowType, schema, externalProps, /* dictionary= */ null); + ArrowType arrowType, + Schema schema, + Map externalProps, + AvroToArrowConfig config) { + return createFieldType(arrowType, schema, externalProps, /* dictionary= */ null, config); + } + + private static FieldType createFieldType( + boolean nullable, + ArrowType arrowType, + Schema schema, + Map externalProps, + AvroToArrowConfig config) { + return createFieldType( + nullable, arrowType, schema, externalProps, /* dictionary= */ null, config); } private static FieldType createFieldType( ArrowType arrowType, Schema schema, Map externalProps, - DictionaryEncoding dictionary) { + DictionaryEncoding dictionary, + AvroToArrowConfig config) { + + return createFieldType( + /* nullable= */ false, arrowType, schema, externalProps, dictionary, config); + } + + private static FieldType createFieldType( + boolean nullable, + ArrowType arrowType, + Schema schema, + Map externalProps, + DictionaryEncoding dictionary, + AvroToArrowConfig config) { return new FieldType( - /* nullable= */ false, arrowType, dictionary, getMetaData(schema, externalProps)); + nullable, arrowType, dictionary, getMetaData(schema, externalProps, config)); } private static String convertAliases(Set aliases) { diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroNullableConsumer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroNullableConsumer.java new file mode 100644 index 0000000000..b67819cb9d --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroNullableConsumer.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.consumers; + +import java.io.IOException; +import org.apache.arrow.vector.FieldVector; +import org.apache.avro.io.Decoder; + +/** + * Consumer wrapper which consumes nullable type values from avro decoder. Write the data to the + * underlying {@link FieldVector}. + * + * @param The vector within consumer or its delegate. + */ +public class AvroNullableConsumer extends BaseAvroConsumer { + + private final Consumer delegate; + private final int nullIndex; + + /** Instantiate a AvroNullableConsumer. */ + @SuppressWarnings("unchecked") + public AvroNullableConsumer(Consumer delegate, int nullIndex) { + super((T) delegate.getVector()); + this.delegate = delegate; + this.nullIndex = nullIndex; + } + + @Override + public void consume(Decoder decoder) throws IOException { + int typeIndex = decoder.readInt(); + if (typeIndex == nullIndex) { + decoder.readNull(); + delegate.addNull(); + } else { + delegate.consume(decoder); + } + currentIndex++; + } + + @Override + public void addNull() { + // Can be called by containers of nullable types + delegate.addNull(); + currentIndex++; + } + + @Override + public void setPosition(int index) { + if (index < 0 || index > vector.getValueCount()) { + throw new IllegalArgumentException("Index out of bounds"); + } + delegate.setPosition(index); + super.setPosition(index); + } + + @Override + public boolean resetValueVector(T vector) { + boolean delegateOk = delegate.resetValueVector(vector); + boolean thisOk = super.resetValueVector(vector); + return thisOk && delegateOk; + } + + @Override + public void close() throws Exception { + super.close(); + delegate.close(); + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDecimal256Consumer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDecimal256Consumer.java new file mode 100644 index 0000000000..12652833a1 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDecimal256Consumer.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.consumers.logical; + +import java.io.IOException; +import java.nio.ByteBuffer; +import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; +import org.apache.arrow.util.Preconditions; +import org.apache.arrow.vector.Decimal256Vector; +import org.apache.avro.io.Decoder; + +/** + * Consumer which consume 256-bit decimal type values from avro decoder. Write the data to {@link + * Decimal256Vector}. + */ +public abstract class AvroDecimal256Consumer extends BaseAvroConsumer { + + protected AvroDecimal256Consumer(Decimal256Vector vector) { + super(vector); + } + + /** Consumer for decimal logical type with 256 bit width and original bytes type. */ + public static class BytesDecimal256Consumer extends AvroDecimal256Consumer { + + private ByteBuffer cacheBuffer; + + /** Instantiate a BytesDecimal256Consumer. */ + public BytesDecimal256Consumer(Decimal256Vector vector) { + super(vector); + } + + @Override + public void consume(Decoder decoder) throws IOException { + cacheBuffer = decoder.readBytes(cacheBuffer); + byte[] bytes = new byte[cacheBuffer.limit()]; + Preconditions.checkArgument(bytes.length <= 32, "Decimal bytes length should <= 32."); + cacheBuffer.get(bytes); + vector.setBigEndian(currentIndex++, bytes); + } + } + + /** Consumer for decimal logical type with 256 bit width and original fixed type. */ + public static class FixedDecimal256Consumer extends AvroDecimal256Consumer { + + private final byte[] reuseBytes; + + /** Instantiate a FixedDecimal256Consumer. */ + public FixedDecimal256Consumer(Decimal256Vector vector, int size) { + super(vector); + Preconditions.checkArgument(size <= 32, "Decimal bytes length should <= 32."); + reuseBytes = new byte[size]; + } + + @Override + public void consume(Decoder decoder) throws IOException { + decoder.readFixed(reuseBytes); + vector.setBigEndian(currentIndex++, reuseBytes); + } + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMicrosConsumer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMicrosConsumer.java index 88acf7b329..5af40ed17d 100644 --- a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMicrosConsumer.java +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMicrosConsumer.java @@ -22,7 +22,7 @@ import org.apache.avro.io.Decoder; /** - * Consumer which consume date timestamp-micro values from avro decoder. Write the data to {@link + * Consumer which consumes local-timestamp-micros values from avro decoder. Write the data to {@link * TimeStampMicroVector}. */ public class AvroTimestampMicrosConsumer extends BaseAvroConsumer { diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMicrosTzConsumer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMicrosTzConsumer.java new file mode 100644 index 0000000000..a5dede4988 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMicrosTzConsumer.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.consumers.logical; + +import java.io.IOException; +import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; +import org.apache.arrow.vector.TimeStampMicroTZVector; +import org.apache.avro.io.Decoder; + +/** + * Consumer which consumes timestamp-micros values from avro decoder. Write the data to {@link + * TimeStampMicroTZVector}. + */ +public class AvroTimestampMicrosTzConsumer extends BaseAvroConsumer { + + /** Instantiate a AvroTimestampMicrosTzConsumer. */ + public AvroTimestampMicrosTzConsumer(TimeStampMicroTZVector vector) { + super(vector); + } + + @Override + public void consume(Decoder decoder) throws IOException { + vector.set(currentIndex++, decoder.readLong()); + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMillisConsumer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMillisConsumer.java index ec50d79023..bc451bd1dc 100644 --- a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMillisConsumer.java +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMillisConsumer.java @@ -22,7 +22,7 @@ import org.apache.avro.io.Decoder; /** - * Consumer which consume date timestamp-millis values from avro decoder. Write the data to {@link + * Consumer which consume local-timestamp-millis values from avro decoder. Write the data to {@link * TimeStampMilliVector}. */ public class AvroTimestampMillisConsumer extends BaseAvroConsumer { diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMillisTzConsumer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMillisTzConsumer.java new file mode 100644 index 0000000000..255fe501fb --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMillisTzConsumer.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.consumers.logical; + +import java.io.IOException; +import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; +import org.apache.arrow.vector.TimeStampMilliTZVector; +import org.apache.avro.io.Decoder; + +/** + * Consumer which consume timestamp-millis values from avro decoder. Write the data to {@link + * TimeStampMilliTZVector}. + */ +public class AvroTimestampMillisTzConsumer extends BaseAvroConsumer { + + /** Instantiate a AvroTimestampMillisTzConsumer. */ + public AvroTimestampMillisTzConsumer(TimeStampMilliTZVector vector) { + super(vector); + } + + @Override + public void consume(Decoder decoder) throws IOException { + vector.set(currentIndex++, decoder.readLong()); + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampNanosConsumer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampNanosConsumer.java new file mode 100644 index 0000000000..b5044d221f --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampNanosConsumer.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.consumers.logical; + +import java.io.IOException; +import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; +import org.apache.arrow.vector.TimeStampNanoVector; +import org.apache.avro.io.Decoder; + +/** + * Consumer which consume local-timestamp-nanos values from avro decoder. Write the data to {@link + * TimeStampNanoVector}. + */ +public class AvroTimestampNanosConsumer extends BaseAvroConsumer { + + /** Instantiate a AvroTimestampNanosConsumer. */ + public AvroTimestampNanosConsumer(TimeStampNanoVector vector) { + super(vector); + } + + @Override + public void consume(Decoder decoder) throws IOException { + vector.set(currentIndex++, decoder.readLong()); + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampNanosTzConsumer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampNanosTzConsumer.java new file mode 100644 index 0000000000..3f42b7ccbb --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampNanosTzConsumer.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.consumers.logical; + +import java.io.IOException; +import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; +import org.apache.arrow.vector.TimeStampNanoTZVector; +import org.apache.avro.io.Decoder; + +/** + * Consumer which consume timestamp-nanos values from avro decoder. Write the data to {@link + * TimeStampNanoTZVector}. + */ +public class AvroTimestampNanosTzConsumer extends BaseAvroConsumer { + + /** Instantiate a AvroTimestampNanosConsumer. */ + public AvroTimestampNanosTzConsumer(TimeStampNanoTZVector vector) { + super(vector); + } + + @Override + public void consume(Decoder decoder) throws IOException { + vector.set(currentIndex++, decoder.readLong()); + } +} diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroNullableProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroNullableProducer.java index f4215dbf84..5f8b314f49 100644 --- a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroNullableProducer.java +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroNullableProducer.java @@ -21,8 +21,8 @@ import org.apache.avro.io.Encoder; /** - * Producer wrapper which producers nullable types to an avro encoder. Write the data to the - * underlying {@link FieldVector}. + * Producer wrapper which produces nullable types to an avro encoder. Read data from the underlying + * {@link FieldVector}. * * @param The vector within producer or its delegate, used for partially produce purpose. */ diff --git a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroSchemaTest.java b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroSchemaTest.java index a05bbc1653..d3e12e763a 100644 --- a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroSchemaTest.java +++ b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroSchemaTest.java @@ -319,10 +319,10 @@ public void testConvertDecimalTypes() { FieldType.notNullable(new ArrowType.Decimal(20, 10, 128)), null), new Field( - "nullableDecimal256", FieldType.nullable(new ArrowType.Decimal(20, 4, 256)), null), + "nullableDecimal256", FieldType.nullable(new ArrowType.Decimal(55, 15, 256)), null), new Field( "nonNullableDecimal2561", - FieldType.notNullable(new ArrowType.Decimal(20, 4, 256)), + FieldType.notNullable(new ArrowType.Decimal(55, 25, 256)), null), new Field( "nonNullableDecimal2562", @@ -330,7 +330,7 @@ public void testConvertDecimalTypes() { null), new Field( "nonNullableDecimal2563", - FieldType.notNullable(new ArrowType.Decimal(30, 15, 256)), + FieldType.notNullable(new ArrowType.Decimal(60, 50, 256)), null)); Schema schema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord"); @@ -383,9 +383,9 @@ public void testConvertDecimalTypes() { schema.getField("nullableDecimal256").schema().getTypes().get(0); assertEquals(Schema.Type.FIXED, nullableDecimal256Schema.getType()); assertEquals(32, nullableDecimal256Schema.getFixedSize()); - assertEquals(LogicalTypes.decimal(20, 4), nullableDecimal256Schema.getLogicalType()); - assertEquals(20, nullableDecimal256Schema.getObjectProp("precision")); - assertEquals(4, nullableDecimal256Schema.getObjectProp("scale")); + assertEquals(LogicalTypes.decimal(55, 15), nullableDecimal256Schema.getLogicalType()); + assertEquals(55, nullableDecimal256Schema.getObjectProp("precision")); + assertEquals(15, nullableDecimal256Schema.getObjectProp("scale")); assertEquals( Schema.Type.NULL, schema.getField("nullableDecimal256").schema().getTypes().get(1).getType()); @@ -394,9 +394,9 @@ public void testConvertDecimalTypes() { Schema nonNullableDecimal2561Schema = schema.getField("nonNullableDecimal2561").schema(); assertEquals(Schema.Type.FIXED, nonNullableDecimal2561Schema.getType()); assertEquals(32, nonNullableDecimal2561Schema.getFixedSize()); - assertEquals(LogicalTypes.decimal(20, 4), nonNullableDecimal2561Schema.getLogicalType()); - assertEquals(20, nonNullableDecimal2561Schema.getObjectProp("precision")); - assertEquals(4, nonNullableDecimal2561Schema.getObjectProp("scale")); + assertEquals(LogicalTypes.decimal(55, 25), nonNullableDecimal2561Schema.getLogicalType()); + assertEquals(55, nonNullableDecimal2561Schema.getObjectProp("precision")); + assertEquals(25, nonNullableDecimal2561Schema.getObjectProp("scale")); // Assertions for nonNullableDecimal2562 Schema nonNullableDecimal2562Schema = schema.getField("nonNullableDecimal2562").schema(); @@ -410,9 +410,9 @@ public void testConvertDecimalTypes() { Schema nonNullableDecimal2563Schema = schema.getField("nonNullableDecimal2563").schema(); assertEquals(Schema.Type.FIXED, nonNullableDecimal2563Schema.getType()); assertEquals(32, nonNullableDecimal2563Schema.getFixedSize()); - assertEquals(LogicalTypes.decimal(30, 15), nonNullableDecimal2563Schema.getLogicalType()); - assertEquals(30, nonNullableDecimal2563Schema.getObjectProp("precision")); - assertEquals(15, nonNullableDecimal2563Schema.getObjectProp("scale")); + assertEquals(LogicalTypes.decimal(60, 50), nonNullableDecimal2563Schema.getLogicalType()); + assertEquals(60, nonNullableDecimal2563Schema.getObjectProp("precision")); + assertEquals(50, nonNullableDecimal2563Schema.getObjectProp("scale")); } @Test diff --git a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroLogicalTypesTest.java b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroLogicalTypesTest.java index 173cc855b1..801456d79b 100644 --- a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroLogicalTypesTest.java +++ b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroLogicalTypesTest.java @@ -173,7 +173,7 @@ public void testInvalidDecimalPrecision() throws Exception { IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> writeAndRead(schema, data)); - assertTrue(e.getMessage().contains("Precision must be in range of 1 to 38")); + assertTrue(e.getMessage().contains("Precision must be in range of 1 to 76")); } @Test diff --git a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripDataTest.java b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripDataTest.java new file mode 100644 index 0000000000..85e6a960b0 --- /dev/null +++ b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripDataTest.java @@ -0,0 +1,1606 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZonedDateTime; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.apache.arrow.adapter.avro.producers.CompositeAvroProducer; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.DateDayVector; +import org.apache.arrow.vector.Decimal256Vector; +import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.FixedSizeBinaryVector; +import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.NullVector; +import org.apache.arrow.vector.TimeMicroVector; +import org.apache.arrow.vector.TimeMilliVector; +import org.apache.arrow.vector.TimeStampMicroTZVector; +import org.apache.arrow.vector.TimeStampMicroVector; +import org.apache.arrow.vector.TimeStampMilliTZVector; +import org.apache.arrow.vector.TimeStampMilliVector; +import org.apache.arrow.vector.TimeStampNanoTZVector; +import org.apache.arrow.vector.TimeStampNanoVector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.StructVector; +import org.apache.arrow.vector.complex.writer.BaseWriter; +import org.apache.arrow.vector.complex.writer.FieldWriter; +import org.apache.arrow.vector.types.DateUnit; +import org.apache.arrow.vector.types.FloatingPointPrecision; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.avro.Schema; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.EncoderFactory; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +public class RoundTripDataTest { + + @TempDir public static File TMP; + + private static AvroToArrowConfig basicConfig(BufferAllocator allocator) { + return new AvroToArrowConfig(allocator, 1000, null, Collections.emptySet(), false); + } + + private static VectorSchemaRoot readDataFile( + Schema schema, File dataFile, BufferAllocator allocator) throws Exception { + + try (FileInputStream fis = new FileInputStream(dataFile)) { + BinaryDecoder decoder = new DecoderFactory().directBinaryDecoder(fis, null); + return AvroToArrow.avroToArrow(schema, decoder, basicConfig(allocator)); + } + } + + private static void roundTripTest( + VectorSchemaRoot root, BufferAllocator allocator, File dataFile, int rowCount) + throws Exception { + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = + ArrowToAvroUtils.createCompositeProducer(root.getFieldVectors()); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Generate AVRO schema + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + + // Read back in and compare + try (VectorSchemaRoot roundTrip = readDataFile(schema, dataFile, allocator)) { + + assertEquals(root.getSchema(), roundTrip.getSchema()); + assertEquals(rowCount, roundTrip.getRowCount()); + + // Read and check values + for (int row = 0; row < rowCount; row++) { + assertEquals(root.getVector(0).getObject(row), roundTrip.getVector(0).getObject(row)); + } + } + } + + private static void roundTripByteArrayTest( + VectorSchemaRoot root, BufferAllocator allocator, File dataFile, int rowCount) + throws Exception { + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = + ArrowToAvroUtils.createCompositeProducer(root.getFieldVectors()); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Generate AVRO schema + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + + // Read back in and compare + try (VectorSchemaRoot roundTrip = readDataFile(schema, dataFile, allocator)) { + + assertEquals(root.getSchema(), roundTrip.getSchema()); + assertEquals(rowCount, roundTrip.getRowCount()); + + // Read and check values + for (int row = 0; row < rowCount; row++) { + byte[] rootBytes = (byte[]) root.getVector(0).getObject(row); + byte[] roundTripBytes = (byte[]) roundTrip.getVector(0).getObject(row); + assertArrayEquals(rootBytes, roundTripBytes); + } + } + } + + // Data round trip for primitive types, nullable and non-nullable + + @Test + public void testRoundTripNullColumn() throws Exception { + + // The current read implementation expects EOF, which never happens for a single null vector + // Include a boolean vector with this test for now, so that EOF exception will be triggered + + // Field definition + FieldType nullField = new FieldType(false, new ArrowType.Null(), null); + FieldType booleanField = new FieldType(false, new ArrowType.Bool(), null); + + // Create empty vector + BufferAllocator allocator = new RootAllocator(); + NullVector nullVector = new NullVector(new Field("nullColumn", nullField, null)); + BitVector booleanVector = new BitVector(new Field("boolean", booleanField, null), allocator); + + int rowCount = 10; + + // Set up VSR + List vectors = Arrays.asList(nullVector, booleanVector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set all values to null + for (int row = 0; row < rowCount; row++) { + nullVector.setNull(row); + booleanVector.set(row, 0); + } + + File dataFile = new File(TMP, "testRoundTripNullColumn.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripBooleans() throws Exception { + + // Field definition + FieldType booleanField = new FieldType(false, new ArrowType.Bool(), null); + + // Create empty vector + BufferAllocator allocator = new RootAllocator(); + BitVector booleanVector = new BitVector(new Field("boolean", booleanField, null), allocator); + + // Set up VSR + List vectors = Arrays.asList(booleanVector); + int rowCount = 10; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + for (int row = 0; row < rowCount; row++) { + booleanVector.set(row, row % 2 == 0 ? 1 : 0); + } + + File dataFile = new File(TMP, "testRoundTripBooleans.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripNullableBooleans() throws Exception { + + // Field definition + FieldType booleanField = new FieldType(true, new ArrowType.Bool(), null); + + // Create empty vector + BufferAllocator allocator = new RootAllocator(); + BitVector booleanVector = new BitVector(new Field("boolean", booleanField, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = Arrays.asList(booleanVector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Null value + booleanVector.setNull(0); + + // False value + booleanVector.set(1, 0); + + // True value + booleanVector.set(2, 1); + + File dataFile = new File(TMP, "testRoundTripNullableBooleans.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripIntegers() throws Exception { + + // Field definitions + FieldType int32Field = new FieldType(false, new ArrowType.Int(32, true), null); + FieldType int64Field = new FieldType(false, new ArrowType.Int(64, true), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + IntVector int32Vector = new IntVector(new Field("int32", int32Field, null), allocator); + BigIntVector int64Vector = new BigIntVector(new Field("int64", int64Field, null), allocator); + + // Set up VSR + List vectors = Arrays.asList(int32Vector, int64Vector); + + int rowCount = 12; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + for (int row = 0; row < 10; row++) { + int32Vector.set(row, 513 * row * (row % 2 == 0 ? 1 : -1)); + int64Vector.set(row, 3791L * row * (row % 2 == 0 ? 1 : -1)); + } + + // Min values + int32Vector.set(10, Integer.MIN_VALUE); + int64Vector.set(10, Long.MIN_VALUE); + + // Max values + int32Vector.set(11, Integer.MAX_VALUE); + int64Vector.set(11, Long.MAX_VALUE); + + File dataFile = new File(TMP, "testRoundTripIntegers.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripNullableIntegers() throws Exception { + + // Field definitions + FieldType int32Field = new FieldType(true, new ArrowType.Int(32, true), null); + FieldType int64Field = new FieldType(true, new ArrowType.Int(64, true), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + IntVector int32Vector = new IntVector(new Field("int32", int32Field, null), allocator); + BigIntVector int64Vector = new BigIntVector(new Field("int64", int64Field, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = Arrays.asList(int32Vector, int64Vector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Null values + int32Vector.setNull(0); + int64Vector.setNull(0); + + // Zero values + int32Vector.set(1, 0); + int64Vector.set(1, 0); + + // Non-zero values + int32Vector.set(2, Integer.MAX_VALUE); + int64Vector.set(2, Long.MAX_VALUE); + + File dataFile = new File(TMP, "testRoundTripNullableIntegers.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripFloatingPoints() throws Exception { + + // Field definitions + FieldType float32Field = + new FieldType(false, new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE), null); + FieldType float64Field = + new FieldType(false, new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + Float4Vector float32Vector = + new Float4Vector(new Field("float32", float32Field, null), allocator); + Float8Vector float64Vector = + new Float8Vector(new Field("float64", float64Field, null), allocator); + + // Set up VSR + List vectors = Arrays.asList(float32Vector, float64Vector); + int rowCount = 15; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + for (int row = 0; row < 10; row++) { + float32Vector.set(row, 37.6f * row * (row % 2 == 0 ? 1 : -1)); + float64Vector.set(row, 37.6d * row * (row % 2 == 0 ? 1 : -1)); + } + + float32Vector.set(10, Float.MIN_VALUE); + float64Vector.set(10, Double.MIN_VALUE); + + float32Vector.set(11, Float.MAX_VALUE); + float64Vector.set(11, Double.MAX_VALUE); + + float32Vector.set(12, Float.NaN); + float64Vector.set(12, Double.NaN); + + float32Vector.set(13, Float.POSITIVE_INFINITY); + float64Vector.set(13, Double.POSITIVE_INFINITY); + + float32Vector.set(14, Float.NEGATIVE_INFINITY); + float64Vector.set(14, Double.NEGATIVE_INFINITY); + + File dataFile = new File(TMP, "testRoundTripFloatingPoints.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripNullableFloatingPoints() throws Exception { + + // Field definitions + FieldType float32Field = + new FieldType(true, new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE), null); + FieldType float64Field = + new FieldType(true, new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + Float4Vector float32Vector = + new Float4Vector(new Field("float32", float32Field, null), allocator); + Float8Vector float64Vector = + new Float8Vector(new Field("float64", float64Field, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = Arrays.asList(float32Vector, float64Vector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Null values + float32Vector.setNull(0); + float64Vector.setNull(0); + + // Zero values + float32Vector.set(1, 0.0f); + float64Vector.set(1, 0.0); + + // Non-zero values + float32Vector.set(2, 1.0f); + float64Vector.set(2, 1.0); + + File dataFile = new File(TMP, "testRoundTripNullableFloatingPoints.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripStrings() throws Exception { + + // Field definition + FieldType stringField = new FieldType(false, new ArrowType.Utf8(), null); + + // Create empty vector + BufferAllocator allocator = new RootAllocator(); + VarCharVector stringVector = + new VarCharVector(new Field("string", stringField, null), allocator); + + // Set up VSR + List vectors = Arrays.asList(stringVector); + int rowCount = 5; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + stringVector.setSafe(0, "Hello world!".getBytes()); + stringVector.setSafe(1, "<%**\r\n\t\\abc\0$$>".getBytes()); + stringVector.setSafe(2, "你好世界".getBytes()); + stringVector.setSafe(3, "مرحبا بالعالم".getBytes()); + stringVector.setSafe(4, "(P ∧ P ⇒ Q) ⇒ Q".getBytes()); + + File dataFile = new File(TMP, "testRoundTripStrings.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripNullableStrings() throws Exception { + + // Field definition + FieldType stringField = new FieldType(true, new ArrowType.Utf8(), null); + + // Create empty vector + BufferAllocator allocator = new RootAllocator(); + VarCharVector stringVector = + new VarCharVector(new Field("string", stringField, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = Arrays.asList(stringVector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + stringVector.setNull(0); + stringVector.setSafe(1, "".getBytes()); + stringVector.setSafe(2, "not empty".getBytes()); + + File dataFile = new File(TMP, "testRoundTripNullableStrings.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripBinary() throws Exception { + + // Field definition + FieldType binaryField = new FieldType(false, new ArrowType.Binary(), null); + FieldType fixedField = new FieldType(false, new ArrowType.FixedSizeBinary(5), null); + + // Create empty vector + BufferAllocator allocator = new RootAllocator(); + VarBinaryVector binaryVector = + new VarBinaryVector(new Field("binary", binaryField, null), allocator); + FixedSizeBinaryVector fixedVector = + new FixedSizeBinaryVector(new Field("fixed", fixedField, null), allocator); + + // Set up VSR + List vectors = Arrays.asList(binaryVector, fixedVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + binaryVector.setSafe(0, new byte[] {1, 2, 3}); + binaryVector.setSafe(1, new byte[] {4, 5, 6, 7}); + binaryVector.setSafe(2, new byte[] {8, 9}); + + fixedVector.setSafe(0, new byte[] {1, 2, 3, 4, 5}); + fixedVector.setSafe(1, new byte[] {4, 5, 6, 7, 8, 9}); + fixedVector.setSafe(2, new byte[] {8, 9, 10, 11, 12}); + + File dataFile = new File(TMP, "testRoundTripBinary.avro"); + + roundTripByteArrayTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripNullableBinary() throws Exception { + + // Field definition + FieldType binaryField = new FieldType(true, new ArrowType.Binary(), null); + FieldType fixedField = new FieldType(true, new ArrowType.FixedSizeBinary(5), null); + + // Create empty vector + BufferAllocator allocator = new RootAllocator(); + VarBinaryVector binaryVector = + new VarBinaryVector(new Field("binary", binaryField, null), allocator); + FixedSizeBinaryVector fixedVector = + new FixedSizeBinaryVector(new Field("fixed", fixedField, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = Arrays.asList(binaryVector, fixedVector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + binaryVector.setNull(0); + binaryVector.setSafe(1, new byte[] {}); + binaryVector.setSafe(2, new byte[] {10, 11, 12}); + + fixedVector.setNull(0); + fixedVector.setSafe(1, new byte[] {0, 0, 0, 0, 0}); + fixedVector.setSafe(2, new byte[] {10, 11, 12, 13, 14}); + + File dataFile = new File(TMP, "testRoundTripNullableBinary.avro"); + + roundTripByteArrayTest(root, allocator, dataFile, rowCount); + } + } + + // Data round trip for logical types, nullable and non-nullable + + @Test + public void testRoundTripDecimals() throws Exception { + + // Field definitions + FieldType decimal128Field1 = new FieldType(false, new ArrowType.Decimal(38, 10, 128), null); + FieldType decimal128Field2 = new FieldType(false, new ArrowType.Decimal(38, 5, 128), null); + FieldType decimal256Field1 = new FieldType(false, new ArrowType.Decimal(76, 20, 256), null); + FieldType decimal256Field2 = new FieldType(false, new ArrowType.Decimal(76, 10, 256), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + DecimalVector decimal128Vector1 = + new DecimalVector(new Field("decimal128_1", decimal128Field1, null), allocator); + DecimalVector decimal128Vector2 = + new DecimalVector(new Field("decimal128_2", decimal128Field2, null), allocator); + Decimal256Vector decimal256Vector1 = + new Decimal256Vector(new Field("decimal256_1", decimal256Field1, null), allocator); + Decimal256Vector decimal256Vector2 = + new Decimal256Vector(new Field("decimal256_2", decimal256Field2, null), allocator); + + // Set up VSR + List vectors = + Arrays.asList(decimal128Vector1, decimal128Vector2, decimal256Vector1, decimal256Vector2); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + decimal128Vector1.setSafe( + 0, new BigDecimal("12345.67890").setScale(10, RoundingMode.UNNECESSARY)); + decimal128Vector1.setSafe( + 1, new BigDecimal("-98765.43210").setScale(10, RoundingMode.UNNECESSARY)); + decimal128Vector1.setSafe( + 2, new BigDecimal("54321.09876").setScale(10, RoundingMode.UNNECESSARY)); + + decimal128Vector2.setSafe( + 0, new BigDecimal("12345.67890").setScale(5, RoundingMode.UNNECESSARY)); + decimal128Vector2.setSafe( + 1, new BigDecimal("-98765.43210").setScale(5, RoundingMode.UNNECESSARY)); + decimal128Vector2.setSafe( + 2, new BigDecimal("54321.09876").setScale(5, RoundingMode.UNNECESSARY)); + + decimal256Vector1.setSafe( + 0, + new BigDecimal("12345678901234567890.12345678901234567890") + .setScale(20, RoundingMode.UNNECESSARY)); + decimal256Vector1.setSafe( + 1, + new BigDecimal("-98765432109876543210.98765432109876543210") + .setScale(20, RoundingMode.UNNECESSARY)); + decimal256Vector1.setSafe( + 2, + new BigDecimal("54321098765432109876.54321098765432109876") + .setScale(20, RoundingMode.UNNECESSARY)); + + decimal256Vector2.setSafe( + 0, + new BigDecimal("12345678901234567890.1234567890").setScale(10, RoundingMode.UNNECESSARY)); + decimal256Vector2.setSafe( + 1, + new BigDecimal("-98765432109876543210.9876543210") + .setScale(10, RoundingMode.UNNECESSARY)); + decimal256Vector2.setSafe( + 2, + new BigDecimal("54321098765432109876.5432109876").setScale(10, RoundingMode.UNNECESSARY)); + + File dataFile = new File(TMP, "testRoundTripDecimals.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripNullableDecimals() throws Exception { + + // Field definitions + FieldType decimal128Field1 = new FieldType(true, new ArrowType.Decimal(38, 10, 128), null); + FieldType decimal128Field2 = new FieldType(true, new ArrowType.Decimal(38, 5, 128), null); + FieldType decimal256Field1 = new FieldType(true, new ArrowType.Decimal(76, 20, 256), null); + FieldType decimal256Field2 = new FieldType(true, new ArrowType.Decimal(76, 10, 256), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + DecimalVector decimal128Vector1 = + new DecimalVector(new Field("decimal128_1", decimal128Field1, null), allocator); + DecimalVector decimal128Vector2 = + new DecimalVector(new Field("decimal128_2", decimal128Field2, null), allocator); + Decimal256Vector decimal256Vector1 = + new Decimal256Vector(new Field("decimal256_1", decimal256Field1, null), allocator); + Decimal256Vector decimal256Vector2 = + new Decimal256Vector(new Field("decimal256_2", decimal256Field2, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = + Arrays.asList(decimal128Vector1, decimal128Vector2, decimal256Vector1, decimal256Vector2); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + decimal128Vector1.setNull(0); + decimal128Vector1.setSafe(1, BigDecimal.ZERO.setScale(10, RoundingMode.UNNECESSARY)); + decimal128Vector1.setSafe( + 2, new BigDecimal("12345.67890").setScale(10, RoundingMode.UNNECESSARY)); + + decimal128Vector2.setNull(0); + decimal128Vector2.setSafe(1, BigDecimal.ZERO.setScale(5, RoundingMode.UNNECESSARY)); + decimal128Vector2.setSafe( + 2, new BigDecimal("98765.43210").setScale(5, RoundingMode.UNNECESSARY)); + + decimal256Vector1.setNull(0); + decimal256Vector1.setSafe(1, BigDecimal.ZERO.setScale(20, RoundingMode.UNNECESSARY)); + decimal256Vector1.setSafe( + 2, + new BigDecimal("12345678901234567890.12345678901234567890") + .setScale(20, RoundingMode.UNNECESSARY)); + + decimal256Vector2.setNull(0); + decimal256Vector2.setSafe(1, BigDecimal.ZERO.setScale(10, RoundingMode.UNNECESSARY)); + decimal256Vector2.setSafe( + 2, + new BigDecimal("98765432109876543210.9876543210").setScale(10, RoundingMode.UNNECESSARY)); + + File dataFile = new File(TMP, "testRoundTripNullableDecimals.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripDates() throws Exception { + + // Field definitions + FieldType dateDayField = new FieldType(false, new ArrowType.Date(DateUnit.DAY), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + DateDayVector dateDayVector = + new DateDayVector(new Field("dateDay", dateDayField, null), allocator); + + // Set up VSR + List vectors = Arrays.asList(dateDayVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + dateDayVector.setSafe(0, (int) LocalDate.now().toEpochDay()); + dateDayVector.setSafe(1, (int) LocalDate.now().toEpochDay() + 1); + dateDayVector.setSafe(2, (int) LocalDate.now().toEpochDay() + 2); + + File dataFile = new File(TMP, "testRoundTripDates.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripNullableDates() throws Exception { + + // Field definitions + FieldType dateDayField = new FieldType(true, new ArrowType.Date(DateUnit.DAY), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + DateDayVector dateDayVector = + new DateDayVector(new Field("dateDay", dateDayField, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = Arrays.asList(dateDayVector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + dateDayVector.setNull(0); + dateDayVector.setSafe(1, 0); + dateDayVector.setSafe(2, (int) LocalDate.now().toEpochDay()); + + File dataFile = new File(TMP, "testRoundTripNullableDates.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripTimes() throws Exception { + + // Field definitions + FieldType timeMillisField = + new FieldType(false, new ArrowType.Time(TimeUnit.MILLISECOND, 32), null); + FieldType timeMicrosField = + new FieldType(false, new ArrowType.Time(TimeUnit.MICROSECOND, 64), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + TimeMilliVector timeMillisVector = + new TimeMilliVector(new Field("timeMillis", timeMillisField, null), allocator); + TimeMicroVector timeMicrosVector = + new TimeMicroVector(new Field("timeMicros", timeMicrosField, null), allocator); + + // Set up VSR + List vectors = Arrays.asList(timeMillisVector, timeMicrosVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + timeMillisVector.setSafe( + 0, (int) (ZonedDateTime.now().toLocalTime().toNanoOfDay() / 1000000)); + timeMillisVector.setSafe( + 1, (int) (ZonedDateTime.now().toLocalTime().toNanoOfDay() / 1000000) - 1000); + timeMillisVector.setSafe( + 2, (int) (ZonedDateTime.now().toLocalTime().toNanoOfDay() / 1000000) - 2000); + + timeMicrosVector.setSafe(0, ZonedDateTime.now().toLocalTime().toNanoOfDay() / 1000); + timeMicrosVector.setSafe(1, ZonedDateTime.now().toLocalTime().toNanoOfDay() / 1000 - 1000000); + timeMicrosVector.setSafe(2, ZonedDateTime.now().toLocalTime().toNanoOfDay() / 1000 - 2000000); + + File dataFile = new File(TMP, "testRoundTripTimes.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripNullableTimes() throws Exception { + + // Field definitions + FieldType timeMillisField = + new FieldType(true, new ArrowType.Time(TimeUnit.MILLISECOND, 32), null); + FieldType timeMicrosField = + new FieldType(true, new ArrowType.Time(TimeUnit.MICROSECOND, 64), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + TimeMilliVector timeMillisVector = + new TimeMilliVector(new Field("timeMillis", timeMillisField, null), allocator); + TimeMicroVector timeMicrosVector = + new TimeMicroVector(new Field("timeMicros", timeMicrosField, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = Arrays.asList(timeMillisVector, timeMicrosVector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + timeMillisVector.setNull(0); + timeMillisVector.setSafe(1, 0); + timeMillisVector.setSafe( + 2, (int) (ZonedDateTime.now().toLocalTime().toNanoOfDay() / 1000000)); + + timeMicrosVector.setNull(0); + timeMicrosVector.setSafe(1, 0); + timeMicrosVector.setSafe(2, ZonedDateTime.now().toLocalTime().toNanoOfDay() / 1000); + + File dataFile = new File(TMP, "testRoundTripNullableTimes.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripZoneAwareTimestamps() throws Exception { + + // Field definitions + FieldType timestampMillisField = + new FieldType(false, new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC"), null); + FieldType timestampMicrosField = + new FieldType(false, new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC"), null); + FieldType timestampNanosField = + new FieldType(false, new ArrowType.Timestamp(TimeUnit.NANOSECOND, "UTC"), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + TimeStampMilliTZVector timestampMillisVector = + new TimeStampMilliTZVector( + new Field("timestampMillis", timestampMillisField, null), allocator); + TimeStampMicroTZVector timestampMicrosVector = + new TimeStampMicroTZVector( + new Field("timestampMicros", timestampMicrosField, null), allocator); + TimeStampNanoTZVector timestampNanosVector = + new TimeStampNanoTZVector( + new Field("timestampNanos", timestampNanosField, null), allocator); + + // Set up VSR + List vectors = + Arrays.asList(timestampMillisVector, timestampMicrosVector, timestampNanosVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + timestampMillisVector.setSafe(0, (int) Instant.now().toEpochMilli()); + timestampMillisVector.setSafe(1, (int) Instant.now().toEpochMilli() - 1000); + timestampMillisVector.setSafe(2, (int) Instant.now().toEpochMilli() - 2000); + + timestampMicrosVector.setSafe(0, Instant.now().toEpochMilli() * 1000); + timestampMicrosVector.setSafe(1, (Instant.now().toEpochMilli() - 1000) * 1000); + timestampMicrosVector.setSafe(2, (Instant.now().toEpochMilli() - 2000) * 1000); + + timestampNanosVector.setSafe(0, Instant.now().toEpochMilli() * 1000000); + timestampNanosVector.setSafe(1, (Instant.now().toEpochMilli() - 1000) * 1000000); + timestampNanosVector.setSafe(2, (Instant.now().toEpochMilli() - 2000) * 1000000); + + File dataFile = new File(TMP, "testRoundTripZoneAwareTimestamps.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripNullableZoneAwareTimestamps() throws Exception { + + // Field definitions + FieldType timestampMillisField = + new FieldType(true, new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC"), null); + FieldType timestampMicrosField = + new FieldType(true, new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC"), null); + FieldType timestampNanosField = + new FieldType(true, new ArrowType.Timestamp(TimeUnit.NANOSECOND, "UTC"), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + TimeStampMilliTZVector timestampMillisVector = + new TimeStampMilliTZVector( + new Field("timestampMillis", timestampMillisField, null), allocator); + TimeStampMicroTZVector timestampMicrosVector = + new TimeStampMicroTZVector( + new Field("timestampMicros", timestampMicrosField, null), allocator); + TimeStampNanoTZVector timestampNanosVector = + new TimeStampNanoTZVector( + new Field("timestampNanos", timestampNanosField, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = + Arrays.asList(timestampMillisVector, timestampMicrosVector, timestampNanosVector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + timestampMillisVector.setNull(0); + timestampMillisVector.setSafe(1, 0); + timestampMillisVector.setSafe(2, (int) Instant.now().toEpochMilli()); + + timestampMicrosVector.setNull(0); + timestampMicrosVector.setSafe(1, 0); + timestampMicrosVector.setSafe(2, Instant.now().toEpochMilli() * 1000); + + timestampNanosVector.setNull(0); + timestampNanosVector.setSafe(1, 0); + timestampNanosVector.setSafe(2, Instant.now().toEpochMilli() * 1000000); + + File dataFile = new File(TMP, "testRoundTripNullableZoneAwareTimestamps.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripLocalTimestamps() throws Exception { + + // Field definitions + FieldType timestampMillisField = + new FieldType(false, new ArrowType.Timestamp(TimeUnit.MILLISECOND, null), null); + FieldType timestampMicrosField = + new FieldType(false, new ArrowType.Timestamp(TimeUnit.MICROSECOND, null), null); + FieldType timestampNanosField = + new FieldType(false, new ArrowType.Timestamp(TimeUnit.NANOSECOND, null), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + TimeStampMilliVector timestampMillisVector = + new TimeStampMilliVector( + new Field("timestampMillis", timestampMillisField, null), allocator); + TimeStampMicroVector timestampMicrosVector = + new TimeStampMicroVector( + new Field("timestampMicros", timestampMicrosField, null), allocator); + TimeStampNanoVector timestampNanosVector = + new TimeStampNanoVector(new Field("timestampNanos", timestampNanosField, null), allocator); + + // Set up VSR + List vectors = + Arrays.asList(timestampMillisVector, timestampMicrosVector, timestampNanosVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + timestampMillisVector.setSafe(0, (int) Instant.now().toEpochMilli()); + timestampMillisVector.setSafe(1, (int) Instant.now().toEpochMilli() - 1000); + timestampMillisVector.setSafe(2, (int) Instant.now().toEpochMilli() - 2000); + + timestampMicrosVector.setSafe(0, Instant.now().toEpochMilli() * 1000); + timestampMicrosVector.setSafe(1, (Instant.now().toEpochMilli() - 1000) * 1000); + timestampMicrosVector.setSafe(2, (Instant.now().toEpochMilli() - 2000) * 1000); + + timestampNanosVector.setSafe(0, Instant.now().toEpochMilli() * 1000000); + timestampNanosVector.setSafe(1, (Instant.now().toEpochMilli() - 1000) * 1000000); + timestampNanosVector.setSafe(2, (Instant.now().toEpochMilli() - 2000) * 1000000); + + File dataFile = new File(TMP, "testRoundTripLocalTimestamps.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripNullableLocalTimestamps() throws Exception { + + // Field definitions + FieldType timestampMillisField = + new FieldType(true, new ArrowType.Timestamp(TimeUnit.MILLISECOND, null), null); + FieldType timestampMicrosField = + new FieldType(true, new ArrowType.Timestamp(TimeUnit.MICROSECOND, null), null); + FieldType timestampNanosField = + new FieldType(true, new ArrowType.Timestamp(TimeUnit.NANOSECOND, null), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + TimeStampMilliVector timestampMillisVector = + new TimeStampMilliVector( + new Field("timestampMillis", timestampMillisField, null), allocator); + TimeStampMicroVector timestampMicrosVector = + new TimeStampMicroVector( + new Field("timestampMicros", timestampMicrosField, null), allocator); + TimeStampNanoVector timestampNanosVector = + new TimeStampNanoVector(new Field("timestampNanos", timestampNanosField, null), allocator); + + int rowCount = 3; + + // Set up VSR + List vectors = + Arrays.asList(timestampMillisVector, timestampMicrosVector, timestampNanosVector); + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + timestampMillisVector.setNull(0); + timestampMillisVector.setSafe(1, 0); + timestampMillisVector.setSafe(2, (int) Instant.now().toEpochMilli()); + + timestampMicrosVector.setNull(0); + timestampMicrosVector.setSafe(1, 0); + timestampMicrosVector.setSafe(2, Instant.now().toEpochMilli() * 1000); + + timestampNanosVector.setNull(0); + timestampNanosVector.setSafe(1, 0); + timestampNanosVector.setSafe(2, Instant.now().toEpochMilli() * 1000000); + + File dataFile = new File(TMP, "testRoundTripNullableLocalTimestamps.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + // Data round trip for containers of primitive and logical types, nullable and non-nullable + + @Test + public void testRoundTripLists() throws Exception { + + // Field definitions + FieldType intListField = new FieldType(false, new ArrowType.List(), null); + FieldType stringListField = new FieldType(false, new ArrowType.List(), null); + FieldType dateListField = new FieldType(false, new ArrowType.List(), null); + + Field intField = new Field("item", FieldType.notNullable(new ArrowType.Int(32, true)), null); + Field stringField = new Field("item", FieldType.notNullable(new ArrowType.Utf8()), null); + Field dateField = + new Field("item", FieldType.notNullable(new ArrowType.Date(DateUnit.DAY)), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + ListVector intListVector = new ListVector("intList", allocator, intListField, null); + ListVector stringListVector = new ListVector("stringList", allocator, stringListField, null); + ListVector dateListVector = new ListVector("dateList", allocator, dateListField, null); + + intListVector.initializeChildrenFromFields(Arrays.asList(intField)); + stringListVector.initializeChildrenFromFields(Arrays.asList(stringField)); + dateListVector.initializeChildrenFromFields(Arrays.asList(dateField)); + + // Set up VSR + List vectors = Arrays.asList(intListVector, stringListVector, dateListVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + FieldWriter intListWriter = intListVector.getWriter(); + FieldWriter stringListWriter = stringListVector.getWriter(); + FieldWriter dateListWriter = dateListVector.getWriter(); + + // Set test data for intList + for (int i = 0; i < rowCount; i++) { + intListWriter.startList(); + for (int j = 0; j < 5 - i; j++) { + intListWriter.writeInt(j); + } + intListWriter.endList(); + } + + // Set test data for stringList + for (int i = 0; i < rowCount; i++) { + stringListWriter.startList(); + for (int j = 0; j < 5 - i; j++) { + stringListWriter.writeVarChar("string" + j); + } + stringListWriter.endList(); + } + + // Set test data for dateList + for (int i = 0; i < rowCount; i++) { + dateListWriter.startList(); + for (int j = 0; j < 5 - i; j++) { + dateListWriter.writeDateDay((int) LocalDate.now().plusDays(j).toEpochDay()); + } + dateListWriter.endList(); + } + + // Update count for the vectors + intListVector.setValueCount(rowCount); + stringListVector.setValueCount(rowCount); + dateListVector.setValueCount(rowCount); + + File dataFile = new File(TMP, "testRoundTripLists.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripNullableLists() throws Exception { + + // Field definitions + FieldType nullListType = new FieldType(true, new ArrowType.List(), null); + FieldType nonNullListType = new FieldType(false, new ArrowType.List(), null); + + Field nullFieldType = new Field("item", FieldType.nullable(new ArrowType.Int(32, true)), null); + Field nonNullFieldType = + new Field("item", FieldType.notNullable(new ArrowType.Int(32, true)), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + ListVector nullEntriesVector = + new ListVector("nullEntriesVector", allocator, nonNullListType, null); + ListVector nullListVector = new ListVector("nullListVector", allocator, nullListType, null); + ListVector nullBothVector = new ListVector("nullBothVector", allocator, nullListType, null); + + nullEntriesVector.initializeChildrenFromFields(Arrays.asList(nullFieldType)); + nullListVector.initializeChildrenFromFields(Arrays.asList(nonNullFieldType)); + nullBothVector.initializeChildrenFromFields(Arrays.asList(nullFieldType)); + + // Set up VSR + List vectors = Arrays.asList(nullEntriesVector, nullListVector, nullBothVector); + int rowCount = 4; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data for nullEntriesVector + FieldWriter nullEntriesWriter = nullEntriesVector.getWriter(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeNull(); + nullEntriesWriter.integer().writeNull(); + nullEntriesWriter.endList(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeInt(0); + nullEntriesWriter.integer().writeInt(0); + nullEntriesWriter.endList(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeInt(123); + nullEntriesWriter.integer().writeInt(456); + nullEntriesWriter.endList(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeInt(789); + nullEntriesWriter.integer().writeInt(789); + nullEntriesWriter.endList(); + + // Set test data for nullListVector + FieldWriter nullListWriter = nullListVector.getWriter(); + nullListWriter.writeNull(); + nullListWriter.setPosition(1); // writeNull() does not inc. idx() on list vector + nullListWriter.startList(); + nullListWriter.integer().writeInt(0); + nullListWriter.integer().writeInt(0); + nullListWriter.endList(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeInt(123); + nullEntriesWriter.integer().writeInt(456); + nullEntriesWriter.endList(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeInt(789); + nullEntriesWriter.integer().writeInt(789); + nullEntriesWriter.endList(); + + // Set test data for nullBothVector + FieldWriter nullBothWriter = nullBothVector.getWriter(); + nullBothWriter.writeNull(); + nullBothWriter.setPosition(1); + nullBothWriter.startList(); + nullBothWriter.integer().writeNull(); + nullBothWriter.integer().writeNull(); + nullBothWriter.endList(); + nullListWriter.startList(); + nullListWriter.integer().writeInt(0); + nullListWriter.integer().writeInt(0); + nullListWriter.endList(); + nullEntriesWriter.startList(); + nullEntriesWriter.integer().writeInt(123); + nullEntriesWriter.integer().writeInt(456); + nullEntriesWriter.endList(); + + // Update count for the vectors + nullListVector.setValueCount(4); + nullEntriesVector.setValueCount(4); + nullBothVector.setValueCount(4); + + File dataFile = new File(TMP, "testRoundTripNullableLists.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripMap() throws Exception { + + // Field definitions + FieldType intMapField = new FieldType(false, new ArrowType.Map(false), null); + FieldType stringMapField = new FieldType(false, new ArrowType.Map(false), null); + FieldType dateMapField = new FieldType(false, new ArrowType.Map(false), null); + + Field keyField = new Field("key", FieldType.notNullable(new ArrowType.Utf8()), null); + Field intField = new Field("value", FieldType.notNullable(new ArrowType.Int(32, true)), null); + Field stringField = new Field("value", FieldType.notNullable(new ArrowType.Utf8()), null); + Field dateField = + new Field("value", FieldType.notNullable(new ArrowType.Date(DateUnit.DAY)), null); + + Field intEntryField = + new Field( + "entries", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList(keyField, intField)); + Field stringEntryField = + new Field( + "entries", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList(keyField, stringField)); + Field dateEntryField = + new Field( + "entries", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList(keyField, dateField)); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + MapVector intMapVector = new MapVector("intMap", allocator, intMapField, null); + MapVector stringMapVector = new MapVector("stringMap", allocator, stringMapField, null); + MapVector dateMapVector = new MapVector("dateMap", allocator, dateMapField, null); + + intMapVector.initializeChildrenFromFields(Arrays.asList(intEntryField)); + stringMapVector.initializeChildrenFromFields(Arrays.asList(stringEntryField)); + dateMapVector.initializeChildrenFromFields(Arrays.asList(dateEntryField)); + + // Set up VSR + List vectors = Arrays.asList(intMapVector, stringMapVector, dateMapVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Total number of entries that will be writen to each vector + int entryCount = 5 + 4 + 3; + + // Set test data for intList + BaseWriter.MapWriter writer = intMapVector.getWriter(); + for (int i = 0; i < rowCount; i++) { + writer.startMap(); + for (int j = 0; j < 5 - i; j++) { + writer.startEntry(); + writer.key().varChar().writeVarChar("key" + j); + writer.value().integer().writeInt(j); + writer.endEntry(); + } + writer.endMap(); + } + + // Update count for data vector (map writer does not do this) + intMapVector.getDataVector().setValueCount(entryCount); + + // Set test data for stringList + BaseWriter.MapWriter stringWriter = stringMapVector.getWriter(); + for (int i = 0; i < rowCount; i++) { + stringWriter.startMap(); + for (int j = 0; j < 5 - i; j++) { + stringWriter.startEntry(); + stringWriter.key().varChar().writeVarChar("key" + j); + stringWriter.value().varChar().writeVarChar("string" + j); + stringWriter.endEntry(); + } + stringWriter.endMap(); + } + + // Update count for the vectors + intMapVector.setValueCount(rowCount); + stringMapVector.setValueCount(rowCount); + dateMapVector.setValueCount(rowCount); + + // Update count for data vector (map writer does not do this) + stringMapVector.getDataVector().setValueCount(entryCount); + + // Set test data for dateList + BaseWriter.MapWriter dateWriter = dateMapVector.getWriter(); + for (int i = 0; i < rowCount; i++) { + dateWriter.startMap(); + for (int j = 0; j < 5 - i; j++) { + dateWriter.startEntry(); + dateWriter.key().varChar().writeVarChar("key" + j); + dateWriter.value().dateDay().writeDateDay((int) LocalDate.now().plusDays(j).toEpochDay()); + dateWriter.endEntry(); + } + dateWriter.endMap(); + } + + // Update count for data vector (map writer does not do this) + dateMapVector.getDataVector().setValueCount(entryCount); + + File dataFile = new File(TMP, "testRoundTripMap.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripNullableMap() throws Exception { + + // Field definitions + FieldType nullMapType = new FieldType(true, new ArrowType.Map(false), null); + FieldType nonNullMapType = new FieldType(false, new ArrowType.Map(false), null); + + Field keyField = new Field("key", FieldType.notNullable(new ArrowType.Utf8()), null); + Field nullFieldType = new Field("value", FieldType.nullable(new ArrowType.Int(32, true)), null); + Field nonNullFieldType = + new Field("value", FieldType.notNullable(new ArrowType.Int(32, true)), null); + Field nullEntryField = + new Field( + "entries", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList(keyField, nullFieldType)); + Field nonNullEntryField = + new Field( + "entries", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList(keyField, nonNullFieldType)); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + MapVector nullEntriesVector = + new MapVector("nullEntriesVector", allocator, nonNullMapType, null); + MapVector nullMapVector = new MapVector("nullMapVector", allocator, nullMapType, null); + MapVector nullBothVector = new MapVector("nullBothVector", allocator, nullMapType, null); + + nullEntriesVector.initializeChildrenFromFields(Arrays.asList(nullEntryField)); + nullMapVector.initializeChildrenFromFields(Arrays.asList(nonNullEntryField)); + nullBothVector.initializeChildrenFromFields(Arrays.asList(nullEntryField)); + + // Set up VSR + List vectors = Arrays.asList(nullEntriesVector, nullMapVector, nullBothVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data for intList + BaseWriter.MapWriter writer = nullEntriesVector.getWriter(); + writer.startMap(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key0"); + writer.value().integer().writeNull(); + writer.endEntry(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key1"); + writer.value().integer().writeNull(); + writer.endEntry(); + writer.endMap(); + writer.startMap(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key2"); + writer.value().integer().writeInt(0); + writer.endEntry(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key3"); + writer.value().integer().writeInt(0); + writer.endEntry(); + writer.endMap(); + writer.startMap(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key4"); + writer.value().integer().writeInt(123); + writer.endEntry(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key5"); + writer.value().integer().writeInt(456); + writer.endEntry(); + writer.endMap(); + + // Set test data for stringList + BaseWriter.MapWriter nullMapWriter = nullMapVector.getWriter(); + nullMapWriter.writeNull(); + nullMapWriter.setPosition(1); // writeNull() does not inc. idx() on map (list) vector + nullMapWriter.startMap(); + nullMapWriter.startEntry(); + nullMapWriter.key().varChar().writeVarChar("key2"); + nullMapWriter.value().integer().writeInt(0); + nullMapWriter.endEntry(); + writer.startMap(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key3"); + writer.value().integer().writeInt(0); + writer.endEntry(); + nullMapWriter.endMap(); + nullMapWriter.startMap(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key4"); + writer.value().integer().writeInt(123); + writer.endEntry(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key5"); + writer.value().integer().writeInt(456); + writer.endEntry(); + nullMapWriter.endMap(); + + // Set test data for dateList + BaseWriter.MapWriter nullBothWriter = nullBothVector.getWriter(); + nullBothWriter.writeNull(); + nullBothWriter.setPosition(1); + nullBothWriter.startMap(); + nullBothWriter.startEntry(); + nullBothWriter.key().varChar().writeVarChar("key2"); + nullBothWriter.value().integer().writeNull(); + nullBothWriter.endEntry(); + nullBothWriter.startEntry(); + nullBothWriter.key().varChar().writeVarChar("key3"); + nullBothWriter.value().integer().writeNull(); + nullBothWriter.endEntry(); + nullBothWriter.endMap(); + nullBothWriter.startMap(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key4"); + writer.value().integer().writeInt(123); + writer.endEntry(); + writer.startEntry(); + writer.key().varChar().writeVarChar("key5"); + writer.value().integer().writeInt(456); + writer.endEntry(); + nullBothWriter.endMap(); + + // Update count for the vectors + nullEntriesVector.setValueCount(3); + nullMapVector.setValueCount(3); + nullBothVector.setValueCount(3); + + File dataFile = new File(TMP, "testRoundTripNullableMap.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripStruct() throws Exception { + + // Field definitions + FieldType structFieldType = new FieldType(false, new ArrowType.Struct(), null); + Field intField = + new Field("intField", FieldType.notNullable(new ArrowType.Int(32, true)), null); + Field stringField = new Field("stringField", FieldType.notNullable(new ArrowType.Utf8()), null); + Field dateField = + new Field("dateField", FieldType.notNullable(new ArrowType.Date(DateUnit.DAY)), null); + + // Create empty vector + BufferAllocator allocator = new RootAllocator(); + StructVector structVector = new StructVector("struct", allocator, structFieldType, null); + structVector.initializeChildrenFromFields(Arrays.asList(intField, stringField, dateField)); + + // Set up VSR + List vectors = Arrays.asList(structVector); + int rowCount = 3; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data + BaseWriter.StructWriter structWriter = structVector.getWriter(); + + for (int i = 0; i < rowCount; i++) { + structWriter.start(); + structWriter.integer("intField").writeInt(i); + structWriter.varChar("stringField").writeVarChar("string" + i); + structWriter.dateDay("dateField").writeDateDay((int) LocalDate.now().toEpochDay() + i); + structWriter.end(); + } + + File dataFile = new File(TMP, "testRoundTripStruct.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } + + @Test + public void testRoundTripNullableStructs() throws Exception { + + // Field definitions + FieldType structFieldType = new FieldType(false, new ArrowType.Struct(), null); + FieldType nullableStructFieldType = new FieldType(true, new ArrowType.Struct(), null); + Field intField = + new Field("intField", FieldType.notNullable(new ArrowType.Int(32, true)), null); + Field nullableIntField = + new Field("nullableIntField", FieldType.nullable(new ArrowType.Int(32, true)), null); + + // Create empty vectors + BufferAllocator allocator = new RootAllocator(); + StructVector structVector = new StructVector("struct", allocator, structFieldType, null); + StructVector nullableStructVector = + new StructVector("nullableStruct", allocator, nullableStructFieldType, null); + structVector.initializeChildrenFromFields(Arrays.asList(intField, nullableIntField)); + nullableStructVector.initializeChildrenFromFields(Arrays.asList(intField, nullableIntField)); + + // Set up VSR + List vectors = Arrays.asList(structVector, nullableStructVector); + int rowCount = 4; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + root.setRowCount(rowCount); + root.allocateNew(); + + // Set test data for structVector + BaseWriter.StructWriter structWriter = structVector.getWriter(); + for (int i = 0; i < rowCount; i++) { + structWriter.setPosition(i); + structWriter.start(); + structWriter.integer("intField").writeInt(i); + if (i % 2 == 0) { + structWriter.integer("nullableIntField").writeInt(i * 10); + } else { + structWriter.integer("nullableIntField").writeNull(); + } + structWriter.end(); + } + + // Set test data for nullableStructVector + BaseWriter.StructWriter nullableStructWriter = nullableStructVector.getWriter(); + for (int i = 0; i < rowCount; i++) { + nullableStructWriter.setPosition(i); + if (i >= 2) { + nullableStructWriter.start(); + nullableStructWriter.integer("intField").writeInt(i); + if (i % 2 == 0) { + nullableStructWriter.integer("nullableIntField").writeInt(i * 10); + } else { + nullableStructWriter.integer("nullableIntField").writeNull(); + } + nullableStructWriter.end(); + } else { + nullableStructWriter.writeNull(); + } + } + + // Update count for the vector + structVector.setValueCount(rowCount); + nullableStructVector.setValueCount(rowCount); + + File dataFile = new File(TMP, "testRoundTripNullableStructs.avro"); + + roundTripTest(root, allocator, dataFile, rowCount); + } + } +} diff --git a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripSchemaTest.java b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripSchemaTest.java new file mode 100644 index 0000000000..864e2c8b59 --- /dev/null +++ b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripSchemaTest.java @@ -0,0 +1,443 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.apache.arrow.vector.types.DateUnit; +import org.apache.arrow.vector.types.FloatingPointPrecision; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.avro.Schema; +import org.junit.jupiter.api.Test; + +public class RoundTripSchemaTest { + + private void doRoundTripTest(List fields) { + + AvroToArrowConfig config = new AvroToArrowConfig(null, 1, null, Collections.emptySet(), false); + + Schema avroSchema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord"); + org.apache.arrow.vector.types.pojo.Schema arrowSchema = + AvroToArrowUtils.createArrowSchema(avroSchema, config); + + // Compare string representations - equality not defined for logical types + assertEquals(fields, arrowSchema.getFields()); + } + + // Schema round trip for primitive types, nullable and non-nullable + + @Test + public void testRoundTripNullType() { + + List fields = + Arrays.asList(new Field("nullType", FieldType.notNullable(new ArrowType.Null()), null)); + + doRoundTripTest(fields); + } + + @Test + public void testRoundTripBooleanType() { + + List fields = + Arrays.asList( + new Field("nullableBool", FieldType.nullable(new ArrowType.Bool()), null), + new Field("nonNullableBool", FieldType.notNullable(new ArrowType.Bool()), null)); + + doRoundTripTest(fields); + } + + @Test + public void testRoundTripIntegerTypes() { + + AvroToArrowConfig config = new AvroToArrowConfig(null, 1, null, Collections.emptySet(), false); + + // Only round trip types with direct equivalent in Avro + + List fields = + Arrays.asList( + new Field("nullableInt32", FieldType.nullable(new ArrowType.Int(32, true)), null), + new Field("nonNullableInt32", FieldType.notNullable(new ArrowType.Int(32, true)), null), + new Field("nullableInt64", FieldType.nullable(new ArrowType.Int(64, true)), null), + new Field( + "nonNullableInt64", FieldType.notNullable(new ArrowType.Int(64, true)), null)); + + Schema avroSchema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord"); + org.apache.arrow.vector.types.pojo.Schema arrowSchema = + AvroToArrowUtils.createArrowSchema(avroSchema, config); + + // Exact match on fields after round trip + assertEquals(fields, arrowSchema.getFields()); + } + + @Test + public void testRoundTripFloatingPointTypes() { + + // Only round trip types with direct equivalent in Avro + + List fields = + Arrays.asList( + new Field( + "nullableFloat32", + FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)), + null), + new Field( + "nonNullableFloat32", + FieldType.notNullable(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)), + null), + new Field( + "nullableFloat64", + FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + null), + new Field( + "nonNullableFloat64", + FieldType.notNullable(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + null)); + + doRoundTripTest(fields); + } + + @Test + public void testRoundTripStringTypes() { + + List fields = + Arrays.asList( + new Field("nullableUtf8", FieldType.nullable(new ArrowType.Utf8()), null), + new Field("nonNullableUtf8", FieldType.notNullable(new ArrowType.Utf8()), null)); + + doRoundTripTest(fields); + } + + @Test + public void testRoundTripBinaryTypes() { + + List fields = + Arrays.asList( + new Field("nullableBinary", FieldType.nullable(new ArrowType.Binary()), null), + new Field("nonNullableBinary", FieldType.notNullable(new ArrowType.Binary()), null)); + + doRoundTripTest(fields); + } + + @Test + public void testRoundTripFixedSizeBinaryTypes() { + + List fields = + Arrays.asList( + new Field( + "nullableFixedSizeBinary", + FieldType.nullable(new ArrowType.FixedSizeBinary(10)), + null), + new Field( + "nonNullableFixedSizeBinary", + FieldType.notNullable(new ArrowType.FixedSizeBinary(10)), + null)); + + doRoundTripTest(fields); + } + + // Schema round trip for logical types, nullable and non-nullable + + @Test + public void testRoundTripDecimalTypes() { + + List fields = + Arrays.asList( + new Field( + "nullableDecimal128", FieldType.nullable(new ArrowType.Decimal(10, 2, 128)), null), + new Field( + "nonNullableDecimal1281", + FieldType.notNullable(new ArrowType.Decimal(10, 2, 128)), + null), + new Field( + "nonNullableDecimal1282", + FieldType.notNullable(new ArrowType.Decimal(15, 5, 128)), + null), + new Field( + "nonNullableDecimal1283", + FieldType.notNullable(new ArrowType.Decimal(20, 10, 128)), + null), + new Field( + "nullableDecimal256", FieldType.nullable(new ArrowType.Decimal(55, 15, 256)), null), + new Field( + "nonNullableDecimal2561", + FieldType.notNullable(new ArrowType.Decimal(55, 25, 256)), + null), + new Field( + "nonNullableDecimal2562", + FieldType.notNullable(new ArrowType.Decimal(25, 8, 256)), + null), + new Field( + "nonNullableDecimal2563", + FieldType.notNullable(new ArrowType.Decimal(60, 50, 256)), + null)); + + doRoundTripTest(fields); + } + + @Test + public void testRoundTripDateTypes() { + + List fields = + Arrays.asList( + new Field( + "nullableDateDay", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null), + new Field( + "nonNullableDateDay", + FieldType.notNullable(new ArrowType.Date(DateUnit.DAY)), + null)); + + doRoundTripTest(fields); + } + + @Test + public void testRoundTripTimeTypes() { + + List fields = + Arrays.asList( + new Field( + "nullableTimeMillis", + FieldType.nullable(new ArrowType.Time(TimeUnit.MILLISECOND, 32)), + null), + new Field( + "nonNullableTimeMillis", + FieldType.notNullable(new ArrowType.Time(TimeUnit.MILLISECOND, 32)), + null), + new Field( + "nullableTimeMicros", + FieldType.nullable(new ArrowType.Time(TimeUnit.MICROSECOND, 64)), + null), + new Field( + "nonNullableTimeMicros", + FieldType.notNullable(new ArrowType.Time(TimeUnit.MICROSECOND, 64)), + null)); + + doRoundTripTest(fields); + } + + @Test + public void testRoundTripZoneAwareTimestampTypes() { + + List fields = + Arrays.asList( + new Field( + "nullableTimestampMillisTz", + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC")), + null), + new Field( + "nonNullableTimestampMillisTz", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC")), + null), + new Field( + "nullableTimestampMicrosTz", + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC")), + null), + new Field( + "nonNullableTimestampMicrosTz", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC")), + null), + new Field( + "nullableTimestampNanosTz", + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.NANOSECOND, "UTC")), + null), + new Field( + "nonNullableTimestampNanosTz", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.NANOSECOND, "UTC")), + null)); + + doRoundTripTest(fields); + } + + @Test + public void testRoundTripLocalTimestampTypes() { + + List fields = + Arrays.asList( + new Field( + "nullableTimestampMillis", + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, null)), + null), + new Field( + "nonNullableTimestampMillis", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, null)), + null), + new Field( + "nullableTimestampMicros", + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MICROSECOND, null)), + null), + new Field( + "nonNullableTimestampMicros", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.MICROSECOND, null)), + null), + new Field( + "nullableTimestampNanos", + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.NANOSECOND, null)), + null), + new Field( + "nonNullableTimestampNanos", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.NANOSECOND, null)), + null)); + + doRoundTripTest(fields); + } + + // Schema round trip for complex types, where the contents are primitive and logical types + + @Test + public void testRoundTripListType() { + + List fields = + Arrays.asList( + new Field( + "nullableIntList", + FieldType.nullable(new ArrowType.List()), + Arrays.asList( + new Field("$data$", FieldType.nullable(new ArrowType.Int(32, true)), null))), + new Field( + "nullableDoubleList", + FieldType.nullable(new ArrowType.List()), + Arrays.asList( + new Field( + "$data$", + FieldType.notNullable( + new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + null))), + new Field( + "nonNullableDecimalList", + FieldType.notNullable(new ArrowType.List()), + Arrays.asList( + new Field( + "$data$", FieldType.nullable(new ArrowType.Decimal(10, 2, 128)), null))), + new Field( + "nonNullableTimestampList", + FieldType.notNullable(new ArrowType.List()), + Arrays.asList( + new Field( + "$data$", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC")), + null)))); + + doRoundTripTest(fields); + } + + @Test + public void testRoundTripMapType() { + + List fields = + Arrays.asList( + new Field( + "nullableMapWithNullableInt", + FieldType.nullable(new ArrowType.Map(false)), + Arrays.asList( + new Field( + "entries", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList( + new Field("key", FieldType.notNullable(new ArrowType.Utf8()), null), + new Field( + "value", FieldType.nullable(new ArrowType.Int(32, true)), null))))), + new Field( + "nullableMapWithNonNullableDouble", + FieldType.nullable(new ArrowType.Map(false)), + Arrays.asList( + new Field( + "entries", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList( + new Field("key", FieldType.notNullable(new ArrowType.Utf8()), null), + new Field( + "value", + FieldType.notNullable( + new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + null))))), + new Field( + "nonNullableMapWithNullableDecimal", + FieldType.notNullable(new ArrowType.Map(false)), + Arrays.asList( + new Field( + "entries", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList( + new Field("key", FieldType.notNullable(new ArrowType.Utf8()), null), + new Field( + "value", + FieldType.nullable(new ArrowType.Decimal(10, 2, 128)), + null))))), + new Field( + "nonNullableMapWithNonNullableTimestamp", + FieldType.notNullable(new ArrowType.Map(false)), + Arrays.asList( + new Field( + "entries", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList( + new Field("key", FieldType.notNullable(new ArrowType.Utf8()), null), + new Field( + "value", + FieldType.notNullable( + new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC")), + null)))))); + + doRoundTripTest(fields); + } + + @Test + public void testRoundTripStructType() { + + List fields = + Arrays.asList( + new Field( + "nullableRecord", + FieldType.nullable(new ArrowType.Struct()), + Arrays.asList( + new Field("field1", FieldType.nullable(new ArrowType.Int(32, true)), null), + new Field( + "field2", + FieldType.notNullable( + new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + null), + new Field( + "field3", FieldType.nullable(new ArrowType.Decimal(10, 2, 128)), null), + new Field( + "field4", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC")), + null))), + new Field( + "nonNullableRecord", + FieldType.notNullable(new ArrowType.Struct()), + Arrays.asList( + new Field("field1", FieldType.nullable(new ArrowType.Int(32, true)), null), + new Field( + "field2", + FieldType.notNullable( + new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + null), + new Field( + "field3", FieldType.nullable(new ArrowType.Decimal(10, 2, 128)), null), + new Field( + "field4", + FieldType.notNullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC")), + null)))); + + doRoundTripTest(fields); + } +} diff --git a/adapter/avro/src/test/resources/schema/logical/test_decimal_invalid1.avsc b/adapter/avro/src/test/resources/schema/logical/test_decimal_invalid1.avsc index 18d7d63fc7..c1867811c7 100644 --- a/adapter/avro/src/test/resources/schema/logical/test_decimal_invalid1.avsc +++ b/adapter/avro/src/test/resources/schema/logical/test_decimal_invalid1.avsc @@ -20,6 +20,6 @@ "name": "test", "type": "bytes", "logicalType" : "decimal", - "precision": 39, + "precision": 77, "scale": 2 } diff --git a/adapter/avro/src/test/resources/schema/logical/test_local_timestamp_micros.avsc b/adapter/avro/src/test/resources/schema/logical/test_local_timestamp_micros.avsc new file mode 100644 index 0000000000..db456e8a84 --- /dev/null +++ b/adapter/avro/src/test/resources/schema/logical/test_local_timestamp_micros.avsc @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "name": "test", + "type": "long", + "logicalType" : "local-timestamp-micros" +} diff --git a/adapter/avro/src/test/resources/schema/logical/test_local_timestamp_millis.avsc b/adapter/avro/src/test/resources/schema/logical/test_local_timestamp_millis.avsc new file mode 100644 index 0000000000..6a3cf9bccb --- /dev/null +++ b/adapter/avro/src/test/resources/schema/logical/test_local_timestamp_millis.avsc @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "name": "test", + "type": "long", + "logicalType" : "local-timestamp-millis" +} diff --git a/adapter/avro/src/test/resources/schema/logical/test_local_timestamp_nanos.avsc b/adapter/avro/src/test/resources/schema/logical/test_local_timestamp_nanos.avsc new file mode 100644 index 0000000000..96ca8bbfa4 --- /dev/null +++ b/adapter/avro/src/test/resources/schema/logical/test_local_timestamp_nanos.avsc @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "name": "test", + "type": "long", + "logicalType" : "local-timestamp-nanos" +} diff --git a/adapter/avro/src/test/resources/schema/logical/test_timestamp_nanos.avsc b/adapter/avro/src/test/resources/schema/logical/test_timestamp_nanos.avsc new file mode 100644 index 0000000000..9e05eab408 --- /dev/null +++ b/adapter/avro/src/test/resources/schema/logical/test_timestamp_nanos.avsc @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "namespace": "org.apache.arrow.avro", + "name": "test", + "type": "long", + "logicalType" : "timestamp-nanos" +} From 0d296dffd39e14dbcf288c2da873777e08091cbd Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 23 Apr 2025 10:29:08 +0900 Subject: [PATCH 014/271] GH-587: [Release] Add .env description to dev/release/README.md (#724) ## What's Changed Add missing `dev/release/.env` description to `dev/release/README.md`. This also moves `GH_TOKEN` to `dev/release/.env`. Closes #587. --- dev/release/.env.example | 5 ++ dev/release/README.md | 99 +++++++++++++++++++++++-------------- dev/release/bump_version.sh | 7 +++ dev/release/release.sh | 7 +++ dev/release/release_rc.sh | 14 +++--- 5 files changed, 89 insertions(+), 43 deletions(-) diff --git a/dev/release/.env.example b/dev/release/.env.example index cc7fd58bfc..4a57e34ed3 100644 --- a/dev/release/.env.example +++ b/dev/release/.env.example @@ -15,6 +15,11 @@ # specific language governing permissions and limitations # under the License. +# The GitHub token to upload artifacts to GitHub Release. +# +# You must set this. +#GH_TOKEN=secret + # The GPG key ID to sign artifacts. The GPG key ID must be registered # to both of the followings: # diff --git a/dev/release/README.md b/dev/release/README.md index 9069a4fdaf..8aee0fd106 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -27,45 +27,38 @@ 4. Publish (detailed later) 5. Bump version for new development (detailed later) -### Bump version for new release +### Prepare release environment -Run `dev/release/bump_version.sh` on a working copy of your fork not -`git@github.com:apache/arrow-java`: +This step is needed only when you act as a release manager first time. -```console -$ git clone git@github.com:${YOUR_GITHUB_ACCOUNT}/arrow-java.git arrow-java.${YOUR_GITHUB_ACCOUNT} -$ cd arrow-java.${YOUR_GITHUB_ACCOUNT} -$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/bump_version.sh ${NEW_VERSION} -``` +We use the following variables in multiple steps: -Here is an example to bump version to 19.0.0: +* `GH_TOKEN`: GitHub personal access token to automate GitHub related + operations +* `GPG_KEY_ID`: PGP key ID that is used for signing official artifacts + by GnuPG -``` -$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/bump_version.sh 19.0.0 -``` +We use `dev/release/.env` to share these variables in multiple +steps. You can use `dev/release/.env.example` as a template: -It creates a feature branch and adds a commit that bumps version. This -opens a pull request from the feature branch by `gh pr create`. So you -need `gh` command and GitHub personal access token. +```console +$ cp dev/release/.env{.example,} +$ chmod go-r dev/release/.env +$ editor dev/release/.env +``` -See also: +See https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens +how to prepare GitHub personal access token for `GH_TOKEN`. -We need to merge the pull request before we cut a RC. If we try cut a -RC without merging the pull request, the script to cut a RC is failed. +Note that you also need to install `gh` command because our scripts +use `gh` command to use GitHub API. See +https://github.com/cli/cli#installation how to install `gh` +command. -### Prepare RC and vote - -You can use `dev/release/release_rc.sh`. - -Requirements to run `release_rc.sh`: - - * You must be an Apache Arrow committer or PMC member - * You must prepare your PGP key for signing - * You must configure Maven - -If you don't have a PGP key, -https://infra.apache.org/release-signing.html#generate may be helpful. +If you don't have a PGP key for `GPG_KEY_ID`, see +https://infra.apache.org/release-signing.html#genegrate how to +generate your PGP key. Your PGP key must be registered to the followings: @@ -85,6 +78,40 @@ $ head KEYS $ svn ci KEYS ``` +### Bump version for new release + +Run `dev/release/bump_version.sh` on a working copy of your fork not +`git@github.com:apache/arrow-java`: + +```console +$ git clone git@github.com:${YOUR_GITHUB_ACCOUNT}/arrow-java.git arrow-java.${YOUR_GITHUB_ACCOUNT} +$ cd arrow-java.${YOUR_GITHUB_ACCOUNT} +$ dev/release/bump_version.sh ${NEW_VERSION} +``` + +Here is an example to bump version to 19.0.0: + +``` +$ dev/release/bump_version.sh 19.0.0 +``` + +It creates a feature branch and adds a commit that bumps version. This +opens a pull request from the feature branch. + +We need to merge the pull request before we cut a RC. If we try +cutting a RC without merging the pull request, the script to cut a RC +is failed. + +### Prepare RC and vote + +You can use `dev/release/release_rc.sh`. + +Requirements to run `release_rc.sh`: + + * You must be an Apache Arrow committer or PMC member + * You must prepare your PGP key for signing + * You must configure Maven + Configure Maven to publish artifacts to Apache repositories. You will need to setup a master password at `~/.m2/settings-security.xml` and `~/.m2/settings.xml` as specified on [the Apache @@ -102,7 +129,7 @@ Run `dev/release/release_rc.sh` on a working copy of ```console $ git clone git@github.com:apache/arrow-java.git $ cd arrow-java -$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release_rc.sh ${RC} +$ dev/release/release_rc.sh ${RC} (Send a vote email to dev@arrow.apache.org. You can use a draft shown by release_rc.sh for the email.) ``` @@ -110,7 +137,7 @@ $ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release_rc.sh ${RC} Here is an example to release RC1: ```console -$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release_rc.sh 1 +$ dev/release/release_rc.sh 1 ``` The argument of `release_rc.sh` is the RC number. If RC1 has a @@ -128,13 +155,13 @@ Run `dev/release/release.sh` on a working copy of archive to apache.org: ```console -$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release.sh ${VERSION} ${RC} +$ dev/release/release.sh ${VERSION} ${RC} ``` Here is an example to release 19.0.0 RC1: ```console -$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release.sh 19.0.0 1 +$ dev/release/release.sh 19.0.0 1 ``` Add the release to ASF's report database via [Apache Committee Report @@ -160,13 +187,13 @@ Run `dev/release/bump_version.sh` on a working copy of your fork not ```console $ git clone git@github.com:${YOUR_GITHUB_ACCOUNT}/arrow-java.git arrow-java.${YOUR_GITHUB_ACCOUNT} $ cd arrow-java.${YOUR_GITHUB_ACCOUNT} -$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/bump_version.sh ${NEW_VERSION}-SNAPSHOT +$ dev/release/bump_version.sh ${NEW_VERSION}-SNAPSHOT ``` Here is an example to bump version to 19.0.1-SNAPSHOT: ``` -$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/bump_version.sh 19.0.0-SNAPSHOT +$ dev/release/bump_version.sh 19.0.0-SNAPSHOT ``` It creates a feature branch and adds a commit that bumps version. This diff --git a/dev/release/bump_version.sh b/dev/release/bump_version.sh index d1c127de35..458e930f98 100755 --- a/dev/release/bump_version.sh +++ b/dev/release/bump_version.sh @@ -31,6 +31,13 @@ fi version=$1 +if [ ! -f "${SOURCE_DIR}/.env" ]; then + echo "You must create ${SOURCE_DIR}/.env" + echo "You can use ${SOURCE_DIR}/.env.example as template" + exit 1 +fi +. "${SOURCE_DIR}/.env" + cd "${SOURCE_TOP_DIR}" git_origin_url="$(git remote get-url origin)" diff --git a/dev/release/release.sh b/dev/release/release.sh index c5e67907e2..70e1f96454 100755 --- a/dev/release/release.sh +++ b/dev/release/release.sh @@ -28,6 +28,13 @@ fi version=$1 rc=$2 +if [ ! -f "${SOURCE_DIR}/.env" ]; then + echo "You must create ${SOURCE_DIR}/.env" + echo "You can use ${SOURCE_DIR}/.env.example as template" + exit 1 +fi +. "${SOURCE_DIR}/.env" + git_origin_url="$(git remote get-url origin)" repository="${git_origin_url#*github.com?}" repository="${repository%.git}" diff --git a/dev/release/release_rc.sh b/dev/release/release_rc.sh index b0107dce85..ff77718b8d 100755 --- a/dev/release/release_rc.sh +++ b/dev/release/release_rc.sh @@ -36,6 +36,13 @@ rc=$1 : "${RELEASE_SIGN:=${RELEASE_DEFAULT}}" : "${RELEASE_UPLOAD:=${RELEASE_DEFAULT}}" +if [ ! -f "${SOURCE_DIR}/.env" ]; then + echo "You must create ${SOURCE_DIR}/.env" + echo "You can use ${SOURCE_DIR}/.env.example as template" + exit 1 +fi +. "${SOURCE_DIR}/.env" + cd "${SOURCE_TOP_DIR}" if [ "${RELEASE_PULL}" -gt 0 ] || [ "${RELEASE_PUSH_TAG}" -gt 0 ]; then @@ -84,13 +91,6 @@ artifacts_dir="apache-arrow-java-${version}-rc${rc}" signed_artifacts_dir="${artifacts_dir}-signed" if [ "${RELEASE_SIGN}" -gt 0 ]; then - if [ ! -f "${SOURCE_DIR}/.env" ]; then - echo "You must create ${SOURCE_DIR}/.env" - echo "You can use ${SOURCE_DIR}/.env.example as template" - exit 1 - fi - . "${SOURCE_DIR}/.env" - git_origin_url="$(git remote get-url origin)" repository="${git_origin_url#*github.com?}" repository="${repository%.git}" From 26630c38f563821aa40d1c5a6df30101dde8c171 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Giraldo?= Date: Sun, 27 Apr 2025 23:28:37 -0600 Subject: [PATCH 015/271] GH-463: Improve TZ support for JDBC driver (#464) This PR adds support for natively fetching `java.time.*` objects through the JDBC driver. DateVector - getObject(LocalDate.class) DateTimeVector - getObject(OffsetDateTime.class) - getObject(LocalDateTime.class) - getObject(ZonedDateTime.class) - getObject(Instant.class) TimeVector - getObject(LocalTime.class) This PR also changes the behavior for vectors that include TZ info. These will now return as `TIMESTAMP_WITH_TIMEZONE`. The behavior for different ways to access a TimeStampVector are as follows: | | Vector with TZ | Vector W/O TZ | |---------------------------------|------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------| | getTimestamp() | Get timestamp in the vector TZ | Get timestamp in UTC | | getTimestamp(calendar) | Get timestamp by adjusting from the vector TZ to the desired calendar TZ | Get timestamp by adjusting from UTC to the desired calendar TZ (a bug, IMO) | | getObject(LocalDateTime.class) | Get LocalDateTime by taking the timestamp at the vector TZ and taking the "wall-clock" time at that moment | Treat the epoch value in the vector as the "wall-clock" time | | getObject(Instant.class) | Get Instant represented by the value in the vector TZ | Get Instant represented by the value in UTC | | getObject(OffsetDateTime.class) | Get OffsetDateTime represented by the value in the vector TZ (will account for daylight adjustment) | Get OffsetDateTime represented by the value in UTC (will account for daylight adjustment) | | getObject(ZonedDateTime.class) | Get ZonedDateTime represented by the value in the vector at in its TZ | Get ZonedDateTime represented by the value in the vector at in UTC | Closes #463 --- .../ArrowFlightJdbcDateVectorAccessor.java | 19 +++ ...rrowFlightJdbcTimeStampVectorAccessor.java | 101 +++++++++++++-- .../ArrowFlightJdbcTimeVectorAccessor.java | 19 +++ .../arrow/driver/jdbc/utils/SqlTypes.java | 8 +- .../jdbc/ArrowDatabaseMetadataTest.java | 5 +- ...FlightJdbcTimeStampVectorAccessorTest.java | 118 +++++++++++++++++- .../arrow/driver/jdbc/utils/SqlTypesTest.java | 12 ++ 7 files changed, 269 insertions(+), 13 deletions(-) diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcDateVectorAccessor.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcDateVectorAccessor.java index ebe4016209..cdafeffc32 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcDateVectorAccessor.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcDateVectorAccessor.java @@ -24,7 +24,9 @@ import static org.apache.calcite.avatica.util.DateTimeUtils.unixDateToString; import java.sql.Date; +import java.sql.SQLException; import java.sql.Timestamp; +import java.time.LocalDate; import java.util.Calendar; import java.util.concurrent.TimeUnit; import java.util.function.IntSupplier; @@ -85,6 +87,19 @@ public Object getObject() { return this.getDate(null); } + @Override + public T getObject(final Class type) throws SQLException { + final Object value; + if (type == LocalDate.class) { + value = getLocalDate(); + } else if (type == Date.class) { + value = getObject(); + } else { + throw new SQLException("Object type not supported for Date Vector"); + } + return !type.isPrimitive() && wasNull ? null : type.cast(value); + } + @Override public Date getDate(Calendar calendar) { fillHolder(); @@ -134,4 +149,8 @@ protected static TimeUnit getTimeUnitForVector(ValueVector vector) { throw new IllegalArgumentException("Invalid Arrow vector"); } + + private LocalDate getLocalDate() { + return getDate(null).toLocalDate(); + } } diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeStampVectorAccessor.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeStampVectorAccessor.java index debdd0fcb4..813fbc7cfd 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeStampVectorAccessor.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeStampVectorAccessor.java @@ -21,11 +21,18 @@ import static org.apache.arrow.driver.jdbc.accessor.impl.calendar.ArrowFlightJdbcTimeStampVectorGetter.createGetter; import java.sql.Date; +import java.sql.SQLException; import java.sql.Time; import java.sql.Timestamp; +import java.time.Instant; import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.time.temporal.ChronoUnit; import java.util.Calendar; +import java.util.Objects; +import java.util.Set; import java.util.TimeZone; import java.util.concurrent.TimeUnit; import java.util.function.IntSupplier; @@ -43,6 +50,7 @@ public class ArrowFlightJdbcTimeStampVectorAccessor extends ArrowFlightJdbcAcces private final TimeUnit timeUnit; private final LongToLocalDateTime longToLocalDateTime; private final Holder holder; + private final boolean isZoned; /** Functional interface used to convert a number (in any time resolution) to LocalDateTime. */ interface LongToLocalDateTime { @@ -58,6 +66,9 @@ public ArrowFlightJdbcTimeStampVectorAccessor( this.holder = new Holder(); this.getter = createGetter(vector); + // whether the vector included TZ info + this.isZoned = getVectorIsZoned(vector); + // non-null, either the vector TZ or default to UTC this.timeZone = getTimeZoneForVector(vector); this.timeUnit = getTimeUnitForVector(vector); this.longToLocalDateTime = getLongToLocalDateTimeForVector(vector, this.timeZone); @@ -68,11 +79,62 @@ public Class getObjectClass() { return Timestamp.class; } + @Override + public T getObject(final Class type) throws SQLException { + final Object value; + if (!this.isZoned + & Set.of(OffsetDateTime.class, ZonedDateTime.class, Instant.class).contains(type)) { + throw new SQLException( + "Vectors without timezones can't be converted to objects with offset/tz info."); + } else if (type == OffsetDateTime.class) { + value = getOffsetDateTime(); + } else if (type == LocalDateTime.class) { + value = getLocalDateTime(null); + } else if (type == ZonedDateTime.class) { + value = getZonedDateTime(); + } else if (type == Instant.class) { + value = getInstant(); + } else if (type == Timestamp.class) { + value = getObject(); + } else { + throw new SQLException("Object type not supported for TimeStamp Vector"); + } + + return !type.isPrimitive() && wasNull ? null : type.cast(value); + } + @Override public Object getObject() { return this.getTimestamp(null); } + private ZonedDateTime getZonedDateTime() { + LocalDateTime localDateTime = getLocalDateTime(null); + if (localDateTime == null) { + return null; + } + + return localDateTime.atZone(this.timeZone.toZoneId()); + } + + private OffsetDateTime getOffsetDateTime() { + LocalDateTime localDateTime = getLocalDateTime(null); + if (localDateTime == null) { + return null; + } + ZoneOffset offset = this.timeZone.toZoneId().getRules().getOffset(localDateTime); + return localDateTime.atOffset(offset); + } + + private Instant getInstant() { + LocalDateTime localDateTime = getLocalDateTime(null); + if (localDateTime == null) { + return null; + } + ZoneOffset offset = this.timeZone.toZoneId().getRules().getOffset(localDateTime); + return localDateTime.toInstant(offset); + } + private LocalDateTime getLocalDateTime(Calendar calendar) { getter.get(getCurrentRow(), holder); this.wasNull = holder.isSet == 0; @@ -85,7 +147,9 @@ private LocalDateTime getLocalDateTime(Calendar calendar) { LocalDateTime localDateTime = this.longToLocalDateTime.fromLong(value); - if (calendar != null) { + // Adjust timestamp to desired calendar (if provided) only if the column includes TZ info, + // otherwise treat as wall-clock time + if (calendar != null && this.isZoned) { TimeZone timeZone = calendar.getTimeZone(); long millis = this.timeUnit.toMillis(value); localDateTime = @@ -102,7 +166,7 @@ public Date getDate(Calendar calendar) { return null; } - return new Date(Timestamp.valueOf(localDateTime).getTime()); + return new Date(getTimestampWithOffset(calendar, localDateTime).getTime()); } @Override @@ -112,7 +176,7 @@ public Time getTime(Calendar calendar) { return null; } - return new Time(Timestamp.valueOf(localDateTime).getTime()); + return new Time(getTimestampWithOffset(calendar, localDateTime).getTime()); } @Override @@ -122,6 +186,24 @@ public Timestamp getTimestamp(Calendar calendar) { return null; } + return getTimestampWithOffset(calendar, localDateTime); + } + + /** + * Apply offset to LocalDateTime to get a Timestamp with legacy behavior. Previously we applied + * the offset to the LocalDateTime even if the underlying Vector did not have a TZ. In order to + * support java.time.* accessors, we fixed this so we only apply the offset if the underlying + * vector includes TZ info. In order to maintain backward compatibility, we apply the offset if + * needed for getDate, getTime, and getTimestamp. + */ + private Timestamp getTimestampWithOffset(Calendar calendar, LocalDateTime localDateTime) { + if (calendar != null && !isZoned) { + TimeZone timeZone = calendar.getTimeZone(); + long millis = Timestamp.valueOf(localDateTime).getTime(); + localDateTime = + localDateTime.minus( + timeZone.getOffset(millis) - this.timeZone.getOffset(millis), ChronoUnit.MILLIS); + } return Timestamp.valueOf(localDateTime); } @@ -170,11 +252,14 @@ protected static TimeZone getTimeZoneForVector(TimeStampVector vector) { ArrowType.Timestamp arrowType = (ArrowType.Timestamp) vector.getField().getFieldType().getType(); - String timezoneName = arrowType.getTimezone(); - if (timezoneName == null) { - return TimeZone.getTimeZone("UTC"); - } - + String timezoneName = Objects.requireNonNullElse(arrowType.getTimezone(), "UTC"); return TimeZone.getTimeZone(timezoneName); } + + protected static boolean getVectorIsZoned(TimeStampVector vector) { + ArrowType.Timestamp arrowType = + (ArrowType.Timestamp) vector.getField().getFieldType().getType(); + + return arrowType.getTimezone() != null; + } } diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeVectorAccessor.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeVectorAccessor.java index 2c03ee631e..d525c2fdd2 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeVectorAccessor.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeVectorAccessor.java @@ -20,8 +20,10 @@ import static org.apache.arrow.driver.jdbc.accessor.impl.calendar.ArrowFlightJdbcTimeVectorGetter.Holder; import static org.apache.arrow.driver.jdbc.accessor.impl.calendar.ArrowFlightJdbcTimeVectorGetter.createGetter; +import java.sql.SQLException; import java.sql.Time; import java.sql.Timestamp; +import java.time.LocalTime; import java.util.Calendar; import java.util.concurrent.TimeUnit; import java.util.function.IntSupplier; @@ -121,6 +123,19 @@ public Object getObject() { return this.getTime(null); } + @Override + public T getObject(final Class type) throws SQLException { + final Object value; + if (type == LocalTime.class) { + value = getLocalTime(); + } else if (type == Time.class) { + value = getObject(); + } else { + throw new SQLException("Object type not supported for Time Vector"); + } + return !type.isPrimitive() && wasNull ? null : type.cast(value); + } + @Override public Time getTime(Calendar calendar) { fillHolder(); @@ -134,6 +149,10 @@ public Time getTime(Calendar calendar) { return new ArrowFlightJdbcTime(DateTimeUtils.applyCalendarOffset(milliseconds, calendar)); } + private LocalTime getLocalTime() { + return getTime(null).toLocalTime(); + } + private void fillHolder() { getter.get(getCurrentRow(), holder); this.wasNull = holder.isSet == 0; diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/SqlTypes.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/SqlTypes.java index 96cb056db2..1b76ca0c95 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/SqlTypes.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/SqlTypes.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.driver.jdbc.utils; +import com.google.common.base.Strings; import java.sql.Types; import java.util.HashMap; import java.util.Map; @@ -120,7 +121,12 @@ public static int getSqlTypeIdFromArrowType(ArrowType arrowType) { case Time: return Types.TIME; case Timestamp: - return Types.TIMESTAMP; + String tz = ((ArrowType.Timestamp) arrowType).getTimezone(); + if (Strings.isNullOrEmpty(tz)) { + return Types.TIMESTAMP; + } else { + return Types.TIMESTAMP_WITH_TIMEZONE; + } case Bool: return Types.BOOLEAN; case Decimal: diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java index 88a172e4f2..70d3bcbd33 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java @@ -299,8 +299,9 @@ public class ArrowDatabaseMetadataTest { private static Connection connection; static { - List expectedGetColumnsDataTypes = Arrays.asList(3, 93, 4); - List expectedGetColumnsTypeName = Arrays.asList("DECIMAL", "TIMESTAMP", "INTEGER"); + List expectedGetColumnsDataTypes = Arrays.asList(3, 2014, 4); + List expectedGetColumnsTypeName = + Arrays.asList("DECIMAL", "TIMESTAMP_WITH_TIMEZONE", "INTEGER"); List expectedGetColumnsRadix = Arrays.asList(10, null, 10); List expectedGetColumnsColumnSize = Arrays.asList(5, 29, 10); List expectedGetColumnsDecimalDigits = Arrays.asList(2, 9, 0); diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeStampVectorAccessorTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeStampVectorAccessorTest.java index 2e329f148e..e4863bd80e 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeStampVectorAccessorTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeStampVectorAccessorTest.java @@ -16,17 +16,23 @@ */ package org.apache.arrow.driver.jdbc.accessor.impl.calendar; -import static org.apache.arrow.driver.jdbc.accessor.impl.calendar.ArrowFlightJdbcTimeStampVectorAccessor.getTimeUnitForVector; -import static org.apache.arrow.driver.jdbc.accessor.impl.calendar.ArrowFlightJdbcTimeStampVectorAccessor.getTimeZoneForVector; +import static org.apache.arrow.driver.jdbc.accessor.impl.calendar.ArrowFlightJdbcTimeStampVectorAccessor.*; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.sql.Date; +import java.sql.SQLException; import java.sql.Time; import java.sql.Timestamp; +import java.time.Instant; import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.Calendar; +import java.util.Objects; import java.util.TimeZone; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; @@ -199,6 +205,99 @@ public void testShouldGetTimestampReturnValidTimestampWithCalendar( }); } + @ParameterizedTest + @MethodSource("data") + public void testShouldGetObjectReturnValidLocalDateTime( + Supplier vectorSupplier, String vectorType, String timeZone) + throws Exception { + setup(vectorSupplier); + final String expectedTimeZone = Objects.requireNonNullElse(timeZone, "UTC"); + + accessorIterator.iterate( + vector, + (accessor, currentRow) -> { + final LocalDateTime value = accessor.getObject(LocalDateTime.class); + final LocalDateTime expectedValue = + getZonedDateTime(currentRow, expectedTimeZone).toLocalDateTime(); + + assertThat(value, equalTo(expectedValue)); + assertThat(accessor.wasNull(), is(false)); + }); + } + + @ParameterizedTest + @MethodSource("data") + public void testShouldGetObjectReturnValidInstant( + Supplier vectorSupplier, String vectorType, String timeZone) + throws Exception { + setup(vectorSupplier); + final String expectedTimeZone = Objects.requireNonNullElse(timeZone, "UTC"); + final boolean vectorHasTz = timeZone != null; + accessorIterator.iterate( + vector, + (accessor, currentRow) -> { + if (vectorHasTz) { + final Instant value = accessor.getObject(Instant.class); + final Instant expectedValue = + getZonedDateTime(currentRow, expectedTimeZone).toInstant(); + + assertThat(value, equalTo(expectedValue)); + assertThat(accessor.wasNull(), is(false)); + } else { + assertThrows(SQLException.class, () -> accessor.getObject(Instant.class)); + } + }); + } + + @ParameterizedTest + @MethodSource("data") + public void testShouldGetObjectReturnValidOffsetDateTime( + Supplier vectorSupplier, String vectorType, String timeZone) + throws Exception { + setup(vectorSupplier); + final String expectedTimeZone = Objects.requireNonNullElse(timeZone, "UTC"); + final boolean vectorHasTz = timeZone != null; + accessorIterator.iterate( + vector, + (accessor, currentRow) -> { + if (vectorHasTz) { + final OffsetDateTime value = accessor.getObject(OffsetDateTime.class); + final OffsetDateTime expectedValue = + getZonedDateTime(currentRow, expectedTimeZone).toOffsetDateTime(); + + assertThat(value, equalTo(expectedValue)); + assertThat(value.getOffset(), equalTo(expectedValue.getOffset())); + assertThat(accessor.wasNull(), is(false)); + } else { + assertThrows(SQLException.class, () -> accessor.getObject(OffsetDateTime.class)); + } + }); + } + + @ParameterizedTest + @MethodSource("data") + public void testShouldGetObjectReturnValidZonedDateTime( + Supplier vectorSupplier, String vectorType, String timeZone) + throws Exception { + setup(vectorSupplier); + final String expectedTimeZone = Objects.requireNonNullElse(timeZone, "UTC"); + final boolean vectorHasTz = timeZone != null; + accessorIterator.iterate( + vector, + (accessor, currentRow) -> { + if (vectorHasTz) { + final ZonedDateTime value = accessor.getObject(ZonedDateTime.class); + final ZonedDateTime expectedValue = getZonedDateTime(currentRow, expectedTimeZone); + + assertThat(value, equalTo(expectedValue)); + assertThat(value.getZone(), equalTo(ZoneId.of(expectedTimeZone))); + assertThat(accessor.wasNull(), is(false)); + } else { + assertThrows(SQLException.class, () -> accessor.getObject(ZonedDateTime.class)); + } + }); + } + @ParameterizedTest @MethodSource("data") public void testShouldGetTimestampReturnNull(Supplier vectorSupplier) { @@ -320,6 +419,21 @@ private Timestamp getTimestampForVector(int currentRow, String timeZone) { return expectedTimestamp; } + /** ZonedDateTime contains all necessary information to generate any java.time object. */ + private ZonedDateTime getZonedDateTime(int currentRow, String timeZone) { + Object object = vector.getObject(currentRow); + TimeZone tz = TimeZone.getTimeZone(timeZone); + ZonedDateTime expectedTimestamp = null; + if (object instanceof LocalDateTime) { + expectedTimestamp = ((LocalDateTime) object).atZone(tz.toZoneId()); + } else if (object instanceof Long) { + TimeUnit timeUnit = getTimeUnitForVector(vector); + Instant instant = Instant.ofEpochMilli(timeUnit.toMillis((Long) object)); + expectedTimestamp = ZonedDateTime.ofInstant(instant, tz.toZoneId()); + } + return expectedTimestamp; + } + @ParameterizedTest @MethodSource("data") public void testShouldGetObjectClass(Supplier vectorSupplier) throws Exception { diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/SqlTypesTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/SqlTypesTest.java index 00af3c96ba..a6dd6b3275 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/SqlTypesTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/SqlTypesTest.java @@ -48,9 +48,15 @@ public void testGetSqlTypeIdFromArrowType() { assertEquals(Types.DATE, getSqlTypeIdFromArrowType(new ArrowType.Date(DateUnit.MILLISECOND))); assertEquals( Types.TIME, getSqlTypeIdFromArrowType(new ArrowType.Time(TimeUnit.MILLISECOND, 32))); + assertEquals( + Types.TIMESTAMP, + getSqlTypeIdFromArrowType(new ArrowType.Timestamp(TimeUnit.MILLISECOND, null))); assertEquals( Types.TIMESTAMP, getSqlTypeIdFromArrowType(new ArrowType.Timestamp(TimeUnit.MILLISECOND, ""))); + assertEquals( + Types.TIMESTAMP_WITH_TIMEZONE, + getSqlTypeIdFromArrowType(new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC"))); assertEquals(Types.BOOLEAN, getSqlTypeIdFromArrowType(new ArrowType.Bool())); @@ -95,9 +101,15 @@ public void testGetSqlTypeNameFromArrowType() { assertEquals("DATE", getSqlTypeNameFromArrowType(new ArrowType.Date(DateUnit.MILLISECOND))); assertEquals("TIME", getSqlTypeNameFromArrowType(new ArrowType.Time(TimeUnit.MILLISECOND, 32))); + assertEquals( + "TIMESTAMP", + getSqlTypeNameFromArrowType(new ArrowType.Timestamp(TimeUnit.MILLISECOND, null))); assertEquals( "TIMESTAMP", getSqlTypeNameFromArrowType(new ArrowType.Timestamp(TimeUnit.MILLISECOND, ""))); + assertEquals( + "TIMESTAMP_WITH_TIMEZONE", + getSqlTypeNameFromArrowType(new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC"))); assertEquals("BOOLEAN", getSqlTypeNameFromArrowType(new ArrowType.Bool())); From c2234549827488d32b6698f91e26053a293d808e Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Mon, 28 Apr 2025 14:23:46 +0800 Subject: [PATCH 016/271] GH-729: [JDBC] Fix BinaryConsumer consuming null value (#730) ## What's Changed Set `startOffset` of the next item when `BinaryConsumer` consuming `null` value. Closes #729 . --- .../adapter/jdbc/consumer/BinaryConsumer.java | 25 ++++++++++--------- .../jdbc/consumer/BinaryConsumerTest.java | 11 +++++++- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumer.java b/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumer.java index edbc6360df..73ec04b8a0 100644 --- a/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumer.java +++ b/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumer.java @@ -51,13 +51,15 @@ public BinaryConsumer(VarBinaryVector vector, int index) { /** consume a InputStream. */ public void consume(InputStream is) throws IOException { + while (currentIndex >= vector.getValueCapacity()) { + vector.reallocValidityAndOffsetBuffers(); + } + + final int startOffset = vector.getStartOffset(currentIndex); + final ArrowBuf offsetBuffer = vector.getOffsetBuffer(); + int dataLength = 0; + if (is != null) { - while (currentIndex >= vector.getValueCapacity()) { - vector.reallocValidityAndOffsetBuffers(); - } - final int startOffset = vector.getStartOffset(currentIndex); - final ArrowBuf offsetBuffer = vector.getOffsetBuffer(); - int dataLength = 0; int read; while ((read = is.read(reuseBytes)) != -1) { while (vector.getDataBuffer().capacity() < (startOffset + dataLength + read)) { @@ -66,11 +68,12 @@ public void consume(InputStream is) throws IOException { vector.getDataBuffer().setBytes(startOffset + dataLength, reuseBytes, 0, read); dataLength += read; } - offsetBuffer.setInt( - (currentIndex + 1) * ((long) VarBinaryVector.OFFSET_WIDTH), startOffset + dataLength); + BitVectorHelper.setBit(vector.getValidityBuffer(), currentIndex); - vector.setLastSet(currentIndex); } + offsetBuffer.setInt( + (currentIndex + 1) * ((long) VarBinaryVector.OFFSET_WIDTH), startOffset + dataLength); + vector.setLastSet(currentIndex); } public void moveWriterPosition() { @@ -95,9 +98,7 @@ public NullableBinaryConsumer(VarBinaryVector vector, int index) { @Override public void consume(ResultSet resultSet) throws SQLException, IOException { InputStream is = resultSet.getBinaryStream(columnIndexInResultSet); - if (!resultSet.wasNull()) { - consume(is); - } + consume(is); moveWriterPosition(); } } diff --git a/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumerTest.java b/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumerTest.java index b1e253794d..bb836578e2 100644 --- a/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumerTest.java +++ b/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumerTest.java @@ -22,6 +22,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import org.apache.arrow.vector.BaseValueVector; import org.apache.arrow.vector.VarBinaryVector; import org.junit.jupiter.api.Test; @@ -65,7 +66,11 @@ public void testConsumeInputStream(byte[][] values, boolean nullable) throws IOE nullable, binaryConsumer -> { for (byte[] value : values) { - binaryConsumer.consume(new ByteArrayInputStream(value)); + if (value != null) { + binaryConsumer.consume(new ByteArrayInputStream(value)); + } else { + binaryConsumer.consume((InputStream) null); + } binaryConsumer.moveWriterPosition(); } }, @@ -119,5 +124,9 @@ public void testConsumeInputStream() throws IOException { testRecords[i] = createBytes(DEFAULT_RECORD_BYTE_COUNT); } testConsumeInputStream(testRecords, false); + + byte[] bytes1 = new byte[] {1, 2, 3}; + byte[] bytes2 = new byte[] {4, 5, 6}; + testConsumeInputStream(new byte[][] {bytes1, null, bytes2}, true); } } From b9e37f0ccecc2651fec3487472c203bd223290e8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Apr 2025 17:32:40 +0900 Subject: [PATCH 017/271] MINOR: [CI] Bump actions/download-artifact from 4.2.1 to 4.3.0 (#733) Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 4.2.1 to 4.3.0. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 5b78cc9395..7e3cf5f6f2 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -101,7 +101,7 @@ jobs: packages: write steps: - name: Download source archive - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: release-source - name: Extract source archive @@ -174,7 +174,7 @@ jobs: MACOSX_DEPLOYMENT_TARGET: "14.0" steps: - name: Download source archive - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: release-source - name: Extract source archive @@ -298,7 +298,7 @@ jobs: arch: "x86_64" steps: - name: Download source archive - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: release-source - name: Extract source archive @@ -371,7 +371,7 @@ jobs: - jni-windows steps: - name: Download artifacts - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: path: artifacts - name: Decompress artifacts @@ -452,11 +452,11 @@ jobs: with: cache: 'pip' - name: Download source archive - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: release-source - name: Download Javadocs - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: reference - name: Extract source archive @@ -521,7 +521,7 @@ jobs: cp ../.asf.yaml ./ git add .nojekyll .asf.yaml - name: Download - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: release-html - name: Extract @@ -557,7 +557,7 @@ jobs: - ubuntu-latest steps: - name: Download release artifacts - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: pattern: release-* - name: Verify @@ -591,7 +591,7 @@ jobs: contents: write steps: - name: Download release artifacts - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: pattern: release-* path: artifacts From 8314c1912ed501171a2085feb1909b07892a6a25 Mon Sep 17 00:00:00 2001 From: Mateusz Rzeszutek Date: Sat, 3 May 2025 11:01:23 +0200 Subject: [PATCH 018/271] GH-737: [FlightSQL] Allow returning column remarks in FlightSQL's CommandGetTables (#727) Resolves #737 ## What's Changed This is an implementation of https://github.com/apache/arrow/pull/46110 for Java --- arrow-format/FlightSql.proto | 4 ++ .../tests/FlightSqlScenarioProducer.java | 2 + .../driver/jdbc/ArrowDatabaseMetadata.java | 6 +++ .../arrow/driver/jdbc/utils/ConvertUtils.java | 4 ++ .../jdbc/ArrowDatabaseMetadataTest.java | 53 ++++++++++++++----- .../driver/jdbc/utils/ConvertUtilsTest.java | 2 + .../flight/sql/FlightSqlColumnMetadata.java | 21 ++++++++ 7 files changed, 80 insertions(+), 12 deletions(-) diff --git a/arrow-format/FlightSql.proto b/arrow-format/FlightSql.proto index 3568d851cb..566230c2a6 100644 --- a/arrow-format/FlightSql.proto +++ b/arrow-format/FlightSql.proto @@ -1212,6 +1212,7 @@ message CommandGetDbSchemas { * - ARROW:FLIGHT:SQL:IS_CASE_SENSITIVE - "1" indicates if the column is case-sensitive, "0" otherwise. * - ARROW:FLIGHT:SQL:IS_READ_ONLY - "1" indicates if the column is read only, "0" otherwise. * - ARROW:FLIGHT:SQL:IS_SEARCHABLE - "1" indicates if the column is searchable via WHERE clause, "0" otherwise. + * - ARROW:FLIGHT:SQL:REMARKS - A comment describing the column. * The returned data should be ordered by catalog_name, db_schema_name, table_name, then table_type, followed by table_schema if requested. */ message CommandGetTables { @@ -1678,6 +1679,7 @@ message ActionEndSavepointRequest { * - ARROW:FLIGHT:SQL:IS_CASE_SENSITIVE - "1" indicates if the column is case-sensitive, "0" otherwise. * - ARROW:FLIGHT:SQL:IS_READ_ONLY - "1" indicates if the column is read only, "0" otherwise. * - ARROW:FLIGHT:SQL:IS_SEARCHABLE - "1" indicates if the column is searchable via WHERE clause, "0" otherwise. + * - ARROW:FLIGHT:SQL:REMARKS - A comment describing the column. * - GetFlightInfo: execute the query. */ message CommandStatementQuery { @@ -1703,6 +1705,7 @@ message CommandStatementQuery { * - ARROW:FLIGHT:SQL:IS_CASE_SENSITIVE - "1" indicates if the column is case-sensitive, "0" otherwise. * - ARROW:FLIGHT:SQL:IS_READ_ONLY - "1" indicates if the column is read only, "0" otherwise. * - ARROW:FLIGHT:SQL:IS_SEARCHABLE - "1" indicates if the column is searchable via WHERE clause, "0" otherwise. + * - ARROW:FLIGHT:SQL:REMARKS - A comment describing the column. * - GetFlightInfo: execute the query. * - DoPut: execute the query. */ @@ -1739,6 +1742,7 @@ message TicketStatementQuery { * - ARROW:FLIGHT:SQL:IS_CASE_SENSITIVE - "1" indicates if the column is case-sensitive, "0" otherwise. * - ARROW:FLIGHT:SQL:IS_READ_ONLY - "1" indicates if the column is read only, "0" otherwise. * - ARROW:FLIGHT:SQL:IS_SEARCHABLE - "1" indicates if the column is searchable via WHERE clause, "0" otherwise. + * - ARROW:FLIGHT:SQL:REMARKS - A comment describing the column. * * If the schema is retrieved after parameter values have been bound with DoPut, then the server should account * for the parameters when determining the schema. diff --git a/flight/flight-integration-tests/src/main/java/org/apache/arrow/flight/integration/tests/FlightSqlScenarioProducer.java b/flight/flight-integration-tests/src/main/java/org/apache/arrow/flight/integration/tests/FlightSqlScenarioProducer.java index be746b5757..e400c031c2 100644 --- a/flight/flight-integration-tests/src/main/java/org/apache/arrow/flight/integration/tests/FlightSqlScenarioProducer.java +++ b/flight/flight-integration-tests/src/main/java/org/apache/arrow/flight/integration/tests/FlightSqlScenarioProducer.java @@ -98,6 +98,7 @@ static Schema getQuerySchema() { .isSearchable(true) .catalogName("catalog_test") .precision(100) + .remarks("test column") .build() .getMetadataMap()), null))); @@ -126,6 +127,7 @@ static Schema getQueryWithTransactionSchema() { .isSearchable(true) .catalogName("catalog_test") .precision(100) + .remarks("test column") .build() .getMetadataMap()), null))); diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java index 3f072d071b..7185ddfe01 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java @@ -1066,6 +1066,7 @@ private int setGetColumnsVectorSchemaRootFromFields( (VarCharVector) currentRoot.getVector("IS_AUTOINCREMENT"); final VarCharVector isGeneratedColumnVector = (VarCharVector) currentRoot.getVector("IS_GENERATEDCOLUMN"); + final VarCharVector remarksVector = (VarCharVector) currentRoot.getVector("REMARKS"); for (int i = 0; i < tableColumnsSize; i++, ordinalIndex++) { final Field field = tableColumns.get(i); @@ -1139,6 +1140,11 @@ private int setGetColumnsVectorSchemaRootFromFields( isAutoincrementVector.setSafe(insertIndex, EMPTY_BYTE_ARRAY); } + String remarks = columnMetadata.getRemarks(); + if (remarks != null) { + remarksVector.setSafe(insertIndex, remarks.getBytes(CHARSET)); + } + // Fields also don't hold information about IS_AUTOINCREMENT and IS_GENERATEDCOLUMN, // so we're setting an empty string (as bytes), which means it couldn't be determined. isGeneratedColumnVector.setSafe(insertIndex, EMPTY_BYTE_ARRAY); diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java index 17b0f42dc7..5dd4c69c73 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java @@ -136,6 +136,10 @@ public static void setOnColumnMetaDataBuilder( if (searchable != null) { builder.setSearchable(searchable); } + final String remarks = columnMetadata.getRemarks(); + if (remarks != null) { + builder.setLabel(remarks); + } } /** diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java index 70d3bcbd33..81579cc387 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java @@ -26,7 +26,6 @@ import static java.util.stream.Collectors.toList; import static java.util.stream.IntStream.range; import static org.apache.arrow.driver.jdbc.utils.MockFlightSqlProducer.serializeSchema; -import static org.apache.arrow.flight.sql.impl.FlightSql.CommandGetCrossReference; import static org.apache.arrow.flight.sql.impl.FlightSql.SqlSupportsConvert.SQL_CONVERT_BIGINT_VALUE; import static org.apache.arrow.flight.sql.impl.FlightSql.SqlSupportsConvert.SQL_CONVERT_BIT_VALUE; import static org.apache.arrow.flight.sql.impl.FlightSql.SqlSupportsConvert.SQL_CONVERT_INTEGER_VALUE; @@ -55,9 +54,11 @@ import org.apache.arrow.driver.jdbc.utils.ResultSetTestUtils; import org.apache.arrow.driver.jdbc.utils.ThrowableAssertionUtils; import org.apache.arrow.flight.FlightProducer.ServerStreamListener; +import org.apache.arrow.flight.sql.FlightSqlColumnMetadata; import org.apache.arrow.flight.sql.FlightSqlProducer.Schemas; import org.apache.arrow.flight.sql.impl.FlightSql; import org.apache.arrow.flight.sql.impl.FlightSql.CommandGetCatalogs; +import org.apache.arrow.flight.sql.impl.FlightSql.CommandGetCrossReference; import org.apache.arrow.flight.sql.impl.FlightSql.CommandGetDbSchemas; import org.apache.arrow.flight.sql.impl.FlightSql.CommandGetExportedKeys; import org.apache.arrow.flight.sql.impl.FlightSql.CommandGetImportedKeys; @@ -79,6 +80,7 @@ import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.Text; import org.junit.jupiter.api.AfterAll; @@ -322,7 +324,7 @@ public class ArrowDatabaseMetadataTest { expectedGetColumnsDecimalDigits.get(i % 3), expectedGetColumnsRadix.get(i % 3), !Objects.equals(expectedGetColumnsIsNullable.get(i % 3), "NO") ? 1 : 0, - null, + format("column description #%d", (i % 3) + 1), null, null, null, @@ -419,17 +421,44 @@ public static void setUpBeforeClass() throws SQLException { try (final BufferAllocator allocator = new RootAllocator(); final VectorSchemaRoot root = VectorSchemaRoot.create(Schemas.GET_TABLES_SCHEMA, allocator)) { + final Field field1 = + new Field( + "column_1", + new FieldType( + true, + ArrowType.Decimal.createDecimal(5, 2, 128), + null, + new FlightSqlColumnMetadata.Builder() + .remarks("column description #1") + .build() + .getMetadataMap()), + null); + final Field field2 = + new Field( + "column_2", + new FieldType( + true, + new ArrowType.Timestamp(TimeUnit.NANOSECOND, "UTC"), + null, + new FlightSqlColumnMetadata.Builder() + .remarks("column description #2") + .build() + .getMetadataMap()), + null); + final Field field3 = + new Field( + "column_3", + new FieldType( + false, + Types.MinorType.INT.getType(), + null, + new FlightSqlColumnMetadata.Builder() + .remarks("column description #3") + .build() + .getMetadataMap()), + null); final byte[] filledTableSchemaBytes = - copyFrom( - serializeSchema( - new Schema( - Arrays.asList( - Field.nullable( - "column_1", ArrowType.Decimal.createDecimal(5, 2, 128)), - Field.nullable( - "column_2", - new ArrowType.Timestamp(TimeUnit.NANOSECOND, "UTC")), - Field.notNullable("column_3", Types.MinorType.INT.getType()))))) + copyFrom(serializeSchema(new Schema(Arrays.asList(field1, field2, field3)))) .toByteArray(); final VarCharVector catalogName = (VarCharVector) root.getVector("catalog_name"); final VarCharVector schemaName = (VarCharVector) root.getVector("db_schema_name"); diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ConvertUtilsTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ConvertUtilsTest.java index f6f549b5ed..b6fdc99694 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ConvertUtilsTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ConvertUtilsTest.java @@ -46,6 +46,7 @@ public void testShouldSetOnColumnMetaDataBuilder() { .isSearchable(true) .precision(20) .scale(10) + .remarks("test column") .build(); ConvertUtils.setOnColumnMetaDataBuilder(builder, expectedColumnMetaData.getMetadataMap()); assertBuilder(builder, expectedColumnMetaData); @@ -119,5 +120,6 @@ private void assertBuilder( assertThat(flightSqlColumnMetaData.isReadOnly(), equalTo(builder.getReadOnly())); assertThat(precision == null ? 0 : precision, equalTo(builder.getPrecision())); assertThat(scale == null ? 0 : scale, equalTo(builder.getScale())); + assertThat(flightSqlColumnMetaData.getRemarks(), equalTo(builder.getLabel())); } } diff --git a/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlColumnMetadata.java b/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlColumnMetadata.java index 1bcc55a660..3a969e10cf 100644 --- a/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlColumnMetadata.java +++ b/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlColumnMetadata.java @@ -53,6 +53,7 @@ public class FlightSqlColumnMetadata { private static final String IS_CASE_SENSITIVE = "ARROW:FLIGHT:SQL:IS_CASE_SENSITIVE"; private static final String IS_READ_ONLY = "ARROW:FLIGHT:SQL:IS_READ_ONLY"; private static final String IS_SEARCHABLE = "ARROW:FLIGHT:SQL:IS_SEARCHABLE"; + private static final String REMARKS = "ARROW:FLIGHT:SQL:REMARKS"; private static final String BOOLEAN_TRUE_STR = "1"; private static final String BOOLEAN_FALSE_STR = "0"; @@ -193,6 +194,15 @@ public Boolean isSearchable() { return stringToBoolean(value); } + /** + * Returns the comment describing the column. + * + * @return The comment describing the column. + */ + public String getRemarks() { + return metadataMap.get(REMARKS); + } + /** Builder of FlightSqlColumnMetadata, used on FlightSqlProducer implementations. */ public static class Builder { private final Map metadataMap; @@ -312,6 +322,17 @@ public Builder isSearchable(boolean isSearchable) { return this; } + /** + * Sets the comment describing the column. + * + * @param remarks The comment describing the column. + * @return This builder. + */ + public Builder remarks(String remarks) { + metadataMap.put(REMARKS, remarks); + return this; + } + /** * Builds a new instance of FlightSqlColumnMetadata. * From f3b50019bdbef5ecc1bfcb293129bbcf7204babd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 May 2025 09:23:07 +0200 Subject: [PATCH 019/271] MINOR: Bump com.google.guava:guava-bom from 33.4.5-jre to 33.4.8-jre (#720) Bumps [com.google.guava:guava-bom](https://github.com/google/guava) from 33.4.5-jre to 33.4.8-jre.

Release notes

Sourced from com.google.guava:guava-bom's releases.

33.4.8

Guava 33.4.8 fixes a problem that we introduced while starting to migrate guava-android off Unsafe in 33.4.7.

Even if you're not upgrading from Guava 33.4.0 or earlier, still read the release notes for Guava 33.4.1. Those release notes contain information about the effects of Guava 33.4.5 and higher on the module system.

Maven

<dependency>
  <groupId>com.google.guava</groupId>
  <artifactId>guava</artifactId>
  <version>33.4.8-jre</version>
  <!-- or, for Android: -->
  <version>33.4.8-android</version>
</dependency>

Jar files

Guava requires one runtime dependency, which you can download here:

Javadoc

JDiff

Changelog

  • util.concurrent: Removed our VarHandle code from guava-android. While the code was never used at runtime under Android, it was causing problems under the Android Gradle Plugin with a minSdkVersion below 26. To continue to avoid sun.misc.Unsafe under the JVM, guava-android will now always use AtomicReferenceFieldUpdater when run there. (75da92419a)

33.4.7

Prefer to upgrade straight to 33.4.8: 33.4.7 breaks the build of Android apps with a minSdkVersion below 26. We will publish a fixed version soon. This problem is fixed in 33.4.8.

Guava 33.4.7, like 33.4.6, fixes two problems that we introduced while modularizing Guava and migrating off Unsafe in 33.4.5.

Even if you're not upgrading from Guava 33.4.0 or earlier, still read the release notes for Guava 33.4.1. Those release notes contain information about the effects of Guava 33.4.5 and higher on the module system.

Maven

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.guava:guava-bom&package-manager=maven&previous-version=33.4.5-jre&new-version=33.4.8-jre)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ab988c7d05..fc274abcff 100644 --- a/pom.xml +++ b/pom.xml @@ -95,7 +95,7 @@ under the License. 1.9.0 5.12.1 2.0.17 - 33.4.5-jre + 33.4.8-jre 4.1.119.Final 1.71.0 4.30.1 From 75741c8ae41b74944ad84de0e2007b8a1dc3c3dd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 May 2025 09:23:50 +0200 Subject: [PATCH 020/271] MINOR: Bump com.google.protobuf:protobuf-bom from 4.30.1 to 4.30.2 (#696) Bumps [com.google.protobuf:protobuf-bom](https://github.com/protocolbuffers/protobuf) from 4.30.1 to 4.30.2.
Commits
  • 43e1626 Updating version.json and repo version numbers to: 30.2
  • 7a4c63b Fix lite classes in the protobuf-java Maven release to be JDK8 compatible. (#...
  • 7831669 Remove dllexport attribute on variable definition. (#20833)
  • da9cadc Restore JDK8 compatibility in Bazel for libraries with dependencies from Mave...
  • 09b5078 Add protobuf_maven artifacts to protobuf_maven_dev as well so they can still ...
  • b7f06f1 Add volatile to featuresResolved (#20766)
  • b69f653 Restore generator headers in cmake install until the next breaking C++ releas...
  • f4b0a79 Restore custom protobuf maven namespaces to avoid polluting main maven namesp...
  • 2dc9f35 Fix Java concurrency issue in feature resolution for old <=3.25.x gencode usi...
  • 221b2a0 Change how we decide which empty string implementation to use. (#20708)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.protobuf:protobuf-bom&package-manager=maven&previous-version=4.30.1&new-version=4.30.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fc274abcff..2e2f1659dd 100644 --- a/pom.xml +++ b/pom.xml @@ -98,7 +98,7 @@ under the License. 33.4.8-jre 4.1.119.Final 1.71.0 - 4.30.1 + 4.30.2 2.18.3 3.4.1 25.2.10 From fc75ff7e2bd7abe187320371a7470c9a7547fdae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 May 2025 18:14:21 +0200 Subject: [PATCH 021/271] MINOR: Bump dep.junit.jupiter.version from 5.12.1 to 5.12.2 (#713) Bumps `dep.junit.jupiter.version` from 5.12.1 to 5.12.2. Updates `org.junit.jupiter:junit-jupiter-engine` from 5.12.1 to 5.12.2
Release notes

Sourced from org.junit.jupiter:junit-jupiter-engine's releases.

JUnit 5.12.2 = Platform 1.12.2 + Jupiter 5.12.2 + Vintage 5.12.2

See Release Notes.

Full Changelog: https://github.com/junit-team/junit5/compare/r5.12.1...r5.12.2

Commits
  • 0a44659 Release 5.12.2
  • 4c7dfdc Finalize 5.12.2 release notes
  • 561613e Fix handling of CleanupMode.ON_SUCCESS
  • 19d07d2 Add 5.12.2 release notes from template
  • 803cbb6 Add build parameter for enabling dry-run mode for test execution
  • eb43e62 Back to snapshots for further development
  • See full diff in compare view

Updates `org.junit.jupiter:junit-jupiter-api` from 5.12.1 to 5.12.2
Release notes

Sourced from org.junit.jupiter:junit-jupiter-api's releases.

JUnit 5.12.2 = Platform 1.12.2 + Jupiter 5.12.2 + Vintage 5.12.2

See Release Notes.

Full Changelog: https://github.com/junit-team/junit5/compare/r5.12.1...r5.12.2

Commits
  • 0a44659 Release 5.12.2
  • 4c7dfdc Finalize 5.12.2 release notes
  • 561613e Fix handling of CleanupMode.ON_SUCCESS
  • 19d07d2 Add 5.12.2 release notes from template
  • 803cbb6 Add build parameter for enabling dry-run mode for test execution
  • eb43e62 Back to snapshots for further development
  • See full diff in compare view

Updates `org.junit.jupiter:junit-jupiter-params` from 5.12.1 to 5.12.2
Release notes

Sourced from org.junit.jupiter:junit-jupiter-params's releases.

JUnit 5.12.2 = Platform 1.12.2 + Jupiter 5.12.2 + Vintage 5.12.2

See Release Notes.

Full Changelog: https://github.com/junit-team/junit5/compare/r5.12.1...r5.12.2

Commits
  • 0a44659 Release 5.12.2
  • 4c7dfdc Finalize 5.12.2 release notes
  • 561613e Fix handling of CleanupMode.ON_SUCCESS
  • 19d07d2 Add 5.12.2 release notes from template
  • 803cbb6 Add build parameter for enabling dry-run mode for test execution
  • eb43e62 Back to snapshots for further development
  • See full diff in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2e2f1659dd..b8b24ca110 100644 --- a/pom.xml +++ b/pom.xml @@ -93,7 +93,7 @@ under the License. ${project.build.directory}/generated-sources 1.9.0 - 5.12.1 + 5.12.2 2.0.17 33.4.8-jre 4.1.119.Final From feb8e95fe2b283c23ec0b380d3522a9e3991bd0c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 May 2025 09:51:19 +0200 Subject: [PATCH 022/271] MINOR: Bump com.diffplug.spotless:spotless-maven-plugin from 2.44.3 to 2.44.4 (#717) Bumps [com.diffplug.spotless:spotless-maven-plugin](https://github.com/diffplug/spotless) from 2.44.3 to 2.44.4.
Release notes

Sourced from com.diffplug.spotless:spotless-maven-plugin's releases.

Maven Plugin v2.44.4

Changed

  • Use palantir-java-format 2.57.0 on Java 21. (#2447)
  • Re-try npm install with --prefer-online after ERESOLVE error. (#2448)
Commits
  • 0ca99e5 Published maven/2.44.4
  • 1b1a4fb Published gradle/7.0.3
  • 0fa3cab Published lib/3.1.1
  • 0fe8f9b Update README.md for android kotlin callouts (#2438)
  • d25f04d Minor tweak.
  • fa3fd1e Add the cool blockquote warning trick to the other Android spot.
  • 60993fd Fix the Android Kotlin warning and adjust its position.
  • 0426db1 Apply Gradle's strict plugin types validation to the Spotless plugin in prepa...
  • 48b4cde fix(deps): update dependency org.mockito:mockito-core to v5.17.0 (#2461)
  • 81f2f8b fix(deps): update dependency org.mockito:mockito-core to v5.17.0
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.diffplug.spotless:spotless-maven-plugin&package-manager=maven&previous-version=2.44.3&new-version=2.44.4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- bom/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index 171d0bc5e9..61b452b9c1 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -203,7 +203,7 @@ under the License. com.diffplug.spotless spotless-maven-plugin - 2.44.3 + 2.44.4 org.codehaus.mojo diff --git a/pom.xml b/pom.xml index b8b24ca110..fe716bad6e 100644 --- a/pom.xml +++ b/pom.xml @@ -487,7 +487,7 @@ under the License. com.diffplug.spotless spotless-maven-plugin - 2.44.3 + 2.44.4 org.codehaus.mojo From f32198fe28687c6e43df21149272764f2710947a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 May 2025 11:06:36 +0200 Subject: [PATCH 023/271] MINOR: Bump org.apache.commons:commons-text from 1.13.0 to 1.13.1 (#714) Bumps org.apache.commons:commons-text from 1.13.0 to 1.13.1. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.commons:commons-text&package-manager=maven&previous-version=1.13.0&new-version=1.13.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql/pom.xml b/flight/flight-sql/pom.xml index 5d8e720fd6..5f06a5e9eb 100644 --- a/flight/flight-sql/pom.xml +++ b/flight/flight-sql/pom.xml @@ -113,7 +113,7 @@ under the License. org.apache.commons commons-text - 1.13.0 + 1.13.1 test From b0cc03d4c62dabc44746f462f274b20003890e95 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 May 2025 11:07:00 +0200 Subject: [PATCH 024/271] MINOR: Bump org.jacoco:jacoco-maven-plugin from 0.8.12 to 0.8.13 (#699) Bumps [org.jacoco:jacoco-maven-plugin](https://github.com/jacoco/jacoco) from 0.8.12 to 0.8.13.
Release notes

Sourced from org.jacoco:jacoco-maven-plugin's releases.

0.8.13

New Features

  • JaCoCo now officially supports Java 23 and Java 24 (GitHub #1757, #1631, #1867).
  • Experimental support for Java 25 class files (GitHub #1807).
  • Calculation of line coverage for Kotlin inline functions (GitHub #1670).
  • Calculation of line coverage for Kotlin inline functions with reified type parameter (GitHub #1670, #1700).
  • Calculation of coverage for Kotlin JvmSynthetic functions (GitHub #1700).
  • Part of bytecode generated by the Kotlin Compose compiler plugin is filtered out during generation of report (GitHub #1616).
  • Part of bytecode generated by the Kotlin compiler for inline value classes is filtered out during generation of report (GitHub #1475).
  • Part of bytecode generated by the Kotlin compiler for suspending lambdas without suspension points is filtered out during generation of report (GitHub #1283).
  • Part of bytecode generated by the Kotlin compiler for when expressions and statements with nullable enum subject is filtered out during generation of report (GitHub #1774).
  • Part of bytecode generated by the Kotlin compiler for when expressions and statements with nullable String subject is filtered out during generation of report (GitHub #1769).
  • Part of bytecode generated by the Kotlin compiler for chains of safe call operators is filtered out during generation of report (GitHub #1810, #1818).
  • Method getEntries generated by the Kotlin compiler for enum classes is filtered out during generation of report (GitHub #1625).
  • Methods generated by the Kotlin compiler for constructors and functions with JvmOverloads annotation are filtered out (GitHub #1768).

Fixed bugs

  • Fixed interpretation of Kotlin SMAP (GitHub #1525).
  • File extensions are preserved in HTML report in case of clashes of normalized file names (GitHub #1660).

Non-functional Changes

  • JaCoCo build now uses Maven Wrapper and requires at least Maven 3.9.9 (GitHub #1708, #1707, #1681).
  • JaCoCo now depends on ASM 9.8 (GitHub #1862).
  • More context information when IllegalArgumentException occurs during reading of zip file (GitHub #1833).
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.jacoco:jacoco-maven-plugin&package-manager=maven&previous-version=0.8.12&new-version=0.8.13)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fe716bad6e..8360e1be2b 100644 --- a/pom.xml +++ b/pom.xml @@ -347,7 +347,7 @@ under the License. org.jacoco jacoco-maven-plugin - 0.8.12 + 0.8.13

... (truncated)

Commits
  • e303ead chore(main): release 2.56.0 (#3715)
  • 3714eee chore: update generation config (#3749)
  • 992e643 ci: Run all downstream protobuf CI checks with Java 8 JVM (#3747)
  • 64ac2c1 feat: Selective gapic generation phase II (#3730)
  • 29a78d3 fix(hermetic-build): use correct image name in templated graalvm jobs (#3743)
  • 243b4da build: introduce testing infra for GraalVM "C" (#3739)
  • 4383ea1 build(deps): update dependency com.google.cloud:google-cloud-shared-config to...
  • 07fb39d chore: introduce .kokoro templates for GraalVM jobs (#3737)
  • 087597e chore(sec): prefix hermetic-build images with infrastructure-public-image ...
  • b77a32b chore: prepare showcase for GraalVM for JDK 17 (#3734)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.api.grpc:proto-google-common-protos&package-manager=maven&previous-version=2.54.1&new-version=2.56.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml index 757de85769..92f58b84de 100644 --- a/flight/flight-core/pom.xml +++ b/flight/flight-core/pom.xml @@ -134,7 +134,7 @@ under the License. com.google.api.grpc proto-google-common-protos - 2.54.1 + 2.56.0 test From 93b7fdc2113c9c21e359edc910a6b44061b0ee50 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 May 2025 10:47:23 +0200 Subject: [PATCH 035/271] MINOR: Bump com.github.ben-manes.caffeine:caffeine from 3.1.8 to 3.2.0 (#747) Bumps [com.github.ben-manes.caffeine:caffeine](https://github.com/ben-manes/caffeine) from 3.1.8 to 3.2.0.
Release notes

Sourced from com.github.ben-manes.caffeine:caffeine's releases.

3.2.0

Cache

  • Added Sigstore signing of maven artifacts
  • Added Expiry static factory methods (#1499)
  • Migrated to JSpecify annotations (was checker framework)
  • Fixed variable expiration calculation when nearing overflow
  • Added logging when an async cache's removal listener fails
  • Added an expiration write optimization to more operations (#1320)
  • Fixed when a Weigher or Expiry fail on an async completion (#1687)
  • Fixed cases when the expiration ticker was also used for statistics (#1678)
  • Fixed refresh handling to skip if the async cache's entry is still loading (#1478)
  • Fixed containsKey for an async cache's synchronous view while in-flight (#1626)
  • Fixed premature expiration for an async cache when using nearly immediate expiration (#1623)
  • For a bulk async load returning extra mappings, wait to be added to the cache before returning (#1409)

Guava

  • Relaxed the OSGi version requirement (#1160)

JCache

  • Allow hibernate.javax.cache.uri to load the configuration from a jar (#1347)
Commits
  • 93d845e prepare for next release
  • 8022a16 expand expire write optimization to more operations (fixes #1320)
  • 9b65365 Avoid early expiration of an pending future due to delayed pinning (fixes #1623)
  • 25405d6 skip refreshing if the async cache entry is still loading (fixes #1478)
  • 2d93f2b fix variable expiration calculation when nearing overflow
  • 3ff2445 migrate to the maintained javapoet project
  • bbf54be minor polish and fixes #1820
  • b09770f upgrade to apache commons collections' junit5 test suite
  • 70f5cf8 use jakarta.inject for jcache guice integration test
  • ae21802 add missing asserts to openjdk tests that check only in their custom harness
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.github.ben-manes.caffeine:caffeine&package-manager=maven&previous-version=3.1.8&new-version=3.2.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index a95fbaca30..04c4691faf 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -151,7 +151,7 @@ under the License. com.github.ben-manes.caffeine caffeine - 3.1.8 + 3.2.0 From 8c74ced9c03e4aba1d1c5fea5f223a42be234dcd Mon Sep 17 00:00:00 2001 From: Gang Wu Date: Tue, 13 May 2025 23:43:14 +0800 Subject: [PATCH 036/271] MINOR: Bump version to 19.0.0-SNAPSHOT (#754) --- adapter/avro/pom.xml | 2 +- adapter/jdbc/pom.xml | 2 +- adapter/orc/pom.xml | 2 +- algorithm/pom.xml | 2 +- bom/pom.xml | 4 ++-- c/pom.xml | 2 +- compression/pom.xml | 2 +- dataset/pom.xml | 2 +- flight/flight-core/pom.xml | 2 +- flight/flight-integration-tests/pom.xml | 2 +- flight/flight-sql-jdbc-core/pom.xml | 2 +- flight/flight-sql-jdbc-driver/pom.xml | 2 +- flight/flight-sql/pom.xml | 2 +- flight/pom.xml | 2 +- format/pom.xml | 2 +- gandiva/pom.xml | 2 +- memory/memory-core/pom.xml | 2 +- memory/memory-netty-buffer-patch/pom.xml | 2 +- memory/memory-netty/pom.xml | 2 +- memory/memory-unsafe/pom.xml | 2 +- memory/pom.xml | 2 +- performance/pom.xml | 2 +- pom.xml | 4 ++-- tools/pom.xml | 2 +- vector/pom.xml | 2 +- 25 files changed, 27 insertions(+), 27 deletions(-) diff --git a/adapter/avro/pom.xml b/adapter/avro/pom.xml index cf9d353330..827d19f2a2 100644 --- a/adapter/avro/pom.xml +++ b/adapter/avro/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT ../../pom.xml diff --git a/adapter/jdbc/pom.xml b/adapter/jdbc/pom.xml index f92863fbd6..2f621d7a05 100644 --- a/adapter/jdbc/pom.xml +++ b/adapter/jdbc/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT ../../pom.xml diff --git a/adapter/orc/pom.xml b/adapter/orc/pom.xml index e60a7ceb3c..e3ae7d5163 100644 --- a/adapter/orc/pom.xml +++ b/adapter/orc/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT ../../pom.xml diff --git a/algorithm/pom.xml b/algorithm/pom.xml index e934eb7b22..898c2605b6 100644 --- a/algorithm/pom.xml +++ b/algorithm/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT arrow-algorithm Arrow Algorithms diff --git a/bom/pom.xml b/bom/pom.xml index 80f03d1205..61b452b9c1 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -29,7 +29,7 @@ under the License. org.apache.arrow arrow-bom - 18.3.0 + 19.0.0-SNAPSHOT pom Arrow Bill of Materials @@ -68,7 +68,7 @@ under the License. scm:git:https://github.com/apache/arrow-java.git scm:git:https://github.com/apache/arrow-java.git - v18.3.0 + main https://github.com/apache/arrow-java/tree/${project.scm.tag} diff --git a/c/pom.xml b/c/pom.xml index 290cb561c1..c90b6dc0ef 100644 --- a/c/pom.xml +++ b/c/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT arrow-c-data diff --git a/compression/pom.xml b/compression/pom.xml index 3443f11478..6f60eb7d0a 100644 --- a/compression/pom.xml +++ b/compression/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT arrow-compression Arrow Compression diff --git a/dataset/pom.xml b/dataset/pom.xml index efbe310ea2..6e56d555b7 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT arrow-dataset diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml index 92f58b84de..24beac391e 100644 --- a/flight/flight-core/pom.xml +++ b/flight/flight-core/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 18.3.0 + 19.0.0-SNAPSHOT flight-core diff --git a/flight/flight-integration-tests/pom.xml b/flight/flight-integration-tests/pom.xml index e7fb999149..78a2d08ee1 100644 --- a/flight/flight-integration-tests/pom.xml +++ b/flight/flight-integration-tests/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 18.3.0 + 19.0.0-SNAPSHOT flight-integration-tests diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 04c4691faf..d8e012101c 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 18.3.0 + 19.0.0-SNAPSHOT flight-sql-jdbc-core diff --git a/flight/flight-sql-jdbc-driver/pom.xml b/flight/flight-sql-jdbc-driver/pom.xml index 3776e97f3f..559c42597d 100644 --- a/flight/flight-sql-jdbc-driver/pom.xml +++ b/flight/flight-sql-jdbc-driver/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 18.3.0 + 19.0.0-SNAPSHOT flight-sql-jdbc-driver diff --git a/flight/flight-sql/pom.xml b/flight/flight-sql/pom.xml index 66ade30306..5f06a5e9eb 100644 --- a/flight/flight-sql/pom.xml +++ b/flight/flight-sql/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 18.3.0 + 19.0.0-SNAPSHOT flight-sql diff --git a/flight/pom.xml b/flight/pom.xml index 7b31e8ce91..2fc3e89ef8 100644 --- a/flight/pom.xml +++ b/flight/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT arrow-flight diff --git a/format/pom.xml b/format/pom.xml index 9b4eebfe3c..d3578b63d2 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT arrow-format diff --git a/gandiva/pom.xml b/gandiva/pom.xml index 167bf39cb9..5367bfdedf 100644 --- a/gandiva/pom.xml +++ b/gandiva/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT org.apache.arrow.gandiva diff --git a/memory/memory-core/pom.xml b/memory/memory-core/pom.xml index 840d3464ba..72ee69d60a 100644 --- a/memory/memory-core/pom.xml +++ b/memory/memory-core/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 18.3.0 + 19.0.0-SNAPSHOT arrow-memory-core diff --git a/memory/memory-netty-buffer-patch/pom.xml b/memory/memory-netty-buffer-patch/pom.xml index e9a63b2122..07dc7d2403 100644 --- a/memory/memory-netty-buffer-patch/pom.xml +++ b/memory/memory-netty-buffer-patch/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 18.3.0 + 19.0.0-SNAPSHOT arrow-memory-netty-buffer-patch diff --git a/memory/memory-netty/pom.xml b/memory/memory-netty/pom.xml index 42f35efb33..6d660da117 100644 --- a/memory/memory-netty/pom.xml +++ b/memory/memory-netty/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 18.3.0 + 19.0.0-SNAPSHOT arrow-memory-netty diff --git a/memory/memory-unsafe/pom.xml b/memory/memory-unsafe/pom.xml index 0af306cbfc..92dc0c9fe5 100644 --- a/memory/memory-unsafe/pom.xml +++ b/memory/memory-unsafe/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 18.3.0 + 19.0.0-SNAPSHOT arrow-memory-unsafe diff --git a/memory/pom.xml b/memory/pom.xml index 09a5bc2924..bc34c26050 100644 --- a/memory/pom.xml +++ b/memory/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT arrow-memory pom diff --git a/performance/pom.xml b/performance/pom.xml index 02bdf46a11..3f18188e3a 100644 --- a/performance/pom.xml +++ b/performance/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT arrow-performance jar diff --git a/pom.xml b/pom.xml index 9d89616356..d40f428b22 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT pom Apache Arrow Java Root POM @@ -81,7 +81,7 @@ under the License. scm:git:https://github.com/apache/arrow-java.git scm:git:https://github.com/apache/arrow-java.git - v18.3.0 + main https://github.com/apache/arrow-java/tree/${project.scm.tag} diff --git a/tools/pom.xml b/tools/pom.xml index d60281d80c..cb9a161308 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT arrow-tools Arrow Tools diff --git a/vector/pom.xml b/vector/pom.xml index 450e4ff71d..52ad5105ea 100644 --- a/vector/pom.xml +++ b/vector/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 18.3.0 + 19.0.0-SNAPSHOT arrow-vector Arrow Vectors From b459647910a6a373fb9f3d9cd0eb8cc717932bed Mon Sep 17 00:00:00 2001 From: Gang Wu Date: Wed, 14 May 2025 10:42:38 +0800 Subject: [PATCH 037/271] MINOR: add missing SOURCE_DIR in dev/release/release.sh (#755) ## What's Changed `dev/release/release.sh` requires `SOURCE_DIR` to locate `.env` but it is missing. --- dev/release/release.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/release/release.sh b/dev/release/release.sh index 70e1f96454..f08a618c4f 100755 --- a/dev/release/release.sh +++ b/dev/release/release.sh @@ -19,6 +19,8 @@ set -eu +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + if [ "$#" -ne 2 ]; then echo "Usage: $0 " echo " e.g.: $0 19.0.1 1" From 6be33bba2366f2335b962f25fe6ea0789665ae89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Po=C5=82omski?= Date: Thu, 15 May 2025 04:57:23 +0200 Subject: [PATCH 038/271] MINOR: Empty stream double check (#742) ## What's Changed In some cases InflaterInputStream can return a non zero on `available()` method call, while it is actually at EOS. A subsequent `read()` call would respond with -1 and eventually cause: 
Caused by: org.apache.arrow.flight.FlightRuntimeException: Failed to
read message.
at
org.apache.arrow.flight.CallStatus.toRuntimeException(CallStatus.java:121)
at
org.apache.arrow.flight.grpc.StatusUtils.fromGrpcRuntimeException(StatusUtils.java:161)
at
org.apache.arrow.flight.grpc.StatusUtils.fromThrowable(StatusUtils.java:182)
at
org.apache.arrow.flight.FlightStream$Observer.onError(FlightStream.java:489)
at org.apache.arrow.flight.FlightClient$1.onError(FlightClient.java:371)
at
io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onClose(ClientCalls.java:564)
at
io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
at
io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
at
io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
at
org.apache.arrow.flight.grpc.ClientInterceptorAdapter$FlightClientCallListener.onClose(ClientInterceptorAdapter.java:118)
at
io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:564)
	at io.grpc.internal.ClientCallImpl.access$100(ClientCallImpl.java:72)
at
io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInternal(ClientCallImpl.java:729)
at
io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:710)
	at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)
at
io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:133)
	... 3 common frames omitted
Caused by: java.lang.RuntimeException:
com.google.protobuf.InvalidProtocolBufferException: While parsing a
protocol message, the input ended unexpectedly in the middle of a field.
This could mean either that the input has been truncated or that an
embedded message misreported its own length.
	at org.apache.arrow.flight.ArrowMessage.frame(ArrowMessage.java:363)
at
org.apache.arrow.flight.ArrowMessage$ArrowMessageHolderMarshaller.parse(ArrowMessage.java:575)
at
org.apache.arrow.flight.ArrowMessage$ArrowMessageHolderMarshaller.parse(ArrowMessage.java:560)
	at io.grpc.MethodDescriptor.parseResponse(MethodDescriptor.java:284)
at
io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:657)
at
io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:644)
	... 5 common frames omitted
Caused by: com.google.protobuf.InvalidProtocolBufferException: While
parsing a protocol message, the input ended unexpectedly in the middle
of a field. This could mean either that the input has been truncated or
that an embedded message misreported its own length.
at
com.google.protobuf.InvalidProtocolBufferException.truncatedMessage(InvalidProtocolBufferException.java:92)
at
com.google.protobuf.CodedInputStream.readRawVarint32(CodedInputStream.java:568)
at
org.apache.arrow.flight.ArrowMessage.readRawVarint32(ArrowMessage.java:369)
	at org.apache.arrow.flight.ArrowMessage.frame(ArrowMessage.java:290)
	... 10 common frames omitted
--- .../java/org/apache/arrow/flight/ArrowMessage.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/flight/flight-core/src/main/java/org/apache/arrow/flight/ArrowMessage.java b/flight/flight-core/src/main/java/org/apache/arrow/flight/ArrowMessage.java index 9cefccb3fe..ab4eab3048 100644 --- a/flight/flight-core/src/main/java/org/apache/arrow/flight/ArrowMessage.java +++ b/flight/flight-core/src/main/java/org/apache/arrow/flight/ArrowMessage.java @@ -287,7 +287,11 @@ private static ArrowMessage frame(BufferAllocator allocator, final InputStream s ArrowBuf body = null; ArrowBuf appMetadata = null; while (stream.available() > 0) { - int tag = readRawVarint32(stream); + final int tagFirstByte = stream.read(); + if (tagFirstByte == -1) { + break; + } + int tag = readRawVarint32(tagFirstByte, stream); switch (tag) { case DESCRIPTOR_TAG: { @@ -366,6 +370,10 @@ private static ArrowMessage frame(BufferAllocator allocator, final InputStream s private static int readRawVarint32(InputStream is) throws IOException { int firstByte = is.read(); + return readRawVarint32(firstByte, is); + } + + private static int readRawVarint32(int firstByte, InputStream is) throws IOException { return CodedInputStream.readRawVarint32(firstByte, is); } From 7c25ce5d86490822600b49928d34a08b4dddad46 Mon Sep 17 00:00:00 2001 From: ViggoC Date: Thu, 22 May 2025 21:40:44 +0800 Subject: [PATCH 039/271] GH-52: Make RangeEqualsVisitor of RunEndEncodedVector more efficient (#761) ## What's Changed Avoid doing a binary search on every step to make the RangeEqualsVisitor of RunEndEncodedVector more efficient. Closes #52 . --- .../vector/compare/RangeEqualsVisitor.java | 44 ++++----- .../vector/complex/RunEndEncodedVector.java | 98 +++++++++++++++++++ .../arrow/vector/TestRunEndEncodedVector.java | 18 ++-- .../compare/TestRangeEqualsVisitor.java | 52 ++++++++++ 4 files changed, 181 insertions(+), 31 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java b/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java index abcf312c5e..bc2e3a6aab 100644 --- a/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java +++ b/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java @@ -43,6 +43,7 @@ import org.apache.arrow.vector.complex.ListViewVector; import org.apache.arrow.vector.complex.NonNullableStructVector; import org.apache.arrow.vector.complex.RunEndEncodedVector; +import org.apache.arrow.vector.complex.RunEndEncodedVector.RangeIterator; import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.complex.UnionVector; @@ -270,42 +271,35 @@ protected boolean compareRunEndEncodedVectors(Range range) { RunEndEncodedVector leftVector = (RunEndEncodedVector) left; RunEndEncodedVector rightVector = (RunEndEncodedVector) right; - final int leftRangeEnd = range.getLeftStart() + range.getLength(); - final int rightRangeEnd = range.getRightStart() + range.getLength(); + final RunEndEncodedVector.RangeIterator leftIterator = + new RunEndEncodedVector.RangeIterator(leftVector, range.getLeftStart(), range.getLength()); + final RunEndEncodedVector.RangeIterator rightIterator = + new RunEndEncodedVector.RangeIterator( + rightVector, range.getRightStart(), range.getLength()); FieldVector leftValuesVector = leftVector.getValuesVector(); FieldVector rightValuesVector = rightVector.getValuesVector(); RangeEqualsVisitor innerVisitor = createInnerVisitor(leftValuesVector, rightValuesVector, null); - int leftLogicalIndex = range.getLeftStart(); - int rightLogicalIndex = range.getRightStart(); + while (nextRun(leftIterator, rightIterator)) { + int leftPhysicalIndex = leftIterator.getRunIndex(); + int rightPhysicalIndex = rightIterator.getRunIndex(); - while (leftLogicalIndex < leftRangeEnd) { - // TODO: implement it more efficient - // https://github.com/apache/arrow/issues/44157 - int leftPhysicalIndex = leftVector.getPhysicalIndex(leftLogicalIndex); - int rightPhysicalIndex = rightVector.getPhysicalIndex(rightLogicalIndex); - if (leftValuesVector.accept( - innerVisitor, new Range(leftPhysicalIndex, rightPhysicalIndex, 1))) { - int leftRunEnd = leftVector.getRunEnd(leftLogicalIndex); - int rightRunEnd = rightVector.getRunEnd(rightLogicalIndex); - - int leftRunLength = Math.min(leftRunEnd, leftRangeEnd) - leftLogicalIndex; - int rightRunLength = Math.min(rightRunEnd, rightRangeEnd) - rightLogicalIndex; - - if (leftRunLength != rightRunLength) { - return false; - } else { - leftLogicalIndex = leftRunEnd; - rightLogicalIndex = rightRunEnd; - } - } else { + if (leftIterator.getRunLength() != rightIterator.getRunLength() + || !leftValuesVector.accept( + innerVisitor, new Range(leftPhysicalIndex, rightPhysicalIndex, 1))) { return false; } } - return true; + return leftIterator.isEnd() && rightIterator.isEnd(); + } + + private static boolean nextRun(RangeIterator leftIterator, RangeIterator rightIterator) { + boolean left = leftIterator.nextRun(); + boolean right = rightIterator.nextRun(); + return left && right; } protected RangeEqualsVisitor createInnerVisitor( diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/RunEndEncodedVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/RunEndEncodedVector.java index 1bb9a3d6c0..b83e13449a 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/RunEndEncodedVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/RunEndEncodedVector.java @@ -28,6 +28,7 @@ import org.apache.arrow.memory.OutOfMemoryException; import org.apache.arrow.memory.util.ByteFunctionHelpers; import org.apache.arrow.memory.util.hash.ArrowBufHasher; +import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.BaseIntVector; import org.apache.arrow.vector.BaseValueVector; import org.apache.arrow.vector.BigIntVector; @@ -820,4 +821,101 @@ static int getPhysicalIndex(FieldVector runEndVector, int logicalIndex) { return result; } + + public static class RangeIterator { + + private final RunEndEncodedVector runEndEncodedVector; + private final int rangeEnd; + private int runIndex; + private int runEnd; + private int logicalPos; + + /** + * Constructs a new RangeIterator for iterating over a range of values in a RunEndEncodedVector. + * + * @param runEndEncodedVector The vector to iterate over + * @param startIndex The logical start index of the range (inclusive) + * @param length The number of values to include in the range + * @throws IllegalArgumentException if startIndex is negative or (startIndex + length) exceeds + * vector bounds + */ + public RangeIterator(RunEndEncodedVector runEndEncodedVector, int startIndex, int length) { + int rangeEnd = startIndex + length; + Preconditions.checkArgument( + startIndex >= 0, "startIndex %s must be non negative.", startIndex); + Preconditions.checkArgument( + rangeEnd <= runEndEncodedVector.getValueCount(), + "(startIndex + length) %s out of range[0, %s].", + rangeEnd, + runEndEncodedVector.getValueCount()); + + this.rangeEnd = rangeEnd; + this.runEndEncodedVector = runEndEncodedVector; + this.runIndex = runEndEncodedVector.getPhysicalIndex(startIndex) - 1; + this.runEnd = startIndex; + this.logicalPos = -1; + } + + /** + * Advances to the next run in the range. + * + * @return true if there is another run available, false if iteration has completed + */ + public boolean nextRun() { + logicalPos = runEnd; + if (logicalPos >= rangeEnd) { + return false; + } + updateRun(); + return true; + } + + private void updateRun() { + runIndex++; + runEnd = (int) ((BaseIntVector) runEndEncodedVector.runEndsVector).getValueAsLong(runIndex); + } + + /** + * Advances to the next value in the range. + * + * @return true if there is another value available, false if iteration has completed + */ + public boolean nextValue() { + logicalPos++; + if (logicalPos >= rangeEnd) { + return false; + } + if (logicalPos == runEnd) { + updateRun(); + } + return true; + } + + /** + * Gets the current run index (physical position in the run-ends vector). + * + * @return the current run index + */ + public int getRunIndex() { + return runIndex; + } + + /** + * Gets the length of the current run within the iterator's range. + * + * @return the number of remaining values in current run within the iterator's range + */ + public int getRunLength() { + return Math.min(runEnd, rangeEnd) - logicalPos; + } + + /** + * Checks if iteration has completed. + * + * @return true if all values in the range have been processed, false otherwise + */ + public boolean isEnd() { + return logicalPos >= rangeEnd; + } + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestRunEndEncodedVector.java b/vector/src/test/java/org/apache/arrow/vector/TestRunEndEncodedVector.java index adf51c0730..9fa153e928 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestRunEndEncodedVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestRunEndEncodedVector.java @@ -148,12 +148,18 @@ public void testRangeCompare() { assertTrue( constantVector.accept( new RangeEqualsVisitor(constantVector, constantVector), new Range(1, 2, 13))); - assertFalse( - constantVector.accept( - new RangeEqualsVisitor(constantVector, constantVector), new Range(1, 10, 10))); - assertFalse( - constantVector.accept( - new RangeEqualsVisitor(constantVector, constantVector), new Range(10, 1, 10))); + + // throws exception if the range end is out the bound of the vector + assertThrows( + IllegalArgumentException.class, + () -> + constantVector.accept( + new RangeEqualsVisitor(constantVector, constantVector), new Range(1, 10, 10))); + assertThrows( + IllegalArgumentException.class, + () -> + constantVector.accept( + new RangeEqualsVisitor(constantVector, constantVector), new Range(10, 1, 10))); // Create REE vector representing: [1, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5]. RunEndEncodedVector reeVector = diff --git a/vector/src/test/java/org/apache/arrow/vector/compare/TestRangeEqualsVisitor.java b/vector/src/test/java/org/apache/arrow/vector/compare/TestRangeEqualsVisitor.java index 08da786eb2..9624734356 100644 --- a/vector/src/test/java/org/apache/arrow/vector/compare/TestRangeEqualsVisitor.java +++ b/vector/src/test/java/org/apache/arrow/vector/compare/TestRangeEqualsVisitor.java @@ -22,6 +22,7 @@ import java.nio.charset.Charset; import java.util.Arrays; +import java.util.List; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BigIntVector; @@ -39,6 +40,7 @@ import org.apache.arrow.vector.complex.LargeListViewVector; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.ListViewVector; +import org.apache.arrow.vector.complex.RunEndEncodedVector; import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.complex.UnionVector; import org.apache.arrow.vector.complex.impl.NullableStructWriter; @@ -53,7 +55,9 @@ import org.apache.arrow.vector.holders.NullableUInt4Holder; import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ArrowType.RunEndEncoded; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.junit.jupiter.api.AfterEach; @@ -1003,6 +1007,54 @@ public void testLargeListViewVectorApproxEquals() { } } + @Test + public void testRunEndEncodedFloat8ApproxEquals() { + try (final Float8Vector vector1 = new Float8Vector("float", allocator); + final Float8Vector vector2 = new Float8Vector("float", allocator); + final Float8Vector vector3 = new Float8Vector("float", allocator); + final IntVector reeVector = new IntVector("ree", allocator)) { + + final float epsilon = 1.0E-6f; + setVector(vector1, 1.1, 2.2); + setVector(vector2, 1.1 + epsilon / 2, 2.2 + epsilon / 2); + setVector(vector3, 1.1 + epsilon * 2, 2.2 + epsilon * 2); + setVector(reeVector, 1, 3); + + ArrowType type = MinorType.FLOAT8.getType(); + final FieldType valueType = FieldType.notNullable(type); + final FieldType runEndType = FieldType.notNullable(MinorType.INT.getType()); + + final Field valueField = new Field("value", valueType, null); + final Field runEndField = new Field("ree", runEndType, null); + + Field field = + new Field( + "ree_float", + FieldType.notNullable(RunEndEncoded.INSTANCE), + List.of(runEndField, valueField)); + + try (final RunEndEncodedVector encodedVector1 = + new RunEndEncodedVector(field, allocator, reeVector, vector1, null); + final RunEndEncodedVector encodedVector2 = + new RunEndEncodedVector(field, allocator, reeVector, vector2, null); + final RunEndEncodedVector encodedVector3 = + new RunEndEncodedVector(field, allocator, reeVector, vector3, null)) { + + encodedVector1.setValueCount(3); + encodedVector2.setValueCount(3); + encodedVector3.setValueCount(3); + + Range range = new Range(0, 0, encodedVector1.getValueCount()); + assertTrue( + new ApproxEqualsVisitor(encodedVector1, encodedVector2, epsilon, epsilon) + .rangeEquals(range)); + assertFalse( + new ApproxEqualsVisitor(encodedVector1, encodedVector3, epsilon, epsilon) + .rangeEquals(range)); + } + } + } + private void writeStructVector(NullableStructWriter writer, int value1, long value2) { writer.start(); writer.integer("f0").writeInt(value1); From 2aef66df8ce771b3e634e28c98e4442b8cb39995 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 24 May 2025 14:22:40 +0900 Subject: [PATCH 040/271] GH-768: Use apache/arrow-js for JS in integration test (#769) ## What's Changed `js/` in apache/arrow moved to apache/arrow-js. So let's use apache/arrow-js for JS. Closes #768. --- .github/workflows/test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5db5c988eb..f1028ce029 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -176,6 +176,11 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: java + - name: Checkout Arrow JavaScript + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: apache/arrow-js + path: js - name: Free up disk space run: | ci/scripts/util_free_space.sh @@ -199,6 +204,7 @@ jobs: -e ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS=java \ -e ARCHERY_INTEGRATION_WITH_GO=1 \ -e ARCHERY_INTEGRATION_WITH_JAVA=1 \ + -e ARCHERY_INTEGRATION_WITH_JS=1 \ -e ARCHERY_INTEGRATION_WITH_NANOARROW=1 \ -e ARCHERY_INTEGRATION_WITH_RUST=1 \ conda-integration From abef7af8490d706d52821e56afc6d1eb21e24faf Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 26 May 2025 08:55:13 +0900 Subject: [PATCH 041/271] GH-770: Ensure updating Homebrew Python on macos-13 (#771) ## What's Changed If we update older Python (e.g. Python 3.12) to newer Python (e.g. Python 3.13), depended packages may update newer Python when we update old Python. Let's use newer Python -> older Python order instead to avoid the situation. Closes #770. --- .github/workflows/rc.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 7e3cf5f6f2..72456fa556 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -221,6 +221,10 @@ jobs: # llvm@14 because llvm is newer than llvm@14. brew uninstall llvm || : + # We can remove this when we drop support for + # macos-13. because macos-14 or later uses /opt/homebrew/ + # not /usr/local/. + # # Ensure updating python@XXX with the "--overwrite" option. # If python@XXX is updated without "--overwrite", it causes # a conflict error. Because Python 3 installed not by @@ -229,10 +233,10 @@ jobs: # tries to replace /usr/local/bin/2to3 and so on and causes # a conflict error. brew update - for python_package in $(brew list | grep python@); do + for python_package in $(brew list | grep python@ | sort -r); do brew install --overwrite ${python_package} done - brew install --overwrite python + brew install --overwrite python3 if [ "$(uname -m)" = "arm64" ]; then # pkg-config formula is deprecated but it's still installed From c6f608e863d3cefdfc41a433ed854e43b20a0925 Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Wed, 28 May 2025 03:06:02 +0200 Subject: [PATCH 042/271] GH-765: Do not close/free imported BaseStruct objects (#766) ## What's Changed This PR removes the direct and indirect calls to `BaseStruct#close` from `org.apache.arrow.c.Data`. By not eagerly closing/freeing these objects callers can reuse instances multiple times. Closes #765. --- .../org/apache/arrow/c/ArrayImporter.java | 1 - .../arrow/c/ArrowArrayStreamReader.java | 1 - c/src/main/java/org/apache/arrow/c/Data.java | 219 ++++++++++++++++-- .../org/apache/arrow/c/RoundtripTest.java | 106 +++++++-- 4 files changed, 287 insertions(+), 40 deletions(-) diff --git a/c/src/main/java/org/apache/arrow/c/ArrayImporter.java b/c/src/main/java/org/apache/arrow/c/ArrayImporter.java index b74fb1b473..f31a8a1faa 100644 --- a/c/src/main/java/org/apache/arrow/c/ArrayImporter.java +++ b/c/src/main/java/org/apache/arrow/c/ArrayImporter.java @@ -58,7 +58,6 @@ void importArray(ArrowArray src) { ArrowArray ownedArray = ArrowArray.allocateNew(allocator); ownedArray.save(snapshot); src.markReleased(); - src.close(); recursionLevel = 0; diff --git a/c/src/main/java/org/apache/arrow/c/ArrowArrayStreamReader.java b/c/src/main/java/org/apache/arrow/c/ArrowArrayStreamReader.java index 07a88cd8d7..34a9c4ec03 100644 --- a/c/src/main/java/org/apache/arrow/c/ArrowArrayStreamReader.java +++ b/c/src/main/java/org/apache/arrow/c/ArrowArrayStreamReader.java @@ -44,7 +44,6 @@ final class ArrowArrayStreamReader extends ArrowReader { this.ownedStream = ArrowArrayStream.allocateNew(allocator); this.ownedStream.save(snapshot); stream.markReleased(); - stream.close(); } @Override diff --git a/c/src/main/java/org/apache/arrow/c/Data.java b/c/src/main/java/org/apache/arrow/c/Data.java index 0b4da33b4e..f9d2ee4542 100644 --- a/c/src/main/java/org/apache/arrow/c/Data.java +++ b/c/src/main/java/org/apache/arrow/c/Data.java @@ -231,6 +231,22 @@ public static void exportArrayStream( new ArrayStreamExporter(allocator).export(out, reader); } + /** + * Equivalent to calling {@link #importField(BufferAllocator, ArrowSchema, + * CDataDictionaryProvider, boolean) importField(allocator, schema, provider, true)}. + * + * @param allocator Buffer allocator for allocating dictionary vectors + * @param schema C data interface struct representing the field [inout] + * @param provider A dictionary provider will be initialized with empty dictionary vectors + * (optional) + * @return Imported field object + * @see #importField(BufferAllocator, ArrowSchema, CDataDictionaryProvider, boolean) + */ + public static Field importField( + BufferAllocator allocator, ArrowSchema schema, CDataDictionaryProvider provider) { + return importField(allocator, schema, provider, true); + } + /** * Import Java Field from the C data interface. * @@ -241,19 +257,42 @@ public static void exportArrayStream( * @param schema C data interface struct representing the field [inout] * @param provider A dictionary provider will be initialized with empty dictionary vectors * (optional) + * @param closeImportedStructs if true, the ArrowSchema struct will be closed when this method + * completes. * @return Imported field object */ public static Field importField( - BufferAllocator allocator, ArrowSchema schema, CDataDictionaryProvider provider) { + BufferAllocator allocator, + ArrowSchema schema, + CDataDictionaryProvider provider, + boolean closeImportedStructs) { try { SchemaImporter importer = new SchemaImporter(allocator); return importer.importField(schema, provider); } finally { schema.release(); - schema.close(); + if (closeImportedStructs) { + schema.close(); + } } } + /** + * Equivalent to calling {@link #importSchema(BufferAllocator, ArrowSchema, + * CDataDictionaryProvider, boolean) importSchema(allocator, schema, provider, true)}. + * + * @param allocator Buffer allocator for allocating dictionary vectors + * @param schema C data interface struct representing the field + * @param provider A dictionary provider will be initialized with empty dictionary vectors + * (optional) + * @return Imported schema object + * @see #importSchema(BufferAllocator, ArrowSchema, CDataDictionaryProvider, boolean) + */ + public static Schema importSchema( + BufferAllocator allocator, ArrowSchema schema, CDataDictionaryProvider provider) { + return importSchema(allocator, schema, provider, true); + } + /** * Import Java Schema from the C data interface. * @@ -264,11 +303,16 @@ public static Field importField( * @param schema C data interface struct representing the field * @param provider A dictionary provider will be initialized with empty dictionary vectors * (optional) + * @param closeImportedStructs if true, the ArrowSchema struct will be closed when this method + * completes. * @return Imported schema object */ public static Schema importSchema( - BufferAllocator allocator, ArrowSchema schema, CDataDictionaryProvider provider) { - Field structField = importField(allocator, schema, provider); + BufferAllocator allocator, + ArrowSchema schema, + CDataDictionaryProvider provider, + boolean closeImportedStructs) { + Field structField = importField(allocator, schema, provider, closeImportedStructs); if (structField.getType().getTypeID() != ArrowTypeID.Struct) { throw new IllegalArgumentException( "Cannot import schema: ArrowSchema describes non-struct type"); @@ -276,24 +320,67 @@ public static Schema importSchema( return new Schema(structField.getChildren(), structField.getMetadata()); } + /** + * Equivalent to calling {@link #importIntoVector(BufferAllocator, ArrowArray, FieldVector, + * DictionaryProvider, boolean)} importIntoVector(allocator, array, vector, provider, true)}. + * + * @param allocator Buffer allocator + * @param array C data interface struct holding the array data + * @param vector Imported vector object [out] + * @param provider Dictionary provider to load dictionary vectors to (optional) + * @see #importIntoVector(BufferAllocator, ArrowArray, FieldVector, DictionaryProvider, boolean) + */ + public static void importIntoVector( + BufferAllocator allocator, + ArrowArray array, + FieldVector vector, + DictionaryProvider provider) { + importIntoVector(allocator, array, vector, provider, true); + } + /** * Import Java vector from the C data interface. * - *

The ArrowArray struct has its contents moved (as per the C data interface specification) to - * a private object held alive by the resulting array. + *

On successful completion, the ArrowArray struct will have been moved (as per the C data + * interface specification) to a private object held alive by the resulting array. * * @param allocator Buffer allocator * @param array C data interface struct holding the array data * @param vector Imported vector object [out] * @param provider Dictionary provider to load dictionary vectors to (optional) + * @param closeImportedStructs if true, the ArrowArray struct will be closed when this method + * completes successfully. */ public static void importIntoVector( BufferAllocator allocator, ArrowArray array, FieldVector vector, - DictionaryProvider provider) { + DictionaryProvider provider, + boolean closeImportedStructs) { ArrayImporter importer = new ArrayImporter(allocator, vector, provider); importer.importArray(array); + if (closeImportedStructs) { + array.close(); + } + } + + /** + * Equivalent to calling {@link #importVector(BufferAllocator, ArrowArray, ArrowSchema, + * CDataDictionaryProvider, boolean) importVector(allocator, array, schema, provider, true)}. + * + * @param allocator Buffer allocator for allocating the output FieldVector + * @param array C data interface struct holding the array data + * @param schema C data interface struct holding the array type + * @param provider Dictionary provider to load dictionary vectors to (optional) + * @return Imported vector object + * @see #importVector(BufferAllocator, ArrowArray, ArrowSchema, CDataDictionaryProvider, boolean) + */ + public static FieldVector importVector( + BufferAllocator allocator, + ArrowArray array, + ArrowSchema schema, + CDataDictionaryProvider provider) { + return importVector(allocator, array, schema, provider, true); } /** @@ -307,19 +394,42 @@ public static void importIntoVector( * @param array C data interface struct holding the array data * @param schema C data interface struct holding the array type * @param provider Dictionary provider to load dictionary vectors to (optional) + * @param closeImportedStructs if true, the ArrowArray struct will be closed when this method + * completes successfully and the ArrowSchema struct will be always be closed. * @return Imported vector object */ public static FieldVector importVector( BufferAllocator allocator, ArrowArray array, ArrowSchema schema, - CDataDictionaryProvider provider) { - Field field = importField(allocator, schema, provider); + CDataDictionaryProvider provider, + boolean closeImportedStructs) { + Field field = importField(allocator, schema, provider, closeImportedStructs); FieldVector vector = field.createVector(allocator); - importIntoVector(allocator, array, vector, provider); + importIntoVector(allocator, array, vector, provider, closeImportedStructs); return vector; } + /** + * Equivalent to calling {@link #importIntoVectorSchemaRoot(BufferAllocator, ArrowArray, + * VectorSchemaRoot, DictionaryProvider, boolean) importIntoVectorSchemaRoot(allocator, array, + * root, provider, true)}. + * + * @param allocator Buffer allocator + * @param array C data interface struct holding the record batch data + * @param root vector schema root to load into + * @param provider Dictionary provider to load dictionary vectors to (optional) + * @see #importIntoVectorSchemaRoot(BufferAllocator, ArrowArray, VectorSchemaRoot, + * DictionaryProvider, boolean) + */ + public static void importIntoVectorSchemaRoot( + BufferAllocator allocator, + ArrowArray array, + VectorSchemaRoot root, + DictionaryProvider provider) { + importIntoVectorSchemaRoot(allocator, array, root, provider, true); + } + /** * Import record batch from the C data interface into vector schema root. * @@ -333,15 +443,18 @@ public static FieldVector importVector( * @param array C data interface struct holding the record batch data * @param root vector schema root to load into * @param provider Dictionary provider to load dictionary vectors to (optional) + * @param closeImportedStructs if true, the ArrowArray struct will be closed when this method + * completes successfully */ public static void importIntoVectorSchemaRoot( BufferAllocator allocator, ArrowArray array, VectorSchemaRoot root, - DictionaryProvider provider) { + DictionaryProvider provider, + boolean closeImportedStructs) { try (StructVector structVector = StructVector.emptyWithDuplicates("", allocator)) { structVector.initializeChildrenFromFields(root.getSchema().getFields()); - importIntoVector(allocator, array, structVector, provider); + importIntoVector(allocator, array, structVector, provider, closeImportedStructs); StructVectorUnloader unloader = new StructVectorUnloader(structVector); VectorLoader loader = new VectorLoader(root); try (ArrowRecordBatch recordBatch = unloader.getRecordBatch()) { @@ -350,6 +463,21 @@ public static void importIntoVectorSchemaRoot( } } + /** + * Equivalent to calling {@link #importVectorSchemaRoot(BufferAllocator, ArrowSchema, + * CDataDictionaryProvider, boolean) importVectorSchemaRoot(allocator, schema, provider, true)}. + * + * @param allocator Buffer allocator for allocating the output VectorSchemaRoot + * @param schema C data interface struct holding the record batch schema + * @param provider Dictionary provider to load dictionary vectors to (optional) + * @return Imported vector schema root + * @see #importVectorSchemaRoot(BufferAllocator, ArrowSchema, CDataDictionaryProvider, boolean) + */ + public static VectorSchemaRoot importVectorSchemaRoot( + BufferAllocator allocator, ArrowSchema schema, CDataDictionaryProvider provider) { + return importVectorSchemaRoot(allocator, schema, provider, true); + } + /** * Import Java vector schema root from a C data interface Schema. * @@ -360,11 +488,37 @@ public static void importIntoVectorSchemaRoot( * @param allocator Buffer allocator for allocating the output VectorSchemaRoot * @param schema C data interface struct holding the record batch schema * @param provider Dictionary provider to load dictionary vectors to (optional) + * @param closeImportedStructs if true, the ArrowSchema struct will be closed when this method + * completes * @return Imported vector schema root */ public static VectorSchemaRoot importVectorSchemaRoot( - BufferAllocator allocator, ArrowSchema schema, CDataDictionaryProvider provider) { - return importVectorSchemaRoot(allocator, null, schema, provider); + BufferAllocator allocator, + ArrowSchema schema, + CDataDictionaryProvider provider, + boolean closeImportedStructs) { + return importVectorSchemaRoot(allocator, null, schema, provider, closeImportedStructs); + } + + /** + * Equivalent to calling {@link #importVectorSchemaRoot(BufferAllocator, ArrowArray, ArrowSchema, + * CDataDictionaryProvider, boolean) importVectorSchemaRoot(allocator, array, schema, provider, + * true)}. + * + * @param allocator Buffer allocator for allocating the output VectorSchemaRoot + * @param array C data interface struct holding the record batch data (optional) + * @param schema C data interface struct holding the record batch schema + * @param provider Dictionary provider to load dictionary vectors to (optional) + * @return Imported vector schema root + * @see #importVectorSchemaRoot(BufferAllocator, ArrowArray, ArrowSchema, CDataDictionaryProvider, + * boolean) + */ + public static VectorSchemaRoot importVectorSchemaRoot( + BufferAllocator allocator, + ArrowArray array, + ArrowSchema schema, + CDataDictionaryProvider provider) { + return importVectorSchemaRoot(allocator, array, schema, provider, true); } /** @@ -383,29 +537,56 @@ public static VectorSchemaRoot importVectorSchemaRoot( * @param array C data interface struct holding the record batch data (optional) * @param schema C data interface struct holding the record batch schema * @param provider Dictionary provider to load dictionary vectors to (optional) + * @param closeImportedStructs if true, the ArrowArray struct will be closed when this method + * completes successfully and the ArrowSchema struct will be always be closed. * @return Imported vector schema root */ public static VectorSchemaRoot importVectorSchemaRoot( BufferAllocator allocator, ArrowArray array, ArrowSchema schema, - CDataDictionaryProvider provider) { + CDataDictionaryProvider provider, + boolean closeImportedStructs) { VectorSchemaRoot vsr = - VectorSchemaRoot.create(importSchema(allocator, schema, provider), allocator); + VectorSchemaRoot.create( + importSchema(allocator, schema, provider, closeImportedStructs), allocator); if (array != null) { - importIntoVectorSchemaRoot(allocator, array, vsr, provider); + importIntoVectorSchemaRoot(allocator, array, vsr, provider, closeImportedStructs); } return vsr; } /** - * Import an ArrowArrayStream as an {@link ArrowReader}. + * Equivalent to calling {@link #importArrayStream(BufferAllocator, ArrowArrayStream, boolean) + * importArrayStream(allocator, stream, true)}. * * @param allocator Buffer allocator for allocating the output data. * @param stream C stream interface struct to import. * @return Imported reader + * @see #importArrayStream(BufferAllocator, ArrowArrayStream, boolean) */ public static ArrowReader importArrayStream(BufferAllocator allocator, ArrowArrayStream stream) { - return new ArrowArrayStreamReader(allocator, stream); + return importArrayStream(allocator, stream, true); + } + + /** + * Import an ArrowArrayStream as an {@link ArrowReader}. + * + *

On successful completion, the ArrowArrayStream struct will have been moved (as per the C + * data interface specification) to a private object held alive by the resulting ArrowReader. + * + * @param allocator Buffer allocator for allocating the output data. + * @param stream C stream interface struct to import. + * @param closeImportedStructs if true, the ArrowArrayStream struct will be closed when this + * method completes successfully + * @return Imported reader + */ + public static ArrowReader importArrayStream( + BufferAllocator allocator, ArrowArrayStream stream, boolean closeImportedStructs) { + ArrowArrayStreamReader reader = new ArrowArrayStreamReader(allocator, stream); + if (closeImportedStructs) { + stream.close(); + } + return reader; } } diff --git a/c/src/test/java/org/apache/arrow/c/RoundtripTest.java b/c/src/test/java/org/apache/arrow/c/RoundtripTest.java index 6d68449c0b..010a305495 100644 --- a/c/src/test/java/org/apache/arrow/c/RoundtripTest.java +++ b/c/src/test/java/org/apache/arrow/c/RoundtripTest.java @@ -17,9 +17,7 @@ package org.apache.arrow.c; import static org.apache.arrow.vector.testing.ValueVectorDataPopulator.setVector; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -958,6 +956,50 @@ public void testVectorSchemaRootWithDuplicatedFieldNames() { @Test public void testSchema() { + Schema schema = createSchema(); + // Consumer allocates empty ArrowSchema + try (ArrowSchema consumerArrowSchema = ArrowSchema.allocateNew(allocator)) { + // Producer fills the schema with data + exportSchema(schema, consumerArrowSchema); + + // Consumer imports schema + Schema importedSchema = Data.importSchema(allocator, consumerArrowSchema, null); + assertEquals(schema.toJson(), importedSchema.toJson()); + } + } + + @Test + public void testSchemaStructReuse() { + Schema schema = createSchema(); + // Consumer allocates empty ArrowSchema + try (ArrowSchema consumerArrowSchema = ArrowSchema.allocateNew(allocator)) { + // Producer fills the schema with data + exportSchema(schema, consumerArrowSchema); + + // Consumer imports schema + Schema importedSchema = Data.importSchema(allocator, consumerArrowSchema, null, false); + assertEquals(schema.toJson(), importedSchema.toJson()); + + // Imported struct should be released but not closed + assertEquals(0, consumerArrowSchema.snapshot().release); + assertNotEquals(0, consumerArrowSchema.memoryAddress()); + + // Export and import again + exportSchema(schema, consumerArrowSchema); + importedSchema = Data.importSchema(allocator, consumerArrowSchema, null, false); + assertEquals(schema.toJson(), importedSchema.toJson()); + assertEquals(0, consumerArrowSchema.snapshot().release); + assertNotEquals(0, consumerArrowSchema.memoryAddress()); + } + } + + private void exportSchema(Schema schema, ArrowSchema targetArrowSchema) { + try (ArrowSchema arrowSchema = ArrowSchema.wrap(targetArrowSchema.memoryAddress())) { + Data.exportSchema(allocator, schema, null, arrowSchema); + } + } + + private static Schema createSchema() { Field decimalField = new Field("inner1", FieldType.nullable(new ArrowType.Decimal(19, 4, 128)), null); Field strField = new Field("inner2", FieldType.nullable(new ArrowType.Utf8()), null); @@ -968,16 +1010,7 @@ public void testSchema() { Arrays.asList(decimalField, strField)); Field intField = new Field("col2", FieldType.nullable(new ArrowType.Int(32, true)), null); Schema schema = new Schema(Arrays.asList(itemField, intField)); - // Consumer allocates empty ArrowSchema - try (ArrowSchema consumerArrowSchema = ArrowSchema.allocateNew(allocator)) { - // Producer fills the schema with data - try (ArrowSchema arrowSchema = ArrowSchema.wrap(consumerArrowSchema.memoryAddress())) { - Data.exportSchema(allocator, schema, null, arrowSchema); - } - // Consumer imports schema - Schema importedSchema = Data.importSchema(allocator, consumerArrowSchema, null); - assertEquals(schema.toJson(), importedSchema.toJson()); - } + return schema; } @Test @@ -1002,12 +1035,8 @@ public void testImportReleasedArray() { try (ArrowSchema consumerArrowSchema = ArrowSchema.allocateNew(allocator); ArrowArray consumerArrowArray = ArrowArray.allocateNew(allocator)) { // Producer creates structures from existing memory pointers - try (ArrowSchema arrowSchema = ArrowSchema.wrap(consumerArrowSchema.memoryAddress()); - ArrowArray arrowArray = ArrowArray.wrap(consumerArrowArray.memoryAddress())) { - // Producer exports vector into the C Data Interface structures - try (final NullVector vector = new NullVector()) { - Data.exportVector(allocator, vector, null, arrowArray, arrowSchema); - } + try (final NullVector vector = new NullVector()) { + exportFieldVector(vector, consumerArrowSchema, consumerArrowArray); } // Release array structure @@ -1025,6 +1054,45 @@ public void testImportReleasedArray() { } } + @Test + public void testArrayStructReuse() { + // Consumer allocates empty structures + try (ArrowSchema consumerArrowSchema = ArrowSchema.allocateNew(allocator); + ArrowArray consumerArrowArray = ArrowArray.allocateNew(allocator)) { + // Producer creates structures from existing memory pointers + try (final NullVector vector = new NullVector()) { + exportFieldVector(vector, consumerArrowSchema, consumerArrowArray); + } + Data.importVector(allocator, consumerArrowArray, consumerArrowSchema, null, false); + + // Imported structs should be released but not closed + assertEquals(0, consumerArrowSchema.snapshot().release); + assertNotEquals(0, consumerArrowSchema.memoryAddress()); + assertEquals(0, consumerArrowArray.snapshot().release); + assertNotEquals(0, consumerArrowArray.memoryAddress()); + + try (final NullVector vector = new NullVector()) { + exportFieldVector(vector, consumerArrowSchema, consumerArrowArray); + } + Data.importVector(allocator, consumerArrowArray, consumerArrowSchema, null, false); + + // Imported structs should be released but not closed + assertEquals(0, consumerArrowSchema.snapshot().release); + assertNotEquals(0, consumerArrowSchema.memoryAddress()); + assertEquals(0, consumerArrowArray.snapshot().release); + assertNotEquals(0, consumerArrowArray.memoryAddress()); + } + } + + private void exportFieldVector( + FieldVector vector, ArrowSchema consumerArrowSchema, ArrowArray consumerArrowArray) { + try (ArrowSchema arrowSchema = ArrowSchema.wrap(consumerArrowSchema.memoryAddress()); + ArrowArray arrowArray = ArrowArray.wrap(consumerArrowArray.memoryAddress())) { + // Producer exports vector into the C Data Interface structures + Data.exportVector(allocator, vector, null, arrowArray, arrowSchema); + } + } + private VectorSchemaRoot createTestVSR() { BitVector bitVector = new BitVector("boolean", allocator); From 17f85a1fba64aef4552ed2fabc5a76bc038a1ed6 Mon Sep 17 00:00:00 2001 From: rtadepalli <105760760+rtadepalli@users.noreply.github.com> Date: Tue, 27 May 2025 21:06:55 -0400 Subject: [PATCH 043/271] GH-70: Move from `hamcrest` to `assertj` in `flight-sql` (#772) ## What's Changed Series of PRs to consolidate on using `assertj` in tests as part of https://github.com/apache/arrow-java/issues/70. --- flight/flight-sql/pom.xml | 10 +- .../arrow/flight/sql/test/TestFlightSql.java | 264 +++++++++--------- .../sql/test/TestFlightSqlStateless.java | 9 +- .../flight/sql/test/TestFlightSqlStreams.java | 29 +- 4 files changed, 150 insertions(+), 162 deletions(-) diff --git a/flight/flight-sql/pom.xml b/flight/flight-sql/pom.xml index 5f06a5e9eb..15d00e3e18 100644 --- a/flight/flight-sql/pom.xml +++ b/flight/flight-sql/pom.xml @@ -116,16 +116,16 @@ under the License. 1.13.1 test - - org.hamcrest - hamcrest - test - commons-cli commons-cli 1.9.0 true + + org.assertj + assertj-core + test + diff --git a/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSql.java b/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSql.java index 3f769363fb..e2934ab1e9 100644 --- a/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSql.java +++ b/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSql.java @@ -21,10 +21,7 @@ import static java.util.Collections.singletonList; import static org.apache.arrow.flight.sql.util.FlightStreamUtils.getResults; import static org.apache.arrow.util.AutoCloseables.close; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.hamcrest.CoreMatchers.nullValue; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -40,6 +37,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.stream.IntStream; import org.apache.arrow.flight.CancelFlightInfoRequest; @@ -76,8 +74,7 @@ import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.Text; import org.apache.arrow.vector.util.VectorBatchAppender; -import org.hamcrest.Matcher; -import org.hamcrest.MatcherAssert; +import org.assertj.core.api.Condition; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -247,16 +244,15 @@ private static List> getNonConformingResultsForGetSqlInfo( @Test public void testGetTablesSchema() { final FlightInfo info = sqlClient.getTables(null, null, null, null, true); - MatcherAssert.assertThat( - info.getSchemaOptional(), is(Optional.of(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA))); + assertThat(info.getSchemaOptional()) + .isEqualTo(Optional.of(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA)); } @Test public void testGetTablesSchemaExcludeSchema() { final FlightInfo info = sqlClient.getTables(null, null, null, null, false); - MatcherAssert.assertThat( - info.getSchemaOptional(), - is(Optional.of(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA_NO_SCHEMA))); + assertThat(info.getSchemaOptional()) + .isEqualTo(Optional.of(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA_NO_SCHEMA)); } @Test @@ -266,8 +262,8 @@ public void testGetTablesResultNoSchema() throws Exception { sqlClient.getTables(null, null, null, null, false).getEndpoints().get(0).getTicket())) { assertAll( () -> { - MatcherAssert.assertThat( - stream.getSchema(), is(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA_NO_SCHEMA)); + assertThat(stream.getSchema()) + .isEqualTo(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA_NO_SCHEMA); }, () -> { final List> results = getResults(stream); @@ -301,7 +297,7 @@ public void testGetTablesResultNoSchema() throws Exception { asList(null /* TODO No catalog yet */, "SYSIBM", "SYSDUMMY1", "SYSTEM TABLE"), asList(null /* TODO No catalog yet */, "APP", "FOREIGNTABLE", "TABLE"), asList(null /* TODO No catalog yet */, "APP", "INTTABLE", "TABLE")); - MatcherAssert.assertThat(results, is(expectedResults)); + assertThat(results).isEqualTo(expectedResults); }); } } @@ -318,8 +314,8 @@ public void testGetTablesResultFilteredNoSchema() throws Exception { assertAll( () -> - MatcherAssert.assertThat( - stream.getSchema(), is(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA_NO_SCHEMA)), + assertThat(stream.getSchema()) + .isEqualTo(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA_NO_SCHEMA), () -> { final List> results = getResults(stream); final List> expectedResults = @@ -327,7 +323,7 @@ public void testGetTablesResultFilteredNoSchema() throws Exception { // catalog_name | schema_name | table_name | table_type | table_schema asList(null /* TODO No catalog yet */, "APP", "FOREIGNTABLE", "TABLE"), asList(null /* TODO No catalog yet */, "APP", "INTTABLE", "TABLE")); - MatcherAssert.assertThat(results, is(expectedResults)); + assertThat(results).isEqualTo(expectedResults); }); } } @@ -343,11 +339,9 @@ public void testGetTablesResultFilteredWithSchema() throws Exception { .getTicket())) { assertAll( () -> - MatcherAssert.assertThat( - stream.getSchema(), is(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA)), + assertThat(stream.getSchema()).isEqualTo(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA), () -> { - MatcherAssert.assertThat( - stream.getSchema(), is(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA)); + assertThat(stream.getSchema()).isEqualTo(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA); final List> results = getResults(stream); final List> expectedResults = ImmutableList.of( @@ -487,7 +481,7 @@ public void testGetTablesResultFilteredWithSchema() throws Exception { .getMetadataMap()), null))) .toJson())); - MatcherAssert.assertThat(results, is(expectedResults)); + assertThat(results).isEqualTo(expectedResults); }); } } @@ -498,11 +492,11 @@ public void testSimplePreparedStatementSchema() throws Exception { assertAll( () -> { final Schema actualSchema = preparedStatement.getResultSetSchema(); - MatcherAssert.assertThat(actualSchema, is(SCHEMA_INT_TABLE)); + assertThat(actualSchema).isEqualTo(SCHEMA_INT_TABLE); }, () -> { final FlightInfo info = preparedStatement.execute(); - MatcherAssert.assertThat(info.getSchemaOptional(), is(Optional.of(SCHEMA_INT_TABLE))); + assertThat(info.getSchemaOptional()).isEqualTo(Optional.of(SCHEMA_INT_TABLE)); }); } } @@ -513,10 +507,8 @@ public void testSimplePreparedStatementResults() throws Exception { final FlightStream stream = sqlClient.getStream(preparedStatement.execute().getEndpoints().get(0).getTicket())) { assertAll( - () -> MatcherAssert.assertThat(stream.getSchema(), is(SCHEMA_INT_TABLE)), - () -> - MatcherAssert.assertThat( - getResults(stream), is(EXPECTED_RESULTS_FOR_STAR_SELECT_QUERY))); + () -> assertThat(stream.getSchema()).isEqualTo(SCHEMA_INT_TABLE), + () -> assertThat(getResults(stream)).isEqualTo(EXPECTED_RESULTS_FOR_STAR_SELECT_QUERY)); } } @@ -538,10 +530,8 @@ public void testSimplePreparedStatementResultsWithParameterBinding() throws Exce FlightStream stream = sqlClient.getStream(flightInfo.getEndpoints().get(0).getTicket()); assertAll( - () -> MatcherAssert.assertThat(stream.getSchema(), is(SCHEMA_INT_TABLE)), - () -> - MatcherAssert.assertThat( - getResults(stream), is(EXPECTED_RESULTS_FOR_PARAMETER_BINDING))); + () -> assertThat(stream.getSchema()).isEqualTo(SCHEMA_INT_TABLE), + () -> assertThat(getResults(stream)).isEqualTo(EXPECTED_RESULTS_FOR_PARAMETER_BINDING)); } } } @@ -579,8 +569,8 @@ public void testSimplePreparedStatementUpdateResults() throws SQLException { deletedRows = deletePrepare.executeUpdate(); } assertAll( - () -> MatcherAssert.assertThat(updatedRows, is(10L)), - () -> MatcherAssert.assertThat(deletedRows, is(10L))); + () -> assertThat(updatedRows).isEqualTo(10L), + () -> assertThat(deletedRows).isEqualTo(10L)); } } } @@ -647,7 +637,7 @@ public void testBulkIngest() throws IOException { null, null)); - MatcherAssert.assertThat(updatedRows, is(-1L)); + assertThat(updatedRows).isEqualTo(-1L); // Ingest directly using VectorSchemaRoot populateNext10RowsInIngestRootBatch( @@ -672,7 +662,7 @@ public void testBulkIngest() throws IOException { deletedRows = deletePrepare.executeUpdate(); } - MatcherAssert.assertThat(deletedRows, is(30L)); + assertThat(deletedRows).isEqualTo(30L); } } } @@ -709,8 +699,7 @@ public void testSimplePreparedStatementUpdateResultsWithoutParameters() throws S final long deletedRows = deletePrepare.executeUpdate(); assertAll( - () -> MatcherAssert.assertThat(updatedRows, is(1L)), - () -> MatcherAssert.assertThat(deletedRows, is(1L))); + () -> assertThat(updatedRows).isEqualTo(1L), () -> assertThat(deletedRows).isEqualTo(1L)); } } @@ -719,19 +708,19 @@ public void testSimplePreparedStatementClosesProperly() { final PreparedStatement preparedStatement = sqlClient.prepare("SELECT * FROM intTable"); assertAll( () -> { - MatcherAssert.assertThat(preparedStatement.isClosed(), is(false)); + assertThat(preparedStatement.isClosed()).isEqualTo(false); }, () -> { preparedStatement.close(); - MatcherAssert.assertThat(preparedStatement.isClosed(), is(true)); + assertThat(preparedStatement.isClosed()).isEqualTo(true); }); } @Test public void testGetCatalogsSchema() { final FlightInfo info = sqlClient.getCatalogs(); - MatcherAssert.assertThat( - info.getSchemaOptional(), is(Optional.of(FlightSqlProducer.Schemas.GET_CATALOGS_SCHEMA))); + assertThat(info.getSchemaOptional()) + .isEqualTo(Optional.of(FlightSqlProducer.Schemas.GET_CATALOGS_SCHEMA)); } @Test @@ -740,11 +729,11 @@ public void testGetCatalogsResults() throws Exception { sqlClient.getStream(sqlClient.getCatalogs().getEndpoints().get(0).getTicket())) { assertAll( () -> - MatcherAssert.assertThat( - stream.getSchema(), is(FlightSqlProducer.Schemas.GET_CATALOGS_SCHEMA)), + assertThat(stream.getSchema()) + .isEqualTo(FlightSqlProducer.Schemas.GET_CATALOGS_SCHEMA), () -> { List> catalogs = getResults(stream); - MatcherAssert.assertThat(catalogs, is(emptyList())); + assertThat(catalogs).isEqualTo(emptyList()); }); } } @@ -752,9 +741,8 @@ public void testGetCatalogsResults() throws Exception { @Test public void testGetTableTypesSchema() { final FlightInfo info = sqlClient.getTableTypes(); - MatcherAssert.assertThat( - info.getSchemaOptional(), - is(Optional.of(FlightSqlProducer.Schemas.GET_TABLE_TYPES_SCHEMA))); + assertThat(info.getSchemaOptional()) + .isEqualTo(Optional.of(FlightSqlProducer.Schemas.GET_TABLE_TYPES_SCHEMA)); } @Test @@ -763,8 +751,8 @@ public void testGetTableTypesResult() throws Exception { sqlClient.getStream(sqlClient.getTableTypes().getEndpoints().get(0).getTicket())) { assertAll( () -> { - MatcherAssert.assertThat( - stream.getSchema(), is(FlightSqlProducer.Schemas.GET_TABLE_TYPES_SCHEMA)); + assertThat(stream.getSchema()) + .isEqualTo(FlightSqlProducer.Schemas.GET_TABLE_TYPES_SCHEMA); }, () -> { final List> tableTypes = getResults(stream); @@ -775,7 +763,7 @@ public void testGetTableTypesResult() throws Exception { singletonList("SYSTEM TABLE"), singletonList("TABLE"), singletonList("VIEW")); - MatcherAssert.assertThat(tableTypes, is(expectedTableTypes)); + assertThat(tableTypes).isEqualTo(expectedTableTypes); }); } } @@ -783,8 +771,8 @@ public void testGetTableTypesResult() throws Exception { @Test public void testGetSchemasSchema() { final FlightInfo info = sqlClient.getSchemas(null, null); - MatcherAssert.assertThat( - info.getSchemaOptional(), is(Optional.of(FlightSqlProducer.Schemas.GET_SCHEMAS_SCHEMA))); + assertThat(info.getSchemaOptional()) + .isEqualTo(Optional.of(FlightSqlProducer.Schemas.GET_SCHEMAS_SCHEMA)); } @Test @@ -793,8 +781,7 @@ public void testGetSchemasResult() throws Exception { sqlClient.getStream(sqlClient.getSchemas(null, null).getEndpoints().get(0).getTicket())) { assertAll( () -> { - MatcherAssert.assertThat( - stream.getSchema(), is(FlightSqlProducer.Schemas.GET_SCHEMAS_SCHEMA)); + assertThat(stream.getSchema()).isEqualTo(FlightSqlProducer.Schemas.GET_SCHEMAS_SCHEMA); }, () -> { final List> schemas = getResults(stream); @@ -812,7 +799,7 @@ public void testGetSchemasResult() throws Exception { asList(null /* TODO Add catalog. */, "SYSIBM"), asList(null /* TODO Add catalog. */, "SYSPROC"), asList(null /* TODO Add catalog. */, "SYSSTAT")); - MatcherAssert.assertThat(schemas, is(expectedSchemas)); + assertThat(schemas).isEqualTo(expectedSchemas); }); } } @@ -825,24 +812,24 @@ public void testGetPrimaryKey() { final List> results = getResults(stream); assertAll( - () -> MatcherAssert.assertThat(results.size(), is(1)), + () -> assertThat(results.size()).isEqualTo(1), () -> { final List result = results.get(0); assertAll( - () -> MatcherAssert.assertThat(result.get(0), is("")), - () -> MatcherAssert.assertThat(result.get(1), is("APP")), - () -> MatcherAssert.assertThat(result.get(2), is("INTTABLE")), - () -> MatcherAssert.assertThat(result.get(3), is("ID")), - () -> MatcherAssert.assertThat(result.get(4), is("1")), - () -> MatcherAssert.assertThat(result.get(5), notNullValue())); + () -> assertThat(result.get(0)).isEqualTo(""), + () -> assertThat(result.get(1)).isEqualTo("APP"), + () -> assertThat(result.get(2)).isEqualTo("INTTABLE"), + () -> assertThat(result.get(3)).isEqualTo("ID"), + () -> assertThat(result.get(4)).isEqualTo("1"), + () -> assertThat(result.get(5)).isNotNull()); }); } @Test public void testGetSqlInfoSchema() { final FlightInfo info = sqlClient.getSqlInfo(); - MatcherAssert.assertThat( - info.getSchemaOptional(), is(Optional.of(FlightSqlProducer.Schemas.GET_SQL_INFO_SCHEMA))); + assertThat(info.getSchemaOptional()) + .isEqualTo(Optional.of(FlightSqlProducer.Schemas.GET_SQL_INFO_SCHEMA)); } @Test @@ -851,11 +838,11 @@ public void testGetSqlInfoResults() throws Exception { try (final FlightStream stream = sqlClient.getStream(info.getEndpoints().get(0).getTicket())) { assertAll( () -> - MatcherAssert.assertThat( - stream.getSchema(), is(FlightSqlProducer.Schemas.GET_SQL_INFO_SCHEMA)), + assertThat(stream.getSchema()) + .isEqualTo(FlightSqlProducer.Schemas.GET_SQL_INFO_SCHEMA), () -> - MatcherAssert.assertThat( - getNonConformingResultsForGetSqlInfo(getResults(stream)), is(emptyList()))); + assertThat(getNonConformingResultsForGetSqlInfo(getResults(stream))) + .isEqualTo(emptyList())); } } @@ -866,11 +853,11 @@ public void testGetSqlInfoResultsWithSingleArg() throws Exception { try (final FlightStream stream = sqlClient.getStream(info.getEndpoints().get(0).getTicket())) { assertAll( () -> - MatcherAssert.assertThat( - stream.getSchema(), is(FlightSqlProducer.Schemas.GET_SQL_INFO_SCHEMA)), + assertThat(stream.getSchema()) + .isEqualTo(FlightSqlProducer.Schemas.GET_SQL_INFO_SCHEMA), () -> - MatcherAssert.assertThat( - getNonConformingResultsForGetSqlInfo(getResults(stream), arg), is(emptyList()))); + assertThat(getNonConformingResultsForGetSqlInfo(getResults(stream), arg)) + .isEqualTo(emptyList())); } } @@ -895,11 +882,11 @@ public void testGetSqlInfoResultsWithManyArgs() throws Exception { try (final FlightStream stream = sqlClient.getStream(info.getEndpoints().get(0).getTicket())) { assertAll( () -> - MatcherAssert.assertThat( - stream.getSchema(), is(FlightSqlProducer.Schemas.GET_SQL_INFO_SCHEMA)), + assertThat(stream.getSchema()) + .isEqualTo(FlightSqlProducer.Schemas.GET_SQL_INFO_SCHEMA), () -> - MatcherAssert.assertThat( - getNonConformingResultsForGetSqlInfo(getResults(stream), args), is(emptyList()))); + assertThat(getNonConformingResultsForGetSqlInfo(getResults(stream), args)) + .isEqualTo(emptyList())); } } @@ -915,28 +902,30 @@ public void testGetCommandExportedKeys() throws Exception { final List> results = getResults(stream); - final List> matchers = + final List> matchers = asList( - nullValue(String.class), // pk_catalog_name - is("APP"), // pk_schema_name - is("FOREIGNTABLE"), // pk_table_name - is("ID"), // pk_column_name - nullValue(String.class), // fk_catalog_name - is("APP"), // fk_schema_name - is("INTTABLE"), // fk_table_name - is("FOREIGNID"), // fk_column_name - is("1"), // key_sequence - containsString("SQL"), // fk_key_name - containsString("SQL"), // pk_key_name - is("3"), // update_rule - is("3")); // delete_rule + new Condition<>(Objects::isNull, "pk_catalog_name expected to be null"), + new Condition<>(c -> c.equals("APP"), "pk_schema_name expected to equal APP"), + new Condition<>( + c -> c.equals("FOREIGNTABLE"), "pk_table_name should equal FOREIGNTABLE"), + new Condition<>(c -> c.equals("ID"), "pk_column_name should equal ID"), + new Condition<>(Objects::isNull, "fk_catalog_name expected to be null"), + new Condition<>(c -> c.equals("APP"), "fk_schema_name expected to be APP"), + new Condition<>(c -> c.equals("INTTABLE"), "fk_table_name expeced to be INTTABLE"), + new Condition<>( + c -> c.equals("FOREIGNID"), "fk_column_name expected to equal FOREIGNID"), + new Condition<>(c -> c.equals("1"), "key_sequence expected to equal 1"), + new Condition<>(c -> c.contains("SQL"), "fk_key_name expected to contain SQL"), + new Condition<>(c -> c.contains("SQL"), "pk_key_name expected to contain SQL"), + new Condition<>(c -> c.equals("3"), "update_rule expected to equal 3"), + new Condition<>(c -> c.equals("3"), "delete_rule expected to equal 3")); final List assertions = new ArrayList<>(); assertEquals(1, results.size()); for (int i = 0; i < matchers.size(); i++) { final String actual = results.get(0).get(i); - final Matcher expected = matchers.get(i); - assertions.add(() -> MatcherAssert.assertThat(actual, expected)); + final Condition expected = matchers.get(i); + assertions.add(() -> assertThat(actual).satisfies(expected)); } assertAll(assertions); } @@ -954,28 +943,30 @@ public void testGetCommandImportedKeys() throws Exception { final List> results = getResults(stream); - final List> matchers = + final List> matchers = asList( - nullValue(String.class), // pk_catalog_name - is("APP"), // pk_schema_name - is("FOREIGNTABLE"), // pk_table_name - is("ID"), // pk_column_name - nullValue(String.class), // fk_catalog_name - is("APP"), // fk_schema_name - is("INTTABLE"), // fk_table_name - is("FOREIGNID"), // fk_column_name - is("1"), // key_sequence - containsString("SQL"), // fk_key_name - containsString("SQL"), // pk_key_name - is("3"), // update_rule - is("3")); // delete_rule + new Condition<>(Objects::isNull, "pk_catalog_name expected to be null"), + new Condition<>(c -> c.equals("APP"), "pk_schema_name expected to equal APP"), + new Condition<>( + c -> c.equals("FOREIGNTABLE"), "pk_table_name should equal FOREIGNTABLE"), + new Condition<>(c -> c.equals("ID"), "pk_column_name should equal ID"), + new Condition<>(Objects::isNull, "fk_catalog_name expected to be null"), + new Condition<>(c -> c.equals("APP"), "fk_schema_name expected to be APP"), + new Condition<>(c -> c.equals("INTTABLE"), "fk_table_name expeced to be INTTABLE"), + new Condition<>( + c -> c.equals("FOREIGNID"), "fk_column_name expected to equal FOREIGNID"), + new Condition<>(c -> c.equals("1"), "key_sequence expected to equal 1"), + new Condition<>(c -> c.contains("SQL"), "fk_key_name expected to contain SQL"), + new Condition<>(c -> c.contains("SQL"), "pk_key_name expected to contain SQL"), + new Condition<>(c -> c.equals("3"), "update_rule expected to equal 3"), + new Condition<>(c -> c.equals("3"), "delete_rule expected to equal 3")); assertEquals(1, results.size()); final List assertions = new ArrayList<>(); for (int i = 0; i < matchers.size(); i++) { final String actual = results.get(0).get(i); - final Matcher expected = matchers.get(i); - assertions.add(() -> MatcherAssert.assertThat(actual, expected)); + final Condition expected = matchers.get(i); + assertions.add(() -> assertThat(actual).satisfies(expected)); } assertAll(assertions); } @@ -1431,7 +1422,7 @@ public void testGetTypeInfo() throws Exception { null, null, null)); - MatcherAssert.assertThat(results, is(matchers)); + assertThat(results).isEqualTo(matchers); } } @@ -1465,7 +1456,7 @@ public void testGetTypeInfoWithFiltering() throws Exception { null, "10", null)); - MatcherAssert.assertThat(results, is(matchers)); + assertThat(results).isEqualTo(matchers); } } @@ -1479,28 +1470,30 @@ public void testGetCommandCrossReference() throws Exception { final List> results = getResults(stream); - final List> matchers = + final List> matchers = asList( - nullValue(String.class), // pk_catalog_name - is("APP"), // pk_schema_name - is("FOREIGNTABLE"), // pk_table_name - is("ID"), // pk_column_name - nullValue(String.class), // fk_catalog_name - is("APP"), // fk_schema_name - is("INTTABLE"), // fk_table_name - is("FOREIGNID"), // fk_column_name - is("1"), // key_sequence - containsString("SQL"), // fk_key_name - containsString("SQL"), // pk_key_name - is("3"), // update_rule - is("3")); // delete_rule + new Condition<>(Objects::isNull, "pk_catalog_name expected to be null"), + new Condition<>(c -> c.equals("APP"), "pk_schema_name expected to equal APP"), + new Condition<>( + c -> c.equals("FOREIGNTABLE"), "pk_table_name should equal FOREIGNTABLE"), + new Condition<>(c -> c.equals("ID"), "pk_column_name should equal ID"), + new Condition<>(Objects::isNull, "fk_catalog_name expected to be null"), + new Condition<>(c -> c.equals("APP"), "fk_schema_name expected to be APP"), + new Condition<>(c -> c.equals("INTTABLE"), "fk_table_name expeced to be INTTABLE"), + new Condition<>( + c -> c.equals("FOREIGNID"), "fk_column_name expected to equal FOREIGNID"), + new Condition<>(c -> c.equals("1"), "key_sequence expected to equal 1"), + new Condition<>(c -> c.contains("SQL"), "fk_key_name expected to contain SQL"), + new Condition<>(c -> c.contains("SQL"), "pk_key_name expected to contain SQL"), + new Condition<>(c -> c.equals("3"), "update_rule expected to equal 3"), + new Condition<>(c -> c.equals("3"), "delete_rule expected to equal 3")); assertEquals(1, results.size()); final List assertions = new ArrayList<>(); for (int i = 0; i < matchers.size(); i++) { final String actual = results.get(0).get(i); - final Matcher expected = matchers.get(i); - assertions.add(() -> MatcherAssert.assertThat(actual, expected)); + final Condition expected = matchers.get(i); + assertions.add(() -> assertThat(actual).satisfies(expected)); } assertAll(assertions); } @@ -1509,7 +1502,7 @@ public void testGetCommandCrossReference() throws Exception { @Test public void testCreateStatementSchema() throws Exception { final FlightInfo info = sqlClient.execute("SELECT * FROM intTable"); - MatcherAssert.assertThat(info.getSchemaOptional(), is(Optional.of(SCHEMA_INT_TABLE))); + assertThat(info.getSchemaOptional()).isEqualTo(Optional.of(SCHEMA_INT_TABLE)); // Consume statement to close connection before cache eviction try (FlightStream stream = sqlClient.getStream(info.getEndpoints().get(0).getTicket())) { @@ -1526,11 +1519,10 @@ public void testCreateStatementResults() throws Exception { sqlClient.execute("SELECT * FROM intTable").getEndpoints().get(0).getTicket())) { assertAll( () -> { - MatcherAssert.assertThat(stream.getSchema(), is(SCHEMA_INT_TABLE)); + assertThat(stream.getSchema()).isEqualTo(SCHEMA_INT_TABLE); }, () -> { - MatcherAssert.assertThat( - getResults(stream), is(EXPECTED_RESULTS_FOR_STAR_SELECT_QUERY)); + assertThat(getResults(stream)).isEqualTo(EXPECTED_RESULTS_FOR_STAR_SELECT_QUERY); }); } } @@ -1543,19 +1535,19 @@ public void testExecuteUpdate() { sqlClient.executeUpdate( "INSERT INTO INTTABLE (keyName, value) VALUES " + "('KEYNAME1', 1001), ('KEYNAME2', 1002), ('KEYNAME3', 1003)"); - MatcherAssert.assertThat(insertedCount, is(3L)); + assertThat(insertedCount).isEqualTo(3L); }, () -> { long updatedCount = sqlClient.executeUpdate( "UPDATE INTTABLE SET keyName = 'KEYNAME1' " + "WHERE keyName = 'KEYNAME2' OR keyName = 'KEYNAME3'"); - MatcherAssert.assertThat(updatedCount, is(2L)); + assertThat(updatedCount).isEqualTo(2L); }, () -> { long deletedCount = sqlClient.executeUpdate("DELETE FROM INTTABLE WHERE keyName = 'KEYNAME1'"); - MatcherAssert.assertThat(deletedCount, is(3L)); + assertThat(deletedCount).isEqualTo(3L); }); } @@ -1566,10 +1558,10 @@ public void testQueryWithNoResultsShouldNotHang() throws Exception { final FlightStream stream = sqlClient.getStream(preparedStatement.execute().getEndpoints().get(0).getTicket())) { assertAll( - () -> MatcherAssert.assertThat(stream.getSchema(), is(SCHEMA_INT_TABLE)), + () -> assertThat(stream.getSchema()).isEqualTo(SCHEMA_INT_TABLE), () -> { final List> result = getResults(stream); - MatcherAssert.assertThat(result, is(emptyList())); + assertThat(result).isEqualTo(emptyList()); }); } } diff --git a/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStateless.java b/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStateless.java index 36d621ad64..ee1507b6af 100644 --- a/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStateless.java +++ b/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStateless.java @@ -18,7 +18,7 @@ import static org.apache.arrow.flight.sql.util.FlightStreamUtils.getResults; import static org.apache.arrow.util.AutoCloseables.close; -import static org.hamcrest.CoreMatchers.is; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertAll; import org.apache.arrow.flight.FlightClient; @@ -34,7 +34,6 @@ import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.types.pojo.Schema; -import org.hamcrest.MatcherAssert; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -89,10 +88,10 @@ public void testSimplePreparedStatementResultsWithParameterBinding() throws Exce for (FlightEndpoint endpoint : flightInfo.getEndpoints()) { try (FlightStream stream = sqlClient.getStream(endpoint.getTicket())) { assertAll( - () -> MatcherAssert.assertThat(stream.getSchema(), is(SCHEMA_INT_TABLE)), + () -> assertThat(stream.getSchema()).isEqualTo(SCHEMA_INT_TABLE), () -> - MatcherAssert.assertThat( - getResults(stream), is(EXPECTED_RESULTS_FOR_PARAMETER_BINDING))); + assertThat(getResults(stream)) + .isEqualTo(EXPECTED_RESULTS_FOR_PARAMETER_BINDING)); } } } diff --git a/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStreams.java b/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStreams.java index 71c0dc88e4..3f527f961e 100644 --- a/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStreams.java +++ b/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStreams.java @@ -22,7 +22,7 @@ import static org.apache.arrow.flight.sql.util.FlightStreamUtils.getResults; import static org.apache.arrow.util.AutoCloseables.close; import static org.apache.arrow.vector.types.Types.MinorType.INT; -import static org.hamcrest.CoreMatchers.is; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertAll; import com.google.common.collect.ImmutableList; @@ -53,7 +53,6 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.Text; -import org.hamcrest.MatcherAssert; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -245,15 +244,15 @@ public void testGetTablesResultNoSchema() throws Exception { sqlClient.getTables(null, null, null, null, false).getEndpoints().get(0).getTicket())) { assertAll( () -> - MatcherAssert.assertThat( - stream.getSchema(), is(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA_NO_SCHEMA)), + assertThat(stream.getSchema()) + .isEqualTo(FlightSqlProducer.Schemas.GET_TABLES_SCHEMA_NO_SCHEMA), () -> { final List> results = getResults(stream); final List> expectedResults = ImmutableList.of( // catalog_name | schema_name | table_name | table_type | table_schema asList(null, null, "test_table", "TABLE")); - MatcherAssert.assertThat(results, is(expectedResults)); + assertThat(results).isEqualTo(expectedResults); }); } } @@ -264,15 +263,15 @@ public void testGetTableTypesResult() throws Exception { sqlClient.getStream(sqlClient.getTableTypes().getEndpoints().get(0).getTicket())) { assertAll( () -> - MatcherAssert.assertThat( - stream.getSchema(), is(FlightSqlProducer.Schemas.GET_TABLE_TYPES_SCHEMA)), + assertThat(stream.getSchema()) + .isEqualTo(FlightSqlProducer.Schemas.GET_TABLE_TYPES_SCHEMA), () -> { final List> tableTypes = getResults(stream); final List> expectedTableTypes = ImmutableList.of( // table_type singletonList("TABLE")); - MatcherAssert.assertThat(tableTypes, is(expectedTableTypes)); + assertThat(tableTypes).isEqualTo(expectedTableTypes); }); } } @@ -283,9 +282,9 @@ public void testGetSqlInfoResults() throws Exception { try (final FlightStream stream = sqlClient.getStream(info.getEndpoints().get(0).getTicket())) { assertAll( () -> - MatcherAssert.assertThat( - stream.getSchema(), is(FlightSqlProducer.Schemas.GET_SQL_INFO_SCHEMA)), - () -> MatcherAssert.assertThat(getResults(stream), is(emptyList()))); + assertThat(stream.getSchema()) + .isEqualTo(FlightSqlProducer.Schemas.GET_SQL_INFO_SCHEMA), + () -> assertThat(getResults(stream)).isEqualTo(emptyList())); } } @@ -303,7 +302,7 @@ public void testGetTypeInfo() throws Exception { "Integer", "4", "400", null, null, "3", "true", null, "true", null, "true", "Integer", null, null, "4", null, "10", null)); - MatcherAssert.assertThat(results, is(matchers)); + assertThat(results).isEqualTo(matchers); } } @@ -317,10 +316,8 @@ public void testExecuteQuery() throws Exception { .get(0) .getTicket())) { assertAll( - () -> - MatcherAssert.assertThat(stream.getSchema(), is(FlightSqlTestProducer.FIXED_SCHEMA)), - () -> - MatcherAssert.assertThat(getResults(stream), is(singletonList(singletonList("1"))))); + () -> assertThat(stream.getSchema()).isEqualTo(FlightSqlTestProducer.FIXED_SCHEMA), + () -> assertThat(getResults(stream)).isEqualTo(singletonList(singletonList("1")))); } } } From fad4e142f398492533bb1dea32accfcad0a33be8 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 2 Jun 2025 11:42:45 +0900 Subject: [PATCH 044/271] MINOR: Add missing permission to milestone assignment bot (#673) ## What's Changed This step needs permissions to write to issues so we can set the milestone. --- .github/workflows/dev_pr.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml index 34b3363c50..7352137b09 100644 --- a/.github/workflows/dev_pr.yml +++ b/.github/workflows/dev_pr.yml @@ -80,5 +80,9 @@ jobs: if: '! github.event.pull_request.draft' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + permissions: + contents: read + issues: write + pull-requests: write run: | ./.github/workflows/dev_pr_milestone.sh "${GITHUB_REPOSITORY}" ${{ github.event.number }} From 17958324c65e0c2109733ed4022ccd25b23e1ff2 Mon Sep 17 00:00:00 2001 From: rtadepalli <105760760+rtadepalli@users.noreply.github.com> Date: Wed, 4 Jun 2025 08:15:34 -0400 Subject: [PATCH 045/271] GH-774: Consoliate `BitVectorHelper.getValidityBufferSize` and `BaseValueVector.getValidityBufferSizeFromCount` (#775) ## What's Changed Just removing some duplicate functions in anticipation of cleaning out some transferPair duplication across complex vectors. Closes #774 --------- Co-authored-by: David Li --- .../arrow/vector/BaseFixedWidthVector.java | 10 ++++---- .../vector/BaseLargeVariableWidthVector.java | 6 ++--- .../apache/arrow/vector/BaseValueVector.java | 9 +++++++- .../arrow/vector/BaseVariableWidthVector.java | 6 ++--- .../vector/BaseVariableWidthViewVector.java | 6 ++--- .../org/apache/arrow/vector/BitVector.java | 6 ++--- .../apache/arrow/vector/BitVectorHelper.java | 12 +++++----- .../vector/complex/FixedSizeListVector.java | 16 +++++++------ .../arrow/vector/complex/LargeListVector.java | 18 ++++++++------- .../vector/complex/LargeListViewVector.java | 20 ++++++++-------- .../arrow/vector/complex/ListVector.java | 20 ++++++++-------- .../arrow/vector/complex/ListViewVector.java | 20 ++++++++-------- .../arrow/vector/complex/MapVector.java | 2 +- .../arrow/vector/complex/StructVector.java | 23 ++++++++++--------- .../arrow/vector/ipc/JsonFileReader.java | 3 ++- .../arrow/vector/TestLargeListVector.java | 3 ++- .../arrow/vector/TestLargeListViewVector.java | 3 ++- .../apache/arrow/vector/TestListVector.java | 3 ++- .../arrow/vector/TestListViewVector.java | 3 ++- .../apache/arrow/vector/TestValueVector.java | 13 ++++++----- .../vector/TestVariableWidthViewVector.java | 3 ++- .../arrow/vector/TestVectorUnloadLoad.java | 3 ++- 22 files changed, 117 insertions(+), 91 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java index 4be55396b7..d126266cf5 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java @@ -359,7 +359,7 @@ public int getBufferSizeFor(final int count) { if (count == 0) { return 0; } - return (count * typeWidth) + getValidityBufferSizeFromCount(count); + return (count * typeWidth) + BitVectorHelper.getValidityBufferSizeFromCount(count); } /** @@ -372,7 +372,7 @@ public int getBufferSize() { if (valueCount == 0) { return 0; } - return (valueCount * typeWidth) + getValidityBufferSizeFromCount(valueCount); + return (valueCount * typeWidth) + BitVectorHelper.getValidityBufferSizeFromCount(valueCount); } /** @@ -536,10 +536,10 @@ private void setReaderAndWriterIndex() { validityBuffer.writerIndex(0); valueBuffer.writerIndex(0); } else { - validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); if (typeWidth == 0) { /* specialized handling for BitVector */ - valueBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + valueBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); } else { valueBuffer.writerIndex((long) valueCount * typeWidth); } @@ -664,7 +664,7 @@ private void splitAndTransferValidityBuffer( int startIndex, int length, BaseFixedWidthVector target) { int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getValidityBufferSizeFromCount(length); + int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); int offset = startIndex % 8; if (length > 0) { diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java index 7e0d0affc6..4245e0053b 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java @@ -379,7 +379,7 @@ private void setReaderAndWriterIndex() { valueBuffer.writerIndex(0); } else { final long lastDataOffset = getStartOffset(valueCount); - validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); offsetBuffer.writerIndex((long) (valueCount + 1) * OFFSET_WIDTH); valueBuffer.writerIndex(lastDataOffset); } @@ -633,7 +633,7 @@ public int getBufferSizeFor(final int valueCount) { return 0; } - final long validityBufferSize = getValidityBufferSizeFromCount(valueCount); + final long validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount); final long offsetBufferSize = (long) (valueCount + 1) * OFFSET_WIDTH; /* get the end offset for this valueCount */ final long dataBufferSize = getStartOffset(valueCount); @@ -816,7 +816,7 @@ private void splitAndTransferValidityBuffer( int startIndex, int length, BaseLargeVariableWidthVector target) { int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getValidityBufferSizeFromCount(length); + int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); int offset = startIndex % 8; if (length > 0) { diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java index 9befcb890f..7bff431e40 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java @@ -110,7 +110,14 @@ protected ArrowBuf releaseBuffer(ArrowBuf buffer) { return buffer; } - /* number of bytes for the validity buffer for the given valueCount */ + /** + * Compute the size of validity buffer required to manage a given number of elements in a vector. + * + * @param valueCount number of elements in the vector + * @return buffer size + * @deprecated -- use {@link BitVectorHelper#getValidityBufferSizeFromCount} instead. + */ + @Deprecated(forRemoval = true, since = "18.4.0") protected static int getValidityBufferSizeFromCount(final int valueCount) { return DataSizeRoundingUtil.divideBy8Ceil(valueCount); } diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java index 1609e64ca5..4f681311ed 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java @@ -395,7 +395,7 @@ private void setReaderAndWriterIndex() { valueBuffer.writerIndex(0); } else { final int lastDataOffset = getStartOffset(valueCount); - validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); offsetBuffer.writerIndex((long) (valueCount + 1) * OFFSET_WIDTH); valueBuffer.writerIndex(lastDataOffset); } @@ -673,7 +673,7 @@ public int getBufferSizeFor(final int valueCount) { return 0; } - final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); + final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount); final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; /* get the end offset for this valueCount */ final int dataBufferSize = offsetBuffer.getInt((long) valueCount * OFFSET_WIDTH); @@ -867,7 +867,7 @@ private void splitAndTransferValidityBuffer( final int firstByteSource = BitVectorHelper.byteIndex(startIndex); final int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - final int byteSizeTarget = getValidityBufferSizeFromCount(length); + final int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); final int offset = startIndex % 8; if (offset == 0) { diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java index beda91dc3f..5e25ffa568 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java @@ -400,7 +400,7 @@ private void setReaderAndWriterIndex() { validityBuffer.writerIndex(0); viewBuffer.writerIndex(0); } else { - validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); viewBuffer.writerIndex(valueCount * ELEMENT_SIZE); } } @@ -683,7 +683,7 @@ public int getBufferSizeFor(final int valueCount) { return 0; } - final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); + final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount); final int viewBufferSize = valueCount * ELEMENT_SIZE; final int dataBufferSize = getDataBufferSize(); return validityBufferSize + viewBufferSize + dataBufferSize; @@ -872,7 +872,7 @@ private void splitAndTransferValidityBuffer( final int firstByteSource = BitVectorHelper.byteIndex(startIndex); final int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - final int byteSizeTarget = getValidityBufferSizeFromCount(length); + final int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); final int offset = startIndex % 8; if (offset == 0) { diff --git a/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/vector/src/main/java/org/apache/arrow/vector/BitVector.java index f8e3342625..ecee02f665 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BitVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BitVector.java @@ -98,7 +98,7 @@ public MinorType getMinorType() { */ @Override public void setInitialCapacity(int valueCount) { - final int size = getValidityBufferSizeFromCount(valueCount); + final int size = BitVectorHelper.getValidityBufferSizeFromCount(valueCount); if (size * 2L > MAX_ALLOCATION_SIZE) { throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); } @@ -121,7 +121,7 @@ public int getBufferSizeFor(final int count) { if (count == 0) { return 0; } - return 2 * getValidityBufferSizeFromCount(count); + return 2 * BitVectorHelper.getValidityBufferSizeFromCount(count); } /** @@ -165,7 +165,7 @@ private ArrowBuf splitAndTransferBuffer( int startIndex, int length, ArrowBuf sourceBuffer, ArrowBuf destBuffer) { int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getValidityBufferSizeFromCount(length); + int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); int offset = startIndex % 8; if (length > 0) { diff --git a/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java b/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java index 0ac56691a6..bc2c3da98f 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java +++ b/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java @@ -135,11 +135,11 @@ public static void setValidityBit(ArrowBuf validityBuffer, int index, int value) public static ArrowBuf setValidityBit( ArrowBuf validityBuffer, BufferAllocator allocator, int valueCount, int index, int value) { if (validityBuffer == null) { - validityBuffer = allocator.buffer(getValidityBufferSize(valueCount)); + validityBuffer = allocator.buffer(getValidityBufferSizeFromCount(valueCount)); } setValidityBit(validityBuffer, index, value); if (index == (valueCount - 1)) { - validityBuffer.writerIndex(getValidityBufferSize(valueCount)); + validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); } return validityBuffer; @@ -165,7 +165,7 @@ public static int get(final ArrowBuf buffer, int index) { * @param valueCount number of elements in the vector * @return buffer size */ - public static int getValidityBufferSize(int valueCount) { + public static int getValidityBufferSizeFromCount(int valueCount) { return DataSizeRoundingUtil.divideBy8Ceil(valueCount); } @@ -182,7 +182,7 @@ public static int getNullCount(final ArrowBuf validityBuffer, final int valueCou return 0; } int count = 0; - final int sizeInBytes = getValidityBufferSize(valueCount); + final int sizeInBytes = getValidityBufferSizeFromCount(valueCount); // If value count is not a multiple of 8, then calculate number of used bits in the last byte final int remainder = valueCount % 8; final int fullBytesCount = remainder == 0 ? sizeInBytes : sizeInBytes - 1; @@ -233,7 +233,7 @@ public static boolean checkAllBitsEqualTo( if (valueCount == 0) { return true; } - final int sizeInBytes = getValidityBufferSize(valueCount); + final int sizeInBytes = getValidityBufferSizeFromCount(valueCount); // boundary check validityBuffer.checkBytes(0, sizeInBytes); @@ -325,7 +325,7 @@ public static ArrowBuf loadValidityBuffer( sourceValidityBuffer == null || sourceValidityBuffer.capacity() == 0; if (isValidityBufferNull && (fieldNode.getNullCount() == 0 || fieldNode.getNullCount() == valueCount)) { - newBuffer = allocator.buffer(getValidityBufferSize(valueCount)); + newBuffer = allocator.buffer(getValidityBufferSizeFromCount(valueCount)); newBuffer.setZero(0, newBuffer.capacity()); if (fieldNode.getNullCount() != 0) { /* all NULLs */ diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index c762eb5172..36d9ff40ed 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -110,7 +110,8 @@ public FixedSizeListVector( this.listSize = ((ArrowType.FixedSizeList) field.getFieldType().getType()).getListSize(); Preconditions.checkArgument(listSize >= 0, "list size must be non-negative"); this.valueCount = 0; - this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); + this.validityAllocationSizeInBytes = + BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); } @Override @@ -189,7 +190,7 @@ public List getFieldBuffers() { private void setReaderAndWriterIndex() { validityBuffer.readerIndex(0); - validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); } /** @@ -268,7 +269,8 @@ private void reallocValidityBuffer() { if (validityAllocationSizeInBytes > 0) { newAllocationSize = validityAllocationSizeInBytes; } else { - newAllocationSize = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L; + newAllocationSize = + BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L; } } @@ -311,7 +313,7 @@ public UnionFixedSizeListWriter getWriter() { @Override public void setInitialCapacity(int numRecords) { - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords); vector.setInitialCapacity(numRecords * listSize); } @@ -328,7 +330,7 @@ public int getBufferSize() { if (getValueCount() == 0) { return 0; } - return getValidityBufferSizeFromCount(valueCount) + vector.getBufferSize(); + return BitVectorHelper.getValidityBufferSizeFromCount(valueCount) + vector.getBufferSize(); } @Override @@ -336,7 +338,7 @@ public int getBufferSizeFor(int valueCount) { if (valueCount == 0) { return 0; } - return getValidityBufferSizeFromCount(valueCount) + return BitVectorHelper.getValidityBufferSizeFromCount(valueCount) + vector.getBufferSizeFor(valueCount * listSize); } @@ -654,7 +656,7 @@ private void splitAndTransferValidityBuffer( int startIndex, int length, FixedSizeListVector target) { int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getValidityBufferSizeFromCount(length); + int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); int offset = startIndex % 8; if (length > 0) { diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java index ed075352c9..71633441cb 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java @@ -131,7 +131,8 @@ public LargeListVector(Field field, BufferAllocator allocator, CallBack callBack this.field = field; this.validityBuffer = allocator.getEmpty(); this.callBack = callBack; - this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); + this.validityAllocationSizeInBytes = + BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); this.lastSet = -1; this.offsetBuffer = allocator.getEmpty(); this.vector = vector == null ? DEFAULT_DATA_VECTOR : vector; @@ -156,7 +157,7 @@ public void initializeChildrenFromFields(List children) { @Override public void setInitialCapacity(int numRecords) { - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords); offsetAllocationSizeInBytes = (long) (numRecords + 1) * OFFSET_WIDTH; if (vector instanceof BaseFixedWidthVector || vector instanceof BaseVariableWidthVector) { vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD); @@ -184,7 +185,7 @@ public void setInitialCapacity(int numRecords) { */ @Override public void setInitialCapacity(int numRecords, double density) { - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords); if ((numRecords * density) >= Integer.MAX_VALUE) { throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); } @@ -311,7 +312,7 @@ private void setReaderAndWriterIndex() { validityBuffer.writerIndex(0); offsetBuffer.writerIndex(0); } else { - validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH); } } @@ -442,7 +443,8 @@ private void reallocValidityBuffer() { if (validityAllocationSizeInBytes > 0) { newAllocationSize = validityAllocationSizeInBytes; } else { - newAllocationSize = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L; + newAllocationSize = + BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L; } } newAllocationSize = CommonUtil.nextPowerOfTwo(newAllocationSize); @@ -699,7 +701,7 @@ private void splitAndTransferValidityBuffer( int startIndex, int length, LargeListVector target) { int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getValidityBufferSizeFromCount(length); + int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); int offset = startIndex % 8; if (length > 0) { @@ -821,7 +823,7 @@ public int getBufferSize() { return 0; } final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; - final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); + final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount); return offsetBufferSize + validityBufferSize + vector.getBufferSize(); } @@ -830,7 +832,7 @@ public int getBufferSizeFor(int valueCount) { if (valueCount == 0) { return 0; } - final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); + final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount); long innerVectorValueCount = offsetBuffer.getLong((long) valueCount * OFFSET_WIDTH); return ((valueCount + 1) * OFFSET_WIDTH) diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java index 84c6f03edb..1b7e6b2280 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java @@ -113,7 +113,8 @@ public LargeListViewVector(Field field, BufferAllocator allocator, CallBack call this.validityBuffer = allocator.getEmpty(); this.field = field; this.callBack = callBack; - this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); + this.validityAllocationSizeInBytes = + BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); } @Override @@ -134,7 +135,7 @@ public void initializeChildrenFromFields(List children) { @Override public void setInitialCapacity(int numRecords) { - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords); super.setInitialCapacity(numRecords); } @@ -157,7 +158,7 @@ public void setInitialCapacity(int numRecords) { */ @Override public void setInitialCapacity(int numRecords, double density) { - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords); super.setInitialCapacity(numRecords, density); } @@ -176,7 +177,7 @@ public void setInitialCapacity(int numRecords, double density) { */ @Override public void setInitialTotalCapacity(int numRecords, int totalNumberOfElements) { - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords); super.setInitialTotalCapacity(numRecords, totalNumberOfElements); } @@ -226,7 +227,7 @@ private void setReaderAndWriterIndex() { offsetBuffer.writerIndex(0); sizeBuffer.writerIndex(0); } else { - validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); offsetBuffer.writerIndex((long) valueCount * OFFSET_WIDTH); sizeBuffer.writerIndex((long) valueCount * SIZE_WIDTH); } @@ -323,7 +324,8 @@ private long getNewAllocationSize(int currentBufferCapacity) { if (validityAllocationSizeInBytes > 0) { newAllocationSize = validityAllocationSizeInBytes; } else { - newAllocationSize = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L; + newAllocationSize = + BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L; } } newAllocationSize = CommonUtil.nextPowerOfTwo(newAllocationSize); @@ -536,7 +538,7 @@ private void splitAndTransferValidityBuffer( int startIndex, int length, LargeListViewVector target) { int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getValidityBufferSizeFromCount(length); + int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); int offset = startIndex % 8; if (length > 0) { @@ -629,7 +631,7 @@ public int getBufferSize() { } final int offsetBufferSize = valueCount * OFFSET_WIDTH; final int sizeBufferSize = valueCount * SIZE_WIDTH; - final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); + final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount); return offsetBufferSize + sizeBufferSize + validityBufferSize + vector.getBufferSize(); } @@ -644,7 +646,7 @@ public int getBufferSizeFor(int valueCount) { if (valueCount == 0) { return 0; } - final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); + final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount); return super.getBufferSizeFor(valueCount) + validityBufferSize; } diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 3daeb6d77b..a8e8dcc436 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -108,7 +108,8 @@ public ListVector(Field field, BufferAllocator allocator, CallBack callBack) { this.validityBuffer = allocator.getEmpty(); this.field = field; this.callBack = callBack; - this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); + this.validityAllocationSizeInBytes = + BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); this.lastSet = -1; } @@ -130,7 +131,7 @@ public void initializeChildrenFromFields(List children) { @Override public void setInitialCapacity(int numRecords) { - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords); super.setInitialCapacity(numRecords); } @@ -153,7 +154,7 @@ public void setInitialCapacity(int numRecords) { */ @Override public void setInitialCapacity(int numRecords, double density) { - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords); super.setInitialCapacity(numRecords, density); } @@ -172,7 +173,7 @@ public void setInitialCapacity(int numRecords, double density) { */ @Override public void setInitialTotalCapacity(int numRecords, int totalNumberOfElements) { - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords); super.setInitialTotalCapacity(numRecords, totalNumberOfElements); } @@ -269,7 +270,7 @@ private void setReaderAndWriterIndex() { validityBuffer.writerIndex(0); offsetBuffer.writerIndex(0); } else { - validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH); } } @@ -366,7 +367,8 @@ private long getNewAllocationSize(int currentBufferCapacity) { if (validityAllocationSizeInBytes > 0) { newAllocationSize = validityAllocationSizeInBytes; } else { - newAllocationSize = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L; + newAllocationSize = + BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L; } } newAllocationSize = CommonUtil.nextPowerOfTwo(newAllocationSize); @@ -579,7 +581,7 @@ public void splitAndTransfer(int startIndex, int length) { private void splitAndTransferValidityBuffer(int startIndex, int length, ListVector target) { int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getValidityBufferSizeFromCount(length); + int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); int offset = startIndex % 8; if (length > 0) { @@ -678,7 +680,7 @@ public int getBufferSize() { return 0; } final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; - final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); + final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount); return offsetBufferSize + validityBufferSize + vector.getBufferSize(); } @@ -687,7 +689,7 @@ public int getBufferSizeFor(int valueCount) { if (valueCount == 0) { return 0; } - final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); + final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount); return super.getBufferSizeFor(valueCount) + validityBufferSize; } diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java index 9b4e6b4c0c..ada25bbaf5 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java @@ -112,7 +112,8 @@ public ListViewVector(Field field, BufferAllocator allocator, CallBack callBack) this.validityBuffer = allocator.getEmpty(); this.field = field; this.callBack = callBack; - this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); + this.validityAllocationSizeInBytes = + BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); } @Override @@ -133,7 +134,7 @@ public void initializeChildrenFromFields(List children) { @Override public void setInitialCapacity(int numRecords) { - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords); super.setInitialCapacity(numRecords); } @@ -156,7 +157,7 @@ public void setInitialCapacity(int numRecords) { */ @Override public void setInitialCapacity(int numRecords, double density) { - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords); super.setInitialCapacity(numRecords, density); } @@ -175,7 +176,7 @@ public void setInitialCapacity(int numRecords, double density) { */ @Override public void setInitialTotalCapacity(int numRecords, int totalNumberOfElements) { - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSizeFromCount(numRecords); super.setInitialTotalCapacity(numRecords, totalNumberOfElements); } @@ -225,7 +226,7 @@ private void setReaderAndWriterIndex() { offsetBuffer.writerIndex(0); sizeBuffer.writerIndex(0); } else { - validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); offsetBuffer.writerIndex(valueCount * OFFSET_WIDTH); sizeBuffer.writerIndex(valueCount * SIZE_WIDTH); } @@ -322,7 +323,8 @@ private long getNewAllocationSize(int currentBufferCapacity) { if (validityAllocationSizeInBytes > 0) { newAllocationSize = validityAllocationSizeInBytes; } else { - newAllocationSize = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L; + newAllocationSize = + BitVectorHelper.getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L; } } newAllocationSize = CommonUtil.nextPowerOfTwo(newAllocationSize); @@ -542,7 +544,7 @@ public void splitAndTransfer(int startIndex, int length) { private void splitAndTransferValidityBuffer(int startIndex, int length, ListViewVector target) { int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getValidityBufferSizeFromCount(length); + int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); int offset = startIndex % 8; if (length > 0) { @@ -634,7 +636,7 @@ public int getBufferSize() { } final int offsetBufferSize = valueCount * OFFSET_WIDTH; final int sizeBufferSize = valueCount * SIZE_WIDTH; - final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); + final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount); return offsetBufferSize + sizeBufferSize + validityBufferSize + vector.getBufferSize(); } @@ -649,7 +651,7 @@ public int getBufferSizeFor(int valueCount) { if (valueCount == 0) { return 0; } - final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); + final int validityBufferSize = BitVectorHelper.getValidityBufferSizeFromCount(valueCount); return super.getBufferSizeFor(valueCount) + validityBufferSize; } diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java index 23cda8401b..5eb857ab94 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java @@ -238,7 +238,7 @@ public void splitAndTransfer(int startIndex, int length) { private void splitAndTransferValidityBuffer(int startIndex, int length, MapVector target) { int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getValidityBufferSizeFromCount(length); + int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); int offset = startIndex % 8; if (length > 0) { diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java index ca5f572034..5e5bb7fc21 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java @@ -18,6 +18,7 @@ import static org.apache.arrow.memory.util.LargeMemoryUtil.checkedCastToInt; import static org.apache.arrow.util.Preconditions.checkNotNull; +import static org.apache.arrow.vector.BitVectorHelper.getValidityBufferSizeFromCount; import java.util.ArrayList; import java.util.Arrays; @@ -89,7 +90,7 @@ public StructVector( super(name, checkNotNull(allocator), fieldType, callBack); this.validityBuffer = allocator.getEmpty(); this.validityAllocationSizeInBytes = - BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION); + getValidityBufferSizeFromCount(BaseValueVector.INITIAL_VALUE_ALLOCATION); } /** @@ -118,7 +119,7 @@ public StructVector( allowConflictPolicyChanges); this.validityBuffer = allocator.getEmpty(); this.validityAllocationSizeInBytes = - BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION); + getValidityBufferSizeFromCount(BaseValueVector.INITIAL_VALUE_ALLOCATION); } /** @@ -132,7 +133,7 @@ public StructVector(Field field, BufferAllocator allocator, CallBack callBack) { super(field, checkNotNull(allocator), callBack); this.validityBuffer = allocator.getEmpty(); this.validityAllocationSizeInBytes = - BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION); + getValidityBufferSizeFromCount(BaseValueVector.INITIAL_VALUE_ALLOCATION); } /** @@ -153,7 +154,7 @@ public StructVector( super(field, checkNotNull(allocator), callBack, conflictPolicy, allowConflictPolicyChanges); this.validityBuffer = allocator.getEmpty(); this.validityAllocationSizeInBytes = - BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION); + getValidityBufferSizeFromCount(BaseValueVector.INITIAL_VALUE_ALLOCATION); } @Override @@ -182,7 +183,7 @@ public List getFieldBuffers() { private void setReaderAndWriterIndex() { validityBuffer.readerIndex(0); - validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSize(valueCount)); + validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); } /** @@ -318,7 +319,7 @@ public void splitAndTransfer(int startIndex, int length) { private void splitAndTransferValidityBuffer(int startIndex, int length, StructVector target) { int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = BitVectorHelper.getValidityBufferSize(length); + int byteSizeTarget = getValidityBufferSizeFromCount(length); int offset = startIndex % 8; if (length > 0) { @@ -464,7 +465,7 @@ public int getBufferSize() { if (valueCount == 0) { return 0; } - return super.getBufferSize() + BitVectorHelper.getValidityBufferSize(valueCount); + return super.getBufferSize() + getValidityBufferSizeFromCount(valueCount); } /** @@ -478,18 +479,18 @@ public int getBufferSizeFor(final int valueCount) { if (valueCount == 0) { return 0; } - return super.getBufferSizeFor(valueCount) + BitVectorHelper.getValidityBufferSize(valueCount); + return super.getBufferSizeFor(valueCount) + getValidityBufferSizeFromCount(valueCount); } @Override public void setInitialCapacity(int numRecords) { - validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(numRecords); + validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); super.setInitialCapacity(numRecords); } @Override public void setInitialCapacity(int numRecords, double density) { - validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(numRecords); + validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); super.setInitialCapacity(numRecords, density); } @@ -547,7 +548,7 @@ private long getNewAllocationSize(int currentBufferCapacity) { newAllocationSize = validityAllocationSizeInBytes; } else { newAllocationSize = - BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION) * 2L; + getValidityBufferSizeFromCount(BaseValueVector.INITIAL_VALUE_ALLOCATION) * 2L; } } newAllocationSize = CommonUtil.nextPowerOfTwo(newAllocationSize); diff --git a/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java b/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java index fe0803d298..e4bab7eb80 100644 --- a/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java +++ b/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java @@ -20,6 +20,7 @@ import static com.fasterxml.jackson.core.JsonToken.END_OBJECT; import static com.fasterxml.jackson.core.JsonToken.START_ARRAY; import static com.fasterxml.jackson.core.JsonToken.START_OBJECT; +import static org.apache.arrow.vector.BitVectorHelper.getValidityBufferSizeFromCount; import static org.apache.arrow.vector.BufferLayout.BufferType.DATA; import static org.apache.arrow.vector.BufferLayout.BufferType.OFFSET; import static org.apache.arrow.vector.BufferLayout.BufferType.SIZE; @@ -381,7 +382,7 @@ private class BufferHelper { new BufferReader() { @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { - final int bufferSize = BitVectorHelper.getValidityBufferSize(count); + final int bufferSize = getValidityBufferSizeFromCount(count); ArrowBuf buf = allocator.buffer(bufferSize); // C++ integration test fails without this. diff --git a/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java index 101d942d2a..d5cbf925b2 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.vector; +import static org.apache.arrow.vector.BitVectorHelper.getValidityBufferSizeFromCount; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; @@ -943,7 +944,7 @@ public void testGetBufferSizeFor() { int[] indices = new int[] {0, 2, 4, 6, 10, 14}; for (int valueCount = 1; valueCount <= 5; valueCount++) { - int validityBufferSize = BitVectorHelper.getValidityBufferSize(valueCount); + int validityBufferSize = getValidityBufferSizeFromCount(valueCount); int offsetBufferSize = (valueCount + 1) * LargeListVector.OFFSET_WIDTH; int expectedSize = diff --git a/vector/src/test/java/org/apache/arrow/vector/TestLargeListViewVector.java b/vector/src/test/java/org/apache/arrow/vector/TestLargeListViewVector.java index 26e7bb4a0d..256aa99687 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestLargeListViewVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestLargeListViewVector.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.vector; +import static org.apache.arrow.vector.BitVectorHelper.getValidityBufferSizeFromCount; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertSame; @@ -1062,7 +1063,7 @@ public void testGetBufferSizeFor() { int[] indices = new int[] {0, 2, 4, 6, 10, 14}; for (int valueCount = 1; valueCount <= 5; valueCount++) { - int validityBufferSize = BitVectorHelper.getValidityBufferSize(valueCount); + int validityBufferSize = getValidityBufferSizeFromCount(valueCount); int offsetBufferSize = valueCount * BaseLargeRepeatedValueViewVector.OFFSET_WIDTH; int sizeBufferSize = valueCount * BaseLargeRepeatedValueViewVector.SIZE_WIDTH; diff --git a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index 1d6fa39f9e..5b2043a014 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.vector; +import static org.apache.arrow.vector.BitVectorHelper.getValidityBufferSizeFromCount; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; @@ -1123,7 +1124,7 @@ public void testGetBufferSizeFor() { int[] indices = new int[] {0, 2, 4, 6, 10, 14}; for (int valueCount = 1; valueCount <= 5; valueCount++) { - int validityBufferSize = BitVectorHelper.getValidityBufferSize(valueCount); + int validityBufferSize = getValidityBufferSizeFromCount(valueCount); int offsetBufferSize = (valueCount + 1) * BaseRepeatedValueVector.OFFSET_WIDTH; int expectedSize = diff --git a/vector/src/test/java/org/apache/arrow/vector/TestListViewVector.java b/vector/src/test/java/org/apache/arrow/vector/TestListViewVector.java index 639585fc48..2f282e1988 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestListViewVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestListViewVector.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.vector; +import static org.apache.arrow.vector.BitVectorHelper.getValidityBufferSizeFromCount; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -1075,7 +1076,7 @@ public void testGetBufferSizeFor() { int[] indices = new int[] {0, 2, 4, 6, 10, 14}; for (int valueCount = 1; valueCount <= 5; valueCount++) { - int validityBufferSize = BitVectorHelper.getValidityBufferSize(valueCount); + int validityBufferSize = getValidityBufferSizeFromCount(valueCount); int offsetBufferSize = valueCount * BaseRepeatedValueViewVector.OFFSET_WIDTH; int sizeBufferSize = valueCount * BaseRepeatedValueViewVector.SIZE_WIDTH; diff --git a/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index daec331831..ac82246671 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.vector; +import static org.apache.arrow.vector.BitVectorHelper.getValidityBufferSizeFromCount; import static org.apache.arrow.vector.TestUtils.newVarBinaryVector; import static org.apache.arrow.vector.TestUtils.newVarCharVector; import static org.apache.arrow.vector.TestUtils.newVector; @@ -1233,7 +1234,7 @@ public void testSplitAndTransfer3() { // the size needed for the validity buffer final long validitySize = DefaultRoundingPolicy.DEFAULT_ROUNDING_POLICY.getRoundedSize( - BaseValueVector.getValidityBufferSizeFromCount(2)); + getValidityBufferSizeFromCount(2)); assertEquals(allocatedMem + validitySize, allocator.getAllocatedMemory()); // The validity and offset buffers are sliced from a same buffer.See // BaseFixedWidthVector#allocateBytes. @@ -2464,7 +2465,7 @@ public void testDefaultAllocNewAll() { assertTrue(intVector.getValueCapacity() >= defaultCapacity); expectedSize = (defaultCapacity * IntVector.TYPE_WIDTH) - + BaseFixedWidthVector.getValidityBufferSizeFromCount(defaultCapacity); + + getValidityBufferSizeFromCount(defaultCapacity); assertTrue(childAllocator.getAllocatedMemory() - beforeSize <= expectedSize * 1.05); // verify that the wastage is within bounds for BigIntVector. @@ -2473,7 +2474,7 @@ public void testDefaultAllocNewAll() { assertTrue(bigIntVector.getValueCapacity() >= defaultCapacity); expectedSize = (defaultCapacity * bigIntVector.TYPE_WIDTH) - + BaseFixedWidthVector.getValidityBufferSizeFromCount(defaultCapacity); + + getValidityBufferSizeFromCount(defaultCapacity); assertTrue(childAllocator.getAllocatedMemory() - beforeSize <= expectedSize * 1.05); // verify that the wastage is within bounds for DecimalVector. @@ -2482,7 +2483,7 @@ public void testDefaultAllocNewAll() { assertTrue(decimalVector.getValueCapacity() >= defaultCapacity); expectedSize = (defaultCapacity * decimalVector.TYPE_WIDTH) - + BaseFixedWidthVector.getValidityBufferSizeFromCount(defaultCapacity); + + getValidityBufferSizeFromCount(defaultCapacity); assertTrue(childAllocator.getAllocatedMemory() - beforeSize <= expectedSize * 1.05); // verify that the wastage is within bounds for VarCharVector. @@ -2492,7 +2493,7 @@ public void testDefaultAllocNewAll() { assertTrue(varCharVector.getValueCapacity() >= defaultCapacity - 1); expectedSize = (defaultCapacity * VarCharVector.OFFSET_WIDTH) - + BaseFixedWidthVector.getValidityBufferSizeFromCount(defaultCapacity) + + getValidityBufferSizeFromCount(defaultCapacity) + defaultCapacity * 8; // wastage should be less than 5%. assertTrue(childAllocator.getAllocatedMemory() - beforeSize <= expectedSize * 1.05); @@ -2501,7 +2502,7 @@ public void testDefaultAllocNewAll() { beforeSize = childAllocator.getAllocatedMemory(); bitVector.allocateNew(); assertTrue(bitVector.getValueCapacity() >= defaultCapacity); - expectedSize = BaseFixedWidthVector.getValidityBufferSizeFromCount(defaultCapacity) * 2; + expectedSize = getValidityBufferSizeFromCount(defaultCapacity) * 2; assertTrue(childAllocator.getAllocatedMemory() - beforeSize <= expectedSize * 1.05); } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestVariableWidthViewVector.java b/vector/src/test/java/org/apache/arrow/vector/TestVariableWidthViewVector.java index 7a3a1bae63..f7c66a00be 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestVariableWidthViewVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestVariableWidthViewVector.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.vector; +import static org.apache.arrow.vector.BitVectorHelper.getValidityBufferSizeFromCount; import static org.apache.arrow.vector.TestUtils.newVector; import static org.apache.arrow.vector.TestUtils.newViewVarBinaryVector; import static org.apache.arrow.vector.TestUtils.newViewVarCharVector; @@ -2367,7 +2368,7 @@ private void testSplitAndTransferOnValiditySplitHelper( // the allocation only consists in the size needed for the validity buffer final long validitySize = DefaultRoundingPolicy.DEFAULT_ROUNDING_POLICY.getRoundedSize( - BaseValueVector.getValidityBufferSizeFromCount(2)); + getValidityBufferSizeFromCount(2)); // we allocate view and data buffers for the target vector assertTrue(allocatedMem + validitySize < allocator.getAllocatedMemory()); // The validity is sliced from the same buffer.See BaseFixedWidthViewVector#allocateBytes. diff --git a/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java index 6121fb67fe..782535fccc 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java @@ -17,6 +17,7 @@ package org.apache.arrow.vector; import static java.util.Arrays.asList; +import static org.apache.arrow.vector.BitVectorHelper.getValidityBufferSizeFromCount; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -215,7 +216,7 @@ public void testLoadValidityBuffer() throws IOException { int count = 10; ArrowBuf[] values = new ArrowBuf[4]; for (int i = 0; i < 4; i += 2) { - ArrowBuf buf1 = allocator.buffer(BitVectorHelper.getValidityBufferSize(count)); + ArrowBuf buf1 = allocator.buffer(getValidityBufferSizeFromCount(count)); ArrowBuf buf2 = allocator.buffer(count * 4); // integers buf1.setZero(0, buf1.capacity()); buf2.setZero(0, buf2.capacity()); From acbc138f75711fc94e842f9b306a48a5a233bca8 Mon Sep 17 00:00:00 2001 From: rtadepalli <105760760+rtadepalli@users.noreply.github.com> Date: Wed, 4 Jun 2025 22:57:25 -0400 Subject: [PATCH 046/271] GH-79: Move `splitAndTransferValidityBuffer` to `BaseValueVector` (#777) ## What's Changed Move `splitAndTransferValidityBuffer` up to `BaseValueVector`. This PR is not touching the implementation of this function in `StructVector` -- that is not being derived from `BaseValueVector` so some amount of duplication is probably fine. Closes #79 --- .../arrow/vector/BaseFixedWidthVector.java | 86 +++---------- .../vector/BaseLargeVariableWidthVector.java | 79 ++---------- .../apache/arrow/vector/BaseValueVector.java | 116 ++++++++++++++++++ .../arrow/vector/BaseVariableWidthVector.java | 79 ++---------- .../vector/BaseVariableWidthViewVector.java | 85 ++----------- .../BaseLargeRepeatedValueViewVector.java | 1 - .../complex/BaseRepeatedValueVector.java | 1 - .../complex/BaseRepeatedValueViewVector.java | 1 - .../vector/complex/FixedSizeListVector.java | 77 +----------- .../arrow/vector/complex/LargeListVector.java | 77 +----------- .../vector/complex/LargeListViewVector.java | 74 +---------- .../arrow/vector/complex/ListVector.java | 73 +---------- .../arrow/vector/complex/ListViewVector.java | 73 +---------- .../arrow/vector/complex/MapVector.java | 65 ---------- 14 files changed, 182 insertions(+), 705 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java index d126266cf5..f6e2a3b225 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java @@ -49,9 +49,7 @@ public abstract class BaseFixedWidthVector extends BaseValueVector protected final Field field; private int allocationMonitor; - protected ArrowBuf validityBuffer; protected ArrowBuf valueBuffer; - protected int valueCount; /** * Constructs a new instance. @@ -87,7 +85,7 @@ public String getName() { /* TODO: * Once the entire hierarchy has been refactored, move common functions - * like getNullCount(), splitAndTransferValidityBuffer to top level + * like getNullCount() to top level * base class BaseValueVector. * * Along with this, some class members (validityBuffer) can also be @@ -342,9 +340,9 @@ private void allocateBytes(int valueCount) { * slice the source buffer so we have to explicitly allocate the validityBuffer of the target * vector. This is unlike the databuffer which we can always slice for the target vector. */ - private void allocateValidityBuffer(final int validityBufferSize) { - validityBuffer = allocator.buffer(validityBufferSize); - validityBuffer.readerIndex(0); + @Override + protected void allocateValidityBuffer(final long validityBufferSize) { + super.allocateValidityBuffer(validityBufferSize); refreshValueCapacity(); } @@ -656,72 +654,18 @@ private void splitAndTransferValueBuffer( target.refreshValueCapacity(); } - /** - * Validity buffer has multiple cases of split and transfer depending on the starting position of - * the source index. - */ - private void splitAndTransferValidityBuffer( - int startIndex, int length, BaseFixedWidthVector target) { - int firstByteSource = BitVectorHelper.byteIndex(startIndex); - int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); - int offset = startIndex % 8; - - if (length > 0) { - if (offset == 0) { - /* slice */ - if (target.validityBuffer != null) { - target.validityBuffer.getReferenceManager().release(); - } - ArrowBuf slicedValidityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer = transferBuffer(slicedValidityBuffer, target.allocator); - target.refreshValueCapacity(); - } else { - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ - target.allocateValidityBuffer(byteSizeTarget); - - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - this.validityBuffer, firstByteSource + i, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - this.validityBuffer, firstByteSource + i + 1, offset); - - target.validityBuffer.setByte(i, (b1 + b2)); - } - - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ - if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - this.validityBuffer, firstByteSource + byteSizeTarget, offset); - - target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } else { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - target.validityBuffer.setByte(byteSizeTarget - 1, b1); - } - } + @Override + protected void sliceAndTransferValidityBuffer( + int startIndex, int length, BaseValueVector target) { + final int firstByteSource = BitVectorHelper.byteIndex(startIndex); + final int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); + + if (target.validityBuffer != null) { + target.validityBuffer.getReferenceManager().release(); } + ArrowBuf slicedValidityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer = transferBuffer(slicedValidityBuffer, target.allocator); + ((BaseFixedWidthVector) target).refreshValueCapacity(); } /*----------------------------------------------------------------* diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java index 4245e0053b..6c451f10a7 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java @@ -52,10 +52,8 @@ public abstract class BaseLargeVariableWidthVector extends BaseValueVector /* protected members */ public static final int OFFSET_WIDTH = 8; /* 8 byte unsigned int to track offsets */ protected static final byte[] emptyByteArray = new byte[] {}; - protected ArrowBuf validityBuffer; protected ArrowBuf valueBuffer; protected ArrowBuf offsetBuffer; - protected int valueCount; protected int lastSet; protected final Field field; @@ -501,10 +499,9 @@ private ArrowBuf allocateOffsetBuffer(final long size) { } /* allocate validity buffer */ - private void allocateValidityBuffer(final long size) { - validityBuffer = allocator.buffer(size); - validityBuffer.readerIndex(0); - initValidityBuffer(); + @Override + protected void allocateValidityBuffer(final long size) { + super.allocateValidityBuffer(size); } /** @@ -809,69 +806,17 @@ private void splitAndTransferOffsetBuffer( target.valueBuffer = transferBuffer(slicedBuffer, target.allocator); } - /* - * Transfer the validity. - */ - private void splitAndTransferValidityBuffer( - int startIndex, int length, BaseLargeVariableWidthVector target) { - int firstByteSource = BitVectorHelper.byteIndex(startIndex); - int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); - int offset = startIndex % 8; + @Override + protected void sliceAndTransferValidityBuffer( + int startIndex, int length, BaseValueVector target) { + final int firstByteSource = BitVectorHelper.byteIndex(startIndex); + final int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); - if (length > 0) { - if (offset == 0) { - // slice - if (target.validityBuffer != null) { - target.validityBuffer.getReferenceManager().release(); - } - target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer.getReferenceManager().retain(); - } else { - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ - target.allocateValidityBuffer(byteSizeTarget); - - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - this.validityBuffer, firstByteSource + i, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - this.validityBuffer, firstByteSource + i + 1, offset); - - target.validityBuffer.setByte(i, (b1 + b2)); - } - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ - if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - this.validityBuffer, firstByteSource + byteSizeTarget, offset); - - target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } else { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - target.validityBuffer.setByte(byteSizeTarget - 1, b1); - } - } + if (target.validityBuffer != null) { + target.validityBuffer.getReferenceManager().release(); } + target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer.getReferenceManager().retain(); } /*----------------------------------------------------------------* diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java index 7bff431e40..37dfa20616 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java @@ -48,6 +48,10 @@ public abstract class BaseValueVector implements ValueVector { protected volatile FieldReader fieldReader; + protected ArrowBuf validityBuffer; + + protected int valueCount; + protected BaseValueVector(BufferAllocator allocator) { this.allocator = Preconditions.checkNotNull(allocator, "allocator cannot be null"); } @@ -255,4 +259,116 @@ public void copyFrom(int fromIndex, int thisIndex, ValueVector from) { public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { throw new UnsupportedOperationException(); } + + /** + * Transfer the validity buffer from `validityBuffer` to the target vector's `validityBuffer`. + * Start at `startIndex` and copy `length` number of elements. If the starting index is 8 byte + * aligned, then the buffer is sliced from that index and ownership is transferred. If not, + * individual bytes are copied. + * + * @param startIndex starting index + * @param length number of elements to be copied + * @param target target vector + */ + protected void splitAndTransferValidityBuffer( + int startIndex, int length, BaseValueVector target) { + int offset = startIndex % 8; + + if (length <= 0) { + return; + } + if (offset == 0) { + sliceAndTransferValidityBuffer(startIndex, length, target); + } else { + copyValidityBuffer(startIndex, length, target); + } + } + + /** + * If the start index is 8 byte aligned, slice `validityBuffer` and transfer ownership to + * `target`'s `validityBuffer`. + * + * @param startIndex starting index + * @param length number of elements to be copied + * @param target target vector + */ + protected void sliceAndTransferValidityBuffer( + int startIndex, int length, BaseValueVector target) { + final int firstByteSource = BitVectorHelper.byteIndex(startIndex); + final int byteSizeTarget = getValidityBufferSizeFromCount(length); + + if (target.validityBuffer != null) { + target.validityBuffer.getReferenceManager().release(); + } + target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer.getReferenceManager().retain(1); + } + + /** + * Allocate new validity buffer for `target` and copy bytes from `validityBuffer`. Precise details + * in the comments below. + * + * @param startIndex starting index + * @param length number of elements to be copied + * @param target target vector + */ + protected void copyValidityBuffer(int startIndex, int length, BaseValueVector target) { + final int firstByteSource = BitVectorHelper.byteIndex(startIndex); + final int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); + final int byteSizeTarget = getValidityBufferSizeFromCount(length); + final int offset = startIndex % 8; + + /* Copy data + * When the first bit starts from the middle of a byte (offset != 0), + * copy data from src BitVector. + * Each byte in the target is composed by a part in i-th byte, + * another part in (i+1)-th byte. + */ + target.allocateValidityBuffer(byteSizeTarget); + + for (int i = 0; i < byteSizeTarget - 1; i++) { + byte b1 = + BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset); + byte b2 = + BitVectorHelper.getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset); + + target.validityBuffer.setByte(i, (b1 + b2)); + } + + /* Copying the last piece is done in the following manner: + * if the source vector has 1 or more bytes remaining, we copy + * the last piece as a byte formed by shifting data + * from the current byte and the next byte. + * + * if the source vector has no more bytes remaining + * (we are at the last byte), we copy the last piece as a byte + * by shifting data from the current byte. + */ + if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + byte b1 = + BitVectorHelper.getBitsFromCurrentByte( + this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); + byte b2 = + BitVectorHelper.getBitsFromNextByte( + this.validityBuffer, firstByteSource + byteSizeTarget, offset); + + target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); + } else { + byte b1 = + BitVectorHelper.getBitsFromCurrentByte( + this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); + target.validityBuffer.setByte(byteSizeTarget - 1, b1); + } + } + + /** + * Allocate new validity buffer for when the bytes need to be copied over. + * + * @param byteSizeTarget desired size of the buffer + */ + protected void allocateValidityBuffer(long byteSizeTarget) { + validityBuffer = allocator.buffer(byteSizeTarget); + validityBuffer.readerIndex(0); + validityBuffer.setZero(0, validityBuffer.capacity()); + } } diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java index 4f681311ed..96e2afbd29 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java @@ -50,10 +50,8 @@ public abstract class BaseVariableWidthVector extends BaseValueVector /* protected members */ public static final int OFFSET_WIDTH = 4; /* 4 byte unsigned int to track offsets */ protected static final byte[] emptyByteArray = new byte[] {}; - protected ArrowBuf validityBuffer; protected ArrowBuf valueBuffer; protected ArrowBuf offsetBuffer; - protected int valueCount; protected int lastSet; protected final Field field; @@ -87,7 +85,7 @@ public String getName() { /* TODO: * Once the entire hierarchy has been refactored, move common functions - * like getNullCount(), splitAndTransferValidityBuffer to top level + * like getNullCount() to top level * base class BaseValueVector. * * Along with this, some class members (validityBuffer) can also be @@ -519,11 +517,9 @@ private ArrowBuf allocateOffsetBuffer(final long size) { } /* allocate validity buffer */ - private void allocateValidityBuffer(final long size) { - final int curSize = (int) size; - validityBuffer = allocator.buffer(curSize); - validityBuffer.readerIndex(0); - initValidityBuffer(); + @Override + protected void allocateValidityBuffer(final long size) { + super.allocateValidityBuffer(size); } /** @@ -856,70 +852,17 @@ private void splitAndTransferOffsetBuffer( target.valueBuffer = transferBuffer(slicedBuffer, target.allocator); } - /* - * Transfer the validity. - */ - private void splitAndTransferValidityBuffer( - int startIndex, int length, BaseVariableWidthVector target) { - if (length <= 0) { - return; - } - + @Override + protected void sliceAndTransferValidityBuffer( + int startIndex, int length, BaseValueVector target) { final int firstByteSource = BitVectorHelper.byteIndex(startIndex); - final int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); final int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); - final int offset = startIndex % 8; - if (offset == 0) { - // slice - if (target.validityBuffer != null) { - target.validityBuffer.getReferenceManager().release(); - } - final ArrowBuf slicedValidityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer = transferBuffer(slicedValidityBuffer, target.allocator); - return; - } - - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ - target.allocateValidityBuffer(byteSizeTarget); - - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset); - - target.validityBuffer.setByte(i, (b1 + b2)); - } - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ - if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - this.validityBuffer, firstByteSource + byteSizeTarget, offset); - - target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } else { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - target.validityBuffer.setByte(byteSizeTarget - 1, b1); + if (target.validityBuffer != null) { + target.validityBuffer.getReferenceManager().release(); } + final ArrowBuf slicedValidityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer = transferBuffer(slicedValidityBuffer, target.allocator); } /*----------------------------------------------------------------* diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java index 5e25ffa568..ea9de8320e 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java @@ -78,13 +78,11 @@ public abstract class BaseVariableWidthViewVector extends BaseValueVector // The third 4 bytes of view are allocated for buffer index public static final int BUF_INDEX_WIDTH = 4; public static final byte[] EMPTY_BYTE_ARRAY = new byte[] {}; - protected ArrowBuf validityBuffer; // The view buffer is used to store the variable width view elements protected ArrowBuf viewBuffer; // The external buffer which stores the long strings protected List dataBuffers; protected int initialDataBufferSize; - protected int valueCount; protected int lastSet; protected final Field field; @@ -117,7 +115,7 @@ public String getName() { /* TODO: * Once the entire hierarchy has been refactored, move common functions - * like getNullCount(), splitAndTransferValidityBuffer to top level + * like getNullCount() to top level * base class BaseValueVector. * * Along with this, some class members (validityBuffer) can also be @@ -129,12 +127,6 @@ public String getName() { * the top class as of now is not a good idea. */ - /* TODO: - * Implement TransferPair functionality - * https://github.com/apache/arrow/issues/40932 - * - */ - /** * Get buffer that manages the validity (NULL or NON-NULL nature) of elements in the vector. * Consider it as a buffer for internal bit vector data structure. @@ -854,77 +846,22 @@ public void splitAndTransferTo(int startIndex, int length, BaseVariableWidthView } /* allocate validity buffer */ - private void allocateValidityBuffer(final long size) { - final int curSize = (int) size; - validityBuffer = allocator.buffer(curSize); - validityBuffer.readerIndex(0); - initValidityBuffer(); + @Override + protected void allocateValidityBuffer(final long size) { + super.allocateValidityBuffer(size); } - /* - * Transfer the validity. - */ - private void splitAndTransferValidityBuffer( - int startIndex, int length, BaseVariableWidthViewVector target) { - if (length <= 0) { - return; - } - + @Override + protected void sliceAndTransferValidityBuffer( + int startIndex, int length, BaseValueVector target) { final int firstByteSource = BitVectorHelper.byteIndex(startIndex); - final int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); final int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); - final int offset = startIndex % 8; - - if (offset == 0) { - // slice - if (target.validityBuffer != null) { - target.validityBuffer.getReferenceManager().release(); - } - final ArrowBuf slicedValidityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer = transferBuffer(slicedValidityBuffer, target.allocator); - return; - } - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ - target.allocateValidityBuffer(byteSizeTarget); - - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset); - - target.validityBuffer.setByte(i, (b1 + b2)); - } - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ - if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - this.validityBuffer, firstByteSource + byteSizeTarget, offset); - - target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } else { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - target.validityBuffer.setByte(byteSizeTarget - 1, b1); + if (target.validityBuffer != null) { + target.validityBuffer.getReferenceManager().release(); } + final ArrowBuf slicedValidityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer = transferBuffer(slicedValidityBuffer, target.allocator); } /** diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/BaseLargeRepeatedValueViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/BaseLargeRepeatedValueViewVector.java index 12edd6557b..fac3f86bba 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/BaseLargeRepeatedValueViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/BaseLargeRepeatedValueViewVector.java @@ -52,7 +52,6 @@ public abstract class BaseLargeRepeatedValueViewVector extends BaseValueVector protected ArrowBuf sizeBuffer; protected FieldVector vector; protected final CallBack repeatedCallBack; - protected int valueCount; protected long offsetAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * OFFSET_WIDTH; protected long sizeAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * SIZE_WIDTH; private final String name; diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java index fbe83bad52..ee1d65d3e3 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java @@ -54,7 +54,6 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector protected ArrowBuf offsetBuffer; protected FieldVector vector; protected final CallBack repeatedCallBack; - protected int valueCount; protected long offsetAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * OFFSET_WIDTH; private final String name; diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueViewVector.java index e6213316b5..fd7a4ff2c6 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueViewVector.java @@ -52,7 +52,6 @@ public abstract class BaseRepeatedValueViewVector extends BaseValueVector protected ArrowBuf sizeBuffer; protected FieldVector vector; protected final CallBack repeatedCallBack; - protected int valueCount; protected long offsetAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * OFFSET_WIDTH; protected long sizeAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * SIZE_WIDTH; private final String name; diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index 36d9ff40ed..e3b4ab477f 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -69,12 +69,10 @@ public static FixedSizeListVector empty(String name, int size, BufferAllocator a } private FieldVector vector; - private ArrowBuf validityBuffer; private final int listSize; private Field field; private UnionFixedSizeListReader reader; - private int valueCount; private int validityAllocationSizeInBytes; /** @@ -248,12 +246,10 @@ public boolean allocateNewSafe() { return success; } - private void allocateValidityBuffer(final long size) { - final int curSize = (int) size; - validityBuffer = allocator.buffer(curSize); - validityBuffer.readerIndex(0); - validityAllocationSizeInBytes = curSize; - validityBuffer.setZero(0, validityBuffer.capacity()); + @Override + protected void allocateValidityBuffer(final long size) { + super.allocateValidityBuffer(size); + validityAllocationSizeInBytes = (int) size; } @Override @@ -649,71 +645,6 @@ public void splitAndTransfer(int startIndex, int length) { to.setValueCount(length); } - /* - * transfer the validity. - */ - private void splitAndTransferValidityBuffer( - int startIndex, int length, FixedSizeListVector target) { - int firstByteSource = BitVectorHelper.byteIndex(startIndex); - int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); - int offset = startIndex % 8; - - if (length > 0) { - if (offset == 0) { - // slice - if (target.validityBuffer != null) { - target.validityBuffer.getReferenceManager().release(); - } - target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer.getReferenceManager().retain(1); - } else { - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ - target.allocateValidityBuffer(byteSizeTarget); - - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - validityBuffer, firstByteSource + i + 1, offset); - - target.validityBuffer.setByte(i, (b1 + b2)); - } - - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ - if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - validityBuffer, firstByteSource + byteSizeTarget, offset); - - target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } else { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - target.validityBuffer.setByte(byteSizeTarget - 1, b1); - } - } - } - } - @Override public ValueVector getTo() { return to; diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java index 71633441cb..835d3468f3 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java @@ -94,11 +94,9 @@ public static LargeListVector empty(String name, BufferAllocator allocator) { protected ArrowBuf offsetBuffer; protected FieldVector vector; protected final CallBack callBack; - protected int valueCount; protected long offsetAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * OFFSET_WIDTH; protected String defaultDataVectorName = DATA_VECTOR_NAME; - protected ArrowBuf validityBuffer; protected UnionLargeListReader reader; private Field field; private int validityAllocationSizeInBytes; @@ -375,12 +373,10 @@ public boolean allocateNewSafe() { return success; } - private void allocateValidityBuffer(final long size) { - final int curSize = (int) size; - validityBuffer = allocator.buffer(curSize); - validityBuffer.readerIndex(0); - validityAllocationSizeInBytes = curSize; - validityBuffer.setZero(0, validityBuffer.capacity()); + @Override + protected void allocateValidityBuffer(final long size) { + super.allocateValidityBuffer(size); + validityAllocationSizeInBytes = (int) size; } protected ArrowBuf allocateOffsetBuffer(final long size) { @@ -694,71 +690,6 @@ public void splitAndTransfer(int startIndex, int length) { to.setValueCount(length); } - /* - * transfer the validity. - */ - private void splitAndTransferValidityBuffer( - int startIndex, int length, LargeListVector target) { - int firstByteSource = BitVectorHelper.byteIndex(startIndex); - int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); - int offset = startIndex % 8; - - if (length > 0) { - if (offset == 0) { - // slice - if (target.validityBuffer != null) { - target.validityBuffer.getReferenceManager().release(); - } - target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer.getReferenceManager().retain(1); - } else { - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ - target.allocateValidityBuffer(byteSizeTarget); - - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - validityBuffer, firstByteSource + i + 1, offset); - - target.validityBuffer.setByte(i, (b1 + b2)); - } - - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ - if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - validityBuffer, firstByteSource + byteSizeTarget, offset); - - target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } else { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - target.validityBuffer.setByte(byteSizeTarget - 1, b1); - } - } - } - } - @Override public ValueVector getTo() { return to; diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java index 1b7e6b2280..394c3c67bb 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java @@ -77,7 +77,6 @@ public class LargeListViewVector extends BaseLargeRepeatedValueViewVector implements PromotableVector, ValueIterableVector> { - protected ArrowBuf validityBuffer; protected UnionLargeListViewReader reader; private CallBack callBack; protected Field field; @@ -285,12 +284,10 @@ public boolean allocateNewSafe() { return success; } + @Override protected void allocateValidityBuffer(final long size) { - final int curSize = (int) size; - validityBuffer = allocator.buffer(curSize); - validityBuffer.readerIndex(0); - validityAllocationSizeInBytes = curSize; - validityBuffer.setZero(0, validityBuffer.capacity()); + super.allocateValidityBuffer(size); + validityAllocationSizeInBytes = (int) size; } @Override @@ -531,71 +528,6 @@ public void splitAndTransfer(int startIndex, int length) { } } - /* - * transfer the validity. - */ - private void splitAndTransferValidityBuffer( - int startIndex, int length, LargeListViewVector target) { - int firstByteSource = BitVectorHelper.byteIndex(startIndex); - int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); - int offset = startIndex % 8; - - if (length > 0) { - if (offset == 0) { - // slice - if (target.validityBuffer != null) { - target.validityBuffer.getReferenceManager().release(); - } - target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer.getReferenceManager().retain(1); - } else { - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ - target.allocateValidityBuffer(byteSizeTarget); - - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - validityBuffer, firstByteSource + i + 1, offset); - - target.validityBuffer.setByte(i, (b1 + b2)); - } - - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ - if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - validityBuffer, firstByteSource + byteSizeTarget, offset); - - target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } else { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - target.validityBuffer.setByte(byteSizeTarget - 1, b1); - } - } - } - } - @Override public ValueVector getTo() { return to; diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index a8e8dcc436..2b2817515f 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -74,7 +74,6 @@ public static ListVector empty(String name, BufferAllocator allocator) { return new ListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null); } - protected ArrowBuf validityBuffer; protected UnionListReader reader; private CallBack callBack; protected Field field; @@ -324,12 +323,10 @@ public boolean allocateNewSafe() { return success; } + @Override protected void allocateValidityBuffer(final long size) { - final int curSize = (int) size; - validityBuffer = allocator.buffer(curSize); - validityBuffer.readerIndex(0); - validityAllocationSizeInBytes = curSize; - validityBuffer.setZero(0, validityBuffer.capacity()); + super.allocateValidityBuffer(size); + validityAllocationSizeInBytes = (int) size; } /** @@ -575,70 +572,6 @@ public void splitAndTransfer(int startIndex, int length) { } } - /* - * transfer the validity. - */ - private void splitAndTransferValidityBuffer(int startIndex, int length, ListVector target) { - int firstByteSource = BitVectorHelper.byteIndex(startIndex); - int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); - int offset = startIndex % 8; - - if (length > 0) { - if (offset == 0) { - // slice - if (target.validityBuffer != null) { - target.validityBuffer.getReferenceManager().release(); - } - target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer.getReferenceManager().retain(1); - } else { - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ - target.allocateValidityBuffer(byteSizeTarget); - - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - validityBuffer, firstByteSource + i + 1, offset); - - target.validityBuffer.setByte(i, (b1 + b2)); - } - - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ - if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - validityBuffer, firstByteSource + byteSizeTarget, offset); - - target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } else { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - target.validityBuffer.setByte(byteSizeTarget - 1, b1); - } - } - } - } - @Override public ValueVector getTo() { return to; diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java index ada25bbaf5..2b80101926 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java @@ -76,7 +76,6 @@ public class ListViewVector extends BaseRepeatedValueViewVector implements PromotableVector, ValueIterableVector> { - protected ArrowBuf validityBuffer; protected UnionListViewReader reader; private CallBack callBack; protected Field field; @@ -284,12 +283,10 @@ public boolean allocateNewSafe() { return success; } + @Override protected void allocateValidityBuffer(final long size) { - final int curSize = (int) size; - validityBuffer = allocator.buffer(curSize); - validityBuffer.readerIndex(0); - validityAllocationSizeInBytes = curSize; - validityBuffer.setZero(0, validityBuffer.capacity()); + super.allocateValidityBuffer(size); + validityAllocationSizeInBytes = (int) size; } @Override @@ -538,70 +535,6 @@ public void splitAndTransfer(int startIndex, int length) { } } - /* - * transfer the validity. - */ - private void splitAndTransferValidityBuffer(int startIndex, int length, ListViewVector target) { - int firstByteSource = BitVectorHelper.byteIndex(startIndex); - int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); - int offset = startIndex % 8; - - if (length > 0) { - if (offset == 0) { - // slice - if (target.validityBuffer != null) { - target.validityBuffer.getReferenceManager().release(); - } - target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer.getReferenceManager().retain(1); - } else { - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ - target.allocateValidityBuffer(byteSizeTarget); - - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - validityBuffer, firstByteSource + i + 1, offset); - - target.validityBuffer.setByte(i, (b1 + b2)); - } - - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ - if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - validityBuffer, firstByteSource + byteSizeTarget, offset); - - target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } else { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - target.validityBuffer.setByte(byteSizeTarget - 1, b1); - } - } - } - } - @Override public ValueVector getTo() { return to; diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java index 5eb857ab94..3f98322ba9 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java @@ -22,7 +22,6 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.AddOrGetResult; -import org.apache.arrow.vector.BitVectorHelper; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.ZeroVector; @@ -232,70 +231,6 @@ public void splitAndTransfer(int startIndex, int length) { } } - /* - * transfer the validity. - */ - private void splitAndTransferValidityBuffer(int startIndex, int length, MapVector target) { - int firstByteSource = BitVectorHelper.byteIndex(startIndex); - int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = BitVectorHelper.getValidityBufferSizeFromCount(length); - int offset = startIndex % 8; - - if (length > 0) { - if (offset == 0) { - // slice - if (target.validityBuffer != null) { - target.validityBuffer.getReferenceManager().release(); - } - target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer.getReferenceManager().retain(1); - } else { - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ - target.allocateValidityBuffer(byteSizeTarget); - - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - validityBuffer, firstByteSource + i + 1, offset); - - target.validityBuffer.setByte(i, (b1 + b2)); - } - - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ - if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - byte b2 = - BitVectorHelper.getBitsFromNextByte( - validityBuffer, firstByteSource + byteSizeTarget, offset); - - target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } else { - byte b1 = - BitVectorHelper.getBitsFromCurrentByte( - validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - target.validityBuffer.setByte(byteSizeTarget - 1, b1); - } - } - } - } - @Override public ValueVector getTo() { return to; From 43b6b6ccf9107353d22cd2a90632ef2608d872c5 Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Tue, 1 Jul 2025 09:16:52 +0800 Subject: [PATCH 047/271] GH-759: Get length of byte[] in TryCopyLastError (#760) ## What's Changed We should get the length of byte[] by `GetArrayLength`, not `strlen` which may cause invalid memory access. Closes #759. --- c/src/main/cpp/jni_wrapper.cc | 3 +- .../org/apache/arrow/c/ExceptionTest.java | 150 ++++++++++++++++++ 2 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 c/src/test/java/org/apache/arrow/c/ExceptionTest.java diff --git a/c/src/main/cpp/jni_wrapper.cc b/c/src/main/cpp/jni_wrapper.cc index 35c2b7787e..436cbdc806 100644 --- a/c/src/main/cpp/jni_wrapper.cc +++ b/c/src/main/cpp/jni_wrapper.cc @@ -205,8 +205,9 @@ void TryCopyLastError(JNIEnv* env, InnerPrivateData* private_data) { return; } + jsize error_bytes_len = env->GetArrayLength(arr); char* error_str = reinterpret_cast(error_bytes); - private_data->last_error_ = std::string(error_str, std::strlen(error_str)); + private_data->last_error_ = std::string(error_str, error_bytes_len); env->ReleaseByteArrayElements(arr, error_bytes, JNI_ABORT); } diff --git a/c/src/test/java/org/apache/arrow/c/ExceptionTest.java b/c/src/test/java/org/apache/arrow/c/ExceptionTest.java new file mode 100644 index 0000000000..5bc96a8f99 --- /dev/null +++ b/c/src/test/java/org/apache/arrow/c/ExceptionTest.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.c; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowableOfType; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.VectorLoader; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.dictionary.Dictionary; +import org.apache.arrow.vector.dictionary.DictionaryProvider; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.Test; + +// Regression test for https://github.com/apache/arrow-java/issues/759 +final class ExceptionTest { + @Test + public void testException() throws IOException { + final Schema schema = + new Schema(Collections.singletonList(Field.nullable("ints", new ArrowType.Int(32, true)))); + final List batches = new ArrayList<>(); + + try (BufferAllocator allocator = new RootAllocator(); + VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { + + final String exceptionMessage = "This is a message for testing exception."; + + RuntimeException exToThrow = new RuntimeException(exceptionMessage); + batches.add(exToThrow); + + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + exToThrow.printStackTrace(pw); + final String expectExceptionMessage = sw.toString(); + + ArrowReader source = new ExceptionMemoryArrowReader(allocator, schema, batches); + + try (final ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator); + final VectorSchemaRoot importRoot = VectorSchemaRoot.create(schema, allocator)) { + final VectorLoader loader = new VectorLoader(importRoot); + Data.exportArrayStream(allocator, source, stream); + + try (final ArrowReader reader = Data.importArrayStream(allocator, stream)) { + IOException jniException = catchThrowableOfType(IOException.class, reader::loadNextBatch); + final String jniMessage = jniException.getMessage(); + assertThat(jniMessage.endsWith(expectExceptionMessage + "}")); + } + } + } + } + + static class ExceptionMemoryArrowReader extends ArrowReader { + private final Schema schema; + private final List batches; // set ArrowRecordBatch or Exception + private final DictionaryProvider provider; + private int nextBatch; + + ExceptionMemoryArrowReader(BufferAllocator allocator, Schema schema, List batches) { + super(allocator); + this.schema = schema; + this.batches = batches; + this.provider = new CDataDictionaryProvider(); + this.nextBatch = 0; + } + + @Override + public Dictionary lookup(long id) { + return provider.lookup(id); + } + + @Override + public Set getDictionaryIds() { + return provider.getDictionaryIds(); + } + + @Override + public Map getDictionaryVectors() { + return getDictionaryIds().stream() + .collect(Collectors.toMap(Function.identity(), this::lookup)); + } + + @Override + public boolean loadNextBatch() throws IOException { + if (nextBatch < batches.size()) { + Object object = batches.get(nextBatch++); + if (object instanceof RuntimeException) { + throw (RuntimeException) object; + } + VectorLoader loader = new VectorLoader(getVectorSchemaRoot()); + loader.load((ArrowRecordBatch) object); + return true; + } + return false; + } + + @Override + public long bytesRead() { + return 0; + } + + @Override + protected void closeReadSource() throws IOException { + try { + for (Object object : batches) { + if (object instanceof ArrowRecordBatch) { + ArrowRecordBatch batch = (ArrowRecordBatch) object; + batch.close(); + } + } + } catch (Exception e) { + throw new IOException(e); + } + } + + @Override + protected Schema readSchema() { + return schema; + } + } +} From 45295a589d64da5f9838cfad7cd29ec4db481047 Mon Sep 17 00:00:00 2001 From: Martin Traverse Date: Tue, 8 Jul 2025 01:57:20 +0100 Subject: [PATCH 048/271] GH-731: Avro adapter, output dictionary-encoded fields as enums (#779) ## What's Changed Updated ArrowToAvro to output dictionary-encoded string vectors as Avro enums, where possible. Apologies for the delay - busy as usual! To output dict encoded vectors as enums, a dictionary provider must be supplied to the top level methods with all the required dictionaries. All dictionary values must be present when the schema is written, i.e. before the data blocks are produced. If data is being written as a schema followed by multiple blocks, values added to a dictionary in between blocks will not be included in the schema resulting in an invalid Avro file (in general supply an invalid dictionary mapping will result in invalid output). Dictionary encoded fields are checked to ensure they are valid Avro enums. If the dictionary encoded field is not a string field, or the string values are not valid Avro enums, the field is decoded and output as literal values. This is done by calling DictionaryEncoder.decode(vector, dictionary), which will consume memory for the vector. An alternative approach would be to decode values one-by-one, however this would require a significant change to the producer pattern since the current producers expect concrete vectors of the output type. Another option would be to throw an error if there are dictionary-encoded vectors that are not string types, i.e. push the responsibility onto client code. I'm not sure which approach is best - happy to take any guidance and I will update the code accordingly. To read enums back the current approach for decoding is unchanged (the AvroToArrow config has to be set up with a MapDictionaryProvider which is populated when data is read). The last part of the Avro work is to add the capability for reading / writing whole files block-by-block, so there is an opportunity to do something with the top level APIs there, for now the current API works and I've used it in the round trip tests. Please let me know any feedback, happy to update as needed! Closes #731. --- .../arrow/adapter/avro/ArrowToAvroUtils.java | 226 +++++++++++++++--- .../avro/producers/AvroEnumProducer.java | 12 +- .../producers/DictionaryDecodingProducer.java | 47 ++++ .../adapter/avro/ArrowToAvroDataTest.java | 81 +++++++ .../adapter/avro/ArrowToAvroSchemaTest.java | 129 ++++++++++ .../arrow/adapter/avro/RoundTripDataTest.java | 110 ++++++++- .../adapter/avro/RoundTripSchemaTest.java | 63 ++++- 7 files changed, 618 insertions(+), 50 deletions(-) create mode 100644 adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/DictionaryDecodingProducer.java diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/ArrowToAvroUtils.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/ArrowToAvroUtils.java index 87b594af9e..e09b99f670 100644 --- a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/ArrowToAvroUtils.java +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/ArrowToAvroUtils.java @@ -17,10 +17,14 @@ package org.apache.arrow.adapter.avro; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; import org.apache.arrow.adapter.avro.producers.AvroBigIntProducer; import org.apache.arrow.adapter.avro.producers.AvroBooleanProducer; import org.apache.arrow.adapter.avro.producers.AvroBytesProducer; +import org.apache.arrow.adapter.avro.producers.AvroEnumProducer; import org.apache.arrow.adapter.avro.producers.AvroFixedSizeBinaryProducer; import org.apache.arrow.adapter.avro.producers.AvroFixedSizeListProducer; import org.apache.arrow.adapter.avro.producers.AvroFloat2Producer; @@ -41,6 +45,7 @@ import org.apache.arrow.adapter.avro.producers.AvroUint8Producer; import org.apache.arrow.adapter.avro.producers.BaseAvroProducer; import org.apache.arrow.adapter.avro.producers.CompositeAvroProducer; +import org.apache.arrow.adapter.avro.producers.DictionaryDecodingProducer; import org.apache.arrow.adapter.avro.producers.Producer; import org.apache.arrow.adapter.avro.producers.logical.AvroDateDayProducer; import org.apache.arrow.adapter.avro.producers.logical.AvroDateMilliProducer; @@ -59,6 +64,7 @@ import org.apache.arrow.adapter.avro.producers.logical.AvroTimestampSecProducer; import org.apache.arrow.adapter.avro.producers.logical.AvroTimestampSecTzProducer; import org.apache.arrow.util.Preconditions; +import org.apache.arrow.vector.BaseIntVector; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.DateDayVector; @@ -96,11 +102,14 @@ import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.StructVector; +import org.apache.arrow.vector.dictionary.Dictionary; +import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.util.Text; import org.apache.avro.LogicalType; import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; @@ -162,17 +171,29 @@ public class ArrowToAvroUtils { * may be nullable. Record types must contain at least one child field and cannot contain multiple * fields with the same name * + *

String fields that are dictionary-encoded will be represented as an Avro enum, so long as + * all the values meet the restrictions on Avro enums (non-null, valid identifiers). Other data + * types that are dictionary encoded, or string fields that do not meet the avro requirements, + * will be output as their decoded type. + * * @param arrowFields The arrow fields used to generate the Avro schema * @param typeName Name of the top level Avro record type * @param namespace Namespace of the top level Avro record type + * @param dictionaries A dictionary provider is required if any fields use dictionary encoding * @return An Avro record schema for the given list of fields, with the specified name and * namespace */ public static Schema createAvroSchema( - List arrowFields, String typeName, String namespace) { + List arrowFields, String typeName, String namespace, DictionaryProvider dictionaries) { SchemaBuilder.RecordBuilder assembler = SchemaBuilder.record(typeName).namespace(namespace); - return buildRecordSchema(assembler, arrowFields, namespace); + return buildRecordSchema(assembler, arrowFields, namespace, dictionaries); + } + + /** Overload provided for convenience, sets dictionaries = null. */ + public static Schema createAvroSchema( + List arrowFields, String typeName, String namespace) { + return createAvroSchema(arrowFields, typeName, namespace, null); } /** Overload provided for convenience, sets namespace = null. */ @@ -185,61 +206,83 @@ public static Schema createAvroSchema(List arrowFields) { return createAvroSchema(arrowFields, GENERIC_RECORD_TYPE_NAME); } + /** + * Overload provided for convenience, sets name = GENERIC_RECORD_TYPE_NAME and namespace = null. + */ + public static Schema createAvroSchema(List arrowFields, DictionaryProvider dictionaries) { + return createAvroSchema(arrowFields, GENERIC_RECORD_TYPE_NAME, null, dictionaries); + } + private static T buildRecordSchema( - SchemaBuilder.RecordBuilder builder, List fields, String namespace) { + SchemaBuilder.RecordBuilder builder, + List fields, + String namespace, + DictionaryProvider dictionaries) { if (fields.isEmpty()) { throw new IllegalArgumentException("Record field must have at least one child field"); } SchemaBuilder.FieldAssembler assembler = builder.namespace(namespace).fields(); for (Field field : fields) { - assembler = buildFieldSchema(assembler, field, namespace); + assembler = buildFieldSchema(assembler, field, namespace, dictionaries); } return assembler.endRecord(); } private static SchemaBuilder.FieldAssembler buildFieldSchema( - SchemaBuilder.FieldAssembler assembler, Field field, String namespace) { + SchemaBuilder.FieldAssembler assembler, + Field field, + String namespace, + DictionaryProvider dictionaries) { return assembler .name(field.getName()) - .type(buildTypeSchema(SchemaBuilder.builder(), field, namespace)) + .type(buildTypeSchema(SchemaBuilder.builder(), field, namespace, dictionaries)) .noDefault(); } private static T buildTypeSchema( - SchemaBuilder.TypeBuilder builder, Field field, String namespace) { + SchemaBuilder.TypeBuilder builder, + Field field, + String namespace, + DictionaryProvider dictionaries) { // Nullable unions need special handling, since union types cannot be directly nested if (field.getType().getTypeID() == ArrowType.ArrowTypeID.Union) { boolean unionNullable = field.getChildren().stream().anyMatch(Field::isNullable); if (unionNullable) { SchemaBuilder.UnionAccumulator union = builder.unionOf().nullType(); - return addTypesToUnion(union, field.getChildren(), namespace); + return addTypesToUnion(union, field.getChildren(), namespace, dictionaries); } else { Field headType = field.getChildren().get(0); List tailTypes = field.getChildren().subList(1, field.getChildren().size()); SchemaBuilder.UnionAccumulator union = - buildBaseTypeSchema(builder.unionOf(), headType, namespace); - return addTypesToUnion(union, tailTypes, namespace); + buildBaseTypeSchema(builder.unionOf(), headType, namespace, dictionaries); + return addTypesToUnion(union, tailTypes, namespace, dictionaries); } } else if (field.isNullable()) { - return buildBaseTypeSchema(builder.nullable(), field, namespace); + return buildBaseTypeSchema(builder.nullable(), field, namespace, dictionaries); } else { - return buildBaseTypeSchema(builder, field, namespace); + return buildBaseTypeSchema(builder, field, namespace, dictionaries); } } private static T buildArraySchema( - SchemaBuilder.ArrayBuilder builder, Field listField, String namespace) { + SchemaBuilder.ArrayBuilder builder, + Field listField, + String namespace, + DictionaryProvider dictionaries) { if (listField.getChildren().size() != 1) { throw new IllegalArgumentException("List field must have exactly one child field"); } Field itemField = listField.getChildren().get(0); - return buildTypeSchema(builder.items(), itemField, namespace); + return buildTypeSchema(builder.items(), itemField, namespace, dictionaries); } private static T buildMapSchema( - SchemaBuilder.MapBuilder builder, Field mapField, String namespace) { + SchemaBuilder.MapBuilder builder, + Field mapField, + String namespace, + DictionaryProvider dictionaries) { if (mapField.getChildren().size() != 1) { throw new IllegalArgumentException("Map field must have exactly one child field"); } @@ -253,11 +296,14 @@ private static T buildMapSchema( throw new IllegalArgumentException( "Map keys must be of type string and cannot be nullable for conversion to Avro"); } - return buildTypeSchema(builder.values(), valueField, namespace); + return buildTypeSchema(builder.values(), valueField, namespace, dictionaries); } private static T buildBaseTypeSchema( - SchemaBuilder.BaseTypeBuilder builder, Field field, String namespace) { + SchemaBuilder.BaseTypeBuilder builder, + Field field, + String namespace, + DictionaryProvider dictionaries) { ArrowType.ArrowTypeID typeID = field.getType().getTypeID(); @@ -269,6 +315,33 @@ private static T buildBaseTypeSchema( return builder.booleanType(); case Int: + if (field.getDictionary() != null) { + if (dictionaries == null) { + throw new IllegalArgumentException( + "Field references a dictionary but no dictionaries were provided: " + + field.getName()); + } + Dictionary dictionary = dictionaries.lookup(field.getDictionary().getId()); + if (dictionary == null) { + throw new IllegalArgumentException( + "Field references a dictionary that does not exist: " + + field.getName() + + ", dictionary ID = " + + field.getDictionary().getId()); + } + if (dictionaryIsValidEnum(dictionary)) { + String[] symbols = dictionarySymbols(dictionary); + return builder.enumeration(field.getName()).symbols(symbols); + } else { + Field decodedField = + new Field( + field.getName(), + dictionary.getVector().getField().getFieldType(), + dictionary.getVector().getField().getChildren()); + return buildBaseTypeSchema(builder, decodedField, namespace, dictionaries); + } + } + ArrowType.Int intType = (ArrowType.Int) field.getType(); if (intType.getBitWidth() > 32 || (intType.getBitWidth() == 32 && !intType.getIsSigned())) { return builder.longType(); @@ -328,7 +401,7 @@ private static T buildBaseTypeSchema( String childNamespace = namespace == null ? field.getName() : namespace + "." + field.getName(); return buildRecordSchema( - builder.record(field.getName()), field.getChildren(), childNamespace); + builder.record(field.getName()), field.getChildren(), childNamespace, dictionaries); case List: case FixedSizeList: @@ -339,13 +412,13 @@ private static T buildBaseTypeSchema( new Field("item", itemField.getFieldType(), itemField.getChildren()); Field safeListField = new Field(field.getName(), field.getFieldType(), List.of(safeItemField)); - return buildArraySchema(builder.array(), safeListField, namespace); + return buildArraySchema(builder.array(), safeListField, namespace, dictionaries); } else { - return buildArraySchema(builder.array(), field, namespace); + return buildArraySchema(builder.array(), field, namespace, dictionaries); } case Map: - return buildMapSchema(builder.map(), field, namespace); + return buildMapSchema(builder.map(), field, namespace, dictionaries); default: throw new IllegalArgumentException( @@ -354,9 +427,12 @@ private static T buildBaseTypeSchema( } private static T addTypesToUnion( - SchemaBuilder.UnionAccumulator accumulator, List unionFields, String namespace) { + SchemaBuilder.UnionAccumulator accumulator, + List unionFields, + String namespace, + DictionaryProvider dictionaries) { for (var field : unionFields) { - accumulator = buildBaseTypeSchema(accumulator.and(), field, namespace); + accumulator = buildBaseTypeSchema(accumulator.and(), field, namespace, dictionaries); } return accumulator.endUnion(); } @@ -373,30 +449,88 @@ private static LogicalType timestampLogicalType(ArrowType.Timestamp timestampTyp } } + private static boolean dictionaryIsValidEnum(Dictionary dictionary) { + + if (dictionary.getVectorType().getTypeID() != ArrowType.ArrowTypeID.Utf8) { + return false; + } + + VarCharVector vector = (VarCharVector) dictionary.getVector(); + Set symbols = new HashSet<>(); + + for (int i = 0; i < vector.getValueCount(); i++) { + if (vector.isNull(i)) { + return false; + } + Text text = vector.getObject(i); + if (text == null) { + return false; + } + String symbol = text.toString(); + if (!ENUM_REGEX.matcher(symbol).matches()) { + return false; + } + if (symbols.contains(symbol)) { + return false; + } + symbols.add(symbol); + } + + return true; + } + + private static String[] dictionarySymbols(Dictionary dictionary) { + + VarCharVector vector = (VarCharVector) dictionary.getVector(); + String[] symbols = new String[vector.getValueCount()]; + + for (int i = 0; i < vector.getValueCount(); i++) { + Text text = vector.getObject(i); + // This should never happen if dictionaryIsValidEnum() succeeded + if (text == null) { + throw new IllegalArgumentException("Illegal null value in enum"); + } + symbols[i] = text.toString(); + } + + return symbols; + } + + private static final Pattern ENUM_REGEX = Pattern.compile("^[A-Za-z_][A-Za-z0-9_]*$"); + /** * Create a composite Avro producer for a set of field vectors (typically the root set of a VSR). * * @param vectors The vectors that will be used to produce Avro data * @return The resulting composite Avro producer */ - public static CompositeAvroProducer createCompositeProducer(List vectors) { + public static CompositeAvroProducer createCompositeProducer( + List vectors, DictionaryProvider dictionaries) { List> producers = new ArrayList<>(vectors.size()); for (FieldVector vector : vectors) { - BaseAvroProducer producer = createProducer(vector); + BaseAvroProducer producer = createProducer(vector, dictionaries); producers.add(producer); } return new CompositeAvroProducer(producers); } - private static BaseAvroProducer createProducer(FieldVector vector) { + /** Overload provided for convenience, sets dictionaries = null. */ + public static CompositeAvroProducer createCompositeProducer(List vectors) { + + return createCompositeProducer(vectors, null); + } + + private static BaseAvroProducer createProducer( + FieldVector vector, DictionaryProvider dictionaries) { boolean nullable = vector.getField().isNullable(); - return createProducer(vector, nullable); + return createProducer(vector, nullable, dictionaries); } - private static BaseAvroProducer createProducer(FieldVector vector, boolean nullable) { + private static BaseAvroProducer createProducer( + FieldVector vector, boolean nullable, DictionaryProvider dictionaries) { Preconditions.checkNotNull(vector, "Arrow vector object can't be null"); @@ -405,10 +539,34 @@ private static BaseAvroProducer createProducer(FieldVector vector, boolean nu // Avro understands nullable types as a union of type | null // Most nullable fields in a VSR will not be unions, so provide a special wrapper if (nullable && minorType != Types.MinorType.UNION) { - final BaseAvroProducer innerProducer = createProducer(vector, false); + final BaseAvroProducer innerProducer = createProducer(vector, false, dictionaries); return new AvroNullableProducer<>(innerProducer); } + if (vector.getField().getDictionary() != null) { + if (dictionaries == null) { + throw new IllegalArgumentException( + "Field references a dictionary but no dictionaries were provided: " + + vector.getField().getName()); + } + Dictionary dictionary = dictionaries.lookup(vector.getField().getDictionary().getId()); + if (dictionary == null) { + throw new IllegalArgumentException( + "Field references a dictionary that does not exist: " + + vector.getField().getName() + + ", dictionary ID = " + + vector.getField().getDictionary().getId()); + } + // If a field is dictionary-encoded but cannot be represented as an Avro enum, + // then decode it before writing + if (dictionaryIsValidEnum(dictionary)) { + return new AvroEnumProducer((BaseIntVector) vector); + } else { + BaseAvroProducer dictProducer = createProducer(dictionary.getVector(), false, null); + return new DictionaryDecodingProducer<>((BaseIntVector) vector, dictProducer); + } + } + switch (minorType) { case NULL: return new AvroNullProducer((NullVector) vector); @@ -486,21 +644,23 @@ private static BaseAvroProducer createProducer(FieldVector vector, boolean nu Producer[] childProducers = new Producer[childVectors.size()]; for (int i = 0; i < childVectors.size(); i++) { FieldVector childVector = childVectors.get(i); - childProducers[i] = createProducer(childVector, childVector.getField().isNullable()); + childProducers[i] = + createProducer(childVector, childVector.getField().isNullable(), dictionaries); } return new AvroStructProducer(structVector, childProducers); case LIST: ListVector listVector = (ListVector) vector; FieldVector itemVector = listVector.getDataVector(); - Producer itemProducer = createProducer(itemVector, itemVector.getField().isNullable()); + Producer itemProducer = + createProducer(itemVector, itemVector.getField().isNullable(), dictionaries); return new AvroListProducer(listVector, itemProducer); case FIXED_SIZE_LIST: FixedSizeListVector fixedListVector = (FixedSizeListVector) vector; FieldVector fixedItemVector = fixedListVector.getDataVector(); Producer fixedItemProducer = - createProducer(fixedItemVector, fixedItemVector.getField().isNullable()); + createProducer(fixedItemVector, fixedItemVector.getField().isNullable(), dictionaries); return new AvroFixedSizeListProducer(fixedListVector, fixedItemProducer); case MAP: @@ -514,7 +674,7 @@ private static BaseAvroProducer createProducer(FieldVector vector, boolean nu FieldVector valueVector = entryVector.getChildrenFromFields().get(1); Producer keyProducer = new AvroStringProducer(keyVector); Producer valueProducer = - createProducer(valueVector, valueVector.getField().isNullable()); + createProducer(valueVector, valueVector.getField().isNullable(), dictionaries); Producer entryProducer = new AvroStructProducer(entryVector, new Producer[] {keyProducer, valueProducer}); return new AvroMapProducer(mapVector, entryProducer); diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroEnumProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroEnumProducer.java index 068566493e..eebfb7d241 100644 --- a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroEnumProducer.java +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/AvroEnumProducer.java @@ -17,22 +17,22 @@ package org.apache.arrow.adapter.avro.producers; import java.io.IOException; -import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.BaseIntVector; import org.apache.avro.io.Encoder; /** - * Producer that produces enum values from a dictionary-encoded {@link IntVector}, writes data to an - * Avro encoder. + * Producer that produces enum values from a dictionary-encoded {@link BaseIntVector}, writes data + * to an Avro encoder. */ -public class AvroEnumProducer extends BaseAvroProducer { +public class AvroEnumProducer extends BaseAvroProducer { /** Instantiate an AvroEnumProducer. */ - public AvroEnumProducer(IntVector vector) { + public AvroEnumProducer(BaseIntVector vector) { super(vector); } @Override public void produce(Encoder encoder) throws IOException { - encoder.writeEnum(vector.get(currentIndex++)); + encoder.writeEnum((int) vector.getValueAsLong(currentIndex++)); } } diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/DictionaryDecodingProducer.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/DictionaryDecodingProducer.java new file mode 100644 index 0000000000..afeba08511 --- /dev/null +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/producers/DictionaryDecodingProducer.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.avro.producers; + +import java.io.IOException; +import org.apache.arrow.vector.BaseIntVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.avro.io.Encoder; + +/** + * Producer that decodes values from a dictionary-encoded {@link FieldVector}, writes the resulting + * values to an Avro encoder. + * + * @param Type of the underlying dictionary vector + */ +public class DictionaryDecodingProducer + extends BaseAvroProducer { + + private final Producer dictProducer; + + /** Instantiate a DictionaryDecodingProducer. */ + public DictionaryDecodingProducer(BaseIntVector indexVector, Producer dictProducer) { + super(indexVector); + this.dictProducer = dictProducer; + } + + @Override + public void produce(Encoder encoder) throws IOException { + int dicIndex = (int) vector.getValueAsLong(currentIndex++); + dictProducer.setPosition(dicIndex); + dictProducer.produce(encoder); + } +} diff --git a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroDataTest.java b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroDataTest.java index 2d70b45021..6d66ee9d45 100644 --- a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroDataTest.java +++ b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroDataTest.java @@ -76,10 +76,14 @@ import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.complex.writer.BaseWriter; import org.apache.arrow.vector.complex.writer.FieldWriter; +import org.apache.arrow.vector.dictionary.Dictionary; +import org.apache.arrow.vector.dictionary.DictionaryEncoder; +import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.types.DateUnit; import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.DictionaryEncoding; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.JsonStringArrayList; @@ -2817,4 +2821,81 @@ record = datumReader.read(record, decoder); } } } + + @Test + public void testWriteDictEnumEncoded() throws Exception { + + BufferAllocator allocator = new RootAllocator(); + + // Create a dictionary + FieldType dictionaryField = new FieldType(false, new ArrowType.Utf8(), null); + VarCharVector dictionaryVector = + new VarCharVector(new Field("dictionary", dictionaryField, null), allocator); + + dictionaryVector.allocateNew(3); + dictionaryVector.set(0, "apple".getBytes()); + dictionaryVector.set(1, "banana".getBytes()); + dictionaryVector.set(2, "cherry".getBytes()); + dictionaryVector.setValueCount(3); + + Dictionary dictionary = + new Dictionary(dictionaryVector, new DictionaryEncoding(1L, false, null)); + DictionaryProvider dictionaries = new DictionaryProvider.MapDictionaryProvider(dictionary); + + // Field definition + FieldType stringField = new FieldType(false, new ArrowType.Utf8(), null); + VarCharVector stringVector = + new VarCharVector(new Field("enumField", stringField, null), allocator); + stringVector.allocateNew(10); + stringVector.setSafe(0, "apple".getBytes()); + stringVector.setSafe(1, "banana".getBytes()); + stringVector.setSafe(2, "cherry".getBytes()); + stringVector.setSafe(3, "cherry".getBytes()); + stringVector.setSafe(4, "apple".getBytes()); + stringVector.setSafe(5, "banana".getBytes()); + stringVector.setSafe(6, "apple".getBytes()); + stringVector.setSafe(7, "cherry".getBytes()); + stringVector.setSafe(8, "banana".getBytes()); + stringVector.setSafe(9, "apple".getBytes()); + stringVector.setValueCount(10); + + IntVector encodedVector = (IntVector) DictionaryEncoder.encode(stringVector, dictionary); + + // Set up VSR + List vectors = Arrays.asList(encodedVector); + int rowCount = 10; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + File dataFile = new File(TMP, "testWriteEnumEncoded.avro"); + + // Write an AVRO block using the producer classes + try (FileOutputStream fos = new FileOutputStream(dataFile)) { + BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); + CompositeAvroProducer producer = + ArrowToAvroUtils.createCompositeProducer(vectors, dictionaries); + for (int row = 0; row < rowCount; row++) { + producer.produce(encoder); + } + encoder.flush(); + } + + // Set up reading the AVRO block as a GenericRecord + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields(), dictionaries); + GenericDatumReader datumReader = new GenericDatumReader<>(schema); + + try (InputStream inputStream = new FileInputStream(dataFile)) { + + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); + GenericRecord record = null; + + // Read and check values + for (int row = 0; row < rowCount; row++) { + record = datumReader.read(record, decoder); + // Values read from Avro should be the decoded enum values + assertEquals(stringVector.getObject(row).toString(), record.get("enumField").toString()); + } + } + } + } } diff --git a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroSchemaTest.java b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroSchemaTest.java index d3e12e763a..d5e0357a8c 100644 --- a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroSchemaTest.java +++ b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/ArrowToAvroSchemaTest.java @@ -20,11 +20,18 @@ import java.util.Arrays; import java.util.List; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.dictionary.Dictionary; +import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.types.DateUnit; import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.UnionMode; import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.DictionaryEncoding; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.avro.LogicalTypes; @@ -1389,4 +1396,126 @@ public void testConvertUnionTypes() { Schema.Type.STRING, schema.getField("nullableDenseUnionField").schema().getTypes().get(3).getType()); } + + @Test + public void testWriteDictEnumEncoded() { + + BufferAllocator allocator = new RootAllocator(); + + // Create a dictionary + FieldType dictionaryField = new FieldType(false, new ArrowType.Utf8(), null); + VarCharVector dictionaryVector = + new VarCharVector(new Field("dictionary", dictionaryField, null), allocator); + + dictionaryVector.allocateNew(3); + dictionaryVector.set(0, "apple".getBytes()); + dictionaryVector.set(1, "banana".getBytes()); + dictionaryVector.set(2, "cherry".getBytes()); + dictionaryVector.setValueCount(3); + + Dictionary dictionary = + new Dictionary( + dictionaryVector, new DictionaryEncoding(0L, false, new ArrowType.Int(8, true))); + DictionaryProvider dictionaries = new DictionaryProvider.MapDictionaryProvider(dictionary); + + List fields = + Arrays.asList( + new Field( + "enumField", + new FieldType(false, new ArrowType.Int(8, true), dictionary.getEncoding(), null), + null)); + + Schema schema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord", null, dictionaries); + + assertEquals(Schema.Type.RECORD, schema.getType()); + assertEquals(1, schema.getFields().size()); + + Schema.Field enumField = schema.getField("enumField"); + + assertEquals(Schema.Type.ENUM, enumField.schema().getType()); + assertEquals(3, enumField.schema().getEnumSymbols().size()); + assertEquals("apple", enumField.schema().getEnumSymbols().get(0)); + assertEquals("banana", enumField.schema().getEnumSymbols().get(1)); + assertEquals("cherry", enumField.schema().getEnumSymbols().get(2)); + } + + @Test + public void testWriteDictEnumInvalid() { + + BufferAllocator allocator = new RootAllocator(); + + // Create a dictionary + FieldType dictionaryField = new FieldType(false, new ArrowType.Utf8(), null); + VarCharVector dictionaryVector = + new VarCharVector(new Field("dictionary", dictionaryField, null), allocator); + + dictionaryVector.allocateNew(3); + dictionaryVector.set(0, "passion fruit".getBytes()); + dictionaryVector.set(1, "banana".getBytes()); + dictionaryVector.set(2, "cherry".getBytes()); + dictionaryVector.setValueCount(3); + + Dictionary dictionary = + new Dictionary( + dictionaryVector, new DictionaryEncoding(0L, false, new ArrowType.Int(8, true))); + DictionaryProvider dictionaries = new DictionaryProvider.MapDictionaryProvider(dictionary); + + List fields = + Arrays.asList( + new Field( + "enumField", + new FieldType(false, new ArrowType.Int(8, true), dictionary.getEncoding(), null), + null)); + + // Dictionary field contains values that are not valid enums + // Should be decoded and output as a string field + + Schema schema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord", null, dictionaries); + + assertEquals(Schema.Type.RECORD, schema.getType()); + assertEquals(1, schema.getFields().size()); + + Schema.Field enumField = schema.getField("enumField"); + assertEquals(Schema.Type.STRING, enumField.schema().getType()); + } + + @Test + public void testWriteDictEnumInvalid2() { + + BufferAllocator allocator = new RootAllocator(); + + // Create a dictionary + FieldType dictionaryField = new FieldType(false, new ArrowType.Int(64, true), null); + BigIntVector dictionaryVector = + new BigIntVector(new Field("dictionary", dictionaryField, null), allocator); + + dictionaryVector.allocateNew(3); + dictionaryVector.set(0, 123L); + dictionaryVector.set(1, 456L); + dictionaryVector.set(2, 789L); + dictionaryVector.setValueCount(3); + + Dictionary dictionary = + new Dictionary( + dictionaryVector, new DictionaryEncoding(0L, false, new ArrowType.Int(8, true))); + DictionaryProvider dictionaries = new DictionaryProvider.MapDictionaryProvider(dictionary); + + List fields = + Arrays.asList( + new Field( + "enumField", + new FieldType(false, new ArrowType.Int(8, true), dictionary.getEncoding(), null), + null)); + + // Dictionary field encodes LONG values rather than STRING + // Should be doecded and output as a LONG field + + Schema schema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord", null, dictionaries); + + assertEquals(Schema.Type.RECORD, schema.getType()); + assertEquals(1, schema.getFields().size()); + + Schema.Field enumField = schema.getField("enumField"); + assertEquals(Schema.Type.LONG, enumField.schema().getType()); + } } diff --git a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripDataTest.java b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripDataTest.java index 85e6a960b0..ceaf59aa72 100644 --- a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripDataTest.java +++ b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripDataTest.java @@ -52,6 +52,7 @@ import org.apache.arrow.vector.TimeStampMilliVector; import org.apache.arrow.vector.TimeStampNanoTZVector; import org.apache.arrow.vector.TimeStampNanoVector; +import org.apache.arrow.vector.TinyIntVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; @@ -60,10 +61,14 @@ import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.complex.writer.BaseWriter; import org.apache.arrow.vector.complex.writer.FieldWriter; +import org.apache.arrow.vector.dictionary.Dictionary; +import org.apache.arrow.vector.dictionary.DictionaryEncoder; +import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.types.DateUnit; import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.DictionaryEncoding; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.avro.Schema; @@ -78,16 +83,21 @@ public class RoundTripDataTest { @TempDir public static File TMP; - private static AvroToArrowConfig basicConfig(BufferAllocator allocator) { - return new AvroToArrowConfig(allocator, 1000, null, Collections.emptySet(), false); + private static AvroToArrowConfig basicConfig( + BufferAllocator allocator, DictionaryProvider.MapDictionaryProvider dictionaries) { + return new AvroToArrowConfig(allocator, 1000, dictionaries, Collections.emptySet(), false); } private static VectorSchemaRoot readDataFile( - Schema schema, File dataFile, BufferAllocator allocator) throws Exception { + Schema schema, + File dataFile, + BufferAllocator allocator, + DictionaryProvider.MapDictionaryProvider dictionaries) + throws Exception { try (FileInputStream fis = new FileInputStream(dataFile)) { BinaryDecoder decoder = new DecoderFactory().directBinaryDecoder(fis, null); - return AvroToArrow.avroToArrow(schema, decoder, basicConfig(allocator)); + return AvroToArrow.avroToArrow(schema, decoder, basicConfig(allocator, dictionaries)); } } @@ -95,11 +105,22 @@ private static void roundTripTest( VectorSchemaRoot root, BufferAllocator allocator, File dataFile, int rowCount) throws Exception { + roundTripTest(root, allocator, dataFile, rowCount, null); + } + + private static void roundTripTest( + VectorSchemaRoot root, + BufferAllocator allocator, + File dataFile, + int rowCount, + DictionaryProvider dictionaries) + throws Exception { + // Write an AVRO block using the producer classes try (FileOutputStream fos = new FileOutputStream(dataFile)) { BinaryEncoder encoder = new EncoderFactory().directBinaryEncoder(fos, null); CompositeAvroProducer producer = - ArrowToAvroUtils.createCompositeProducer(root.getFieldVectors()); + ArrowToAvroUtils.createCompositeProducer(root.getFieldVectors(), dictionaries); for (int row = 0; row < rowCount; row++) { producer.produce(encoder); } @@ -107,10 +128,14 @@ private static void roundTripTest( } // Generate AVRO schema - Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); + Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields(), dictionaries); + + DictionaryProvider.MapDictionaryProvider roundTripDictionaries = + new DictionaryProvider.MapDictionaryProvider(); // Read back in and compare - try (VectorSchemaRoot roundTrip = readDataFile(schema, dataFile, allocator)) { + try (VectorSchemaRoot roundTrip = + readDataFile(schema, dataFile, allocator, roundTripDictionaries)) { assertEquals(root.getSchema(), roundTrip.getSchema()); assertEquals(rowCount, roundTrip.getRowCount()); @@ -119,6 +144,21 @@ private static void roundTripTest( for (int row = 0; row < rowCount; row++) { assertEquals(root.getVector(0).getObject(row), roundTrip.getVector(0).getObject(row)); } + + if (dictionaries != null) { + for (long id : dictionaries.getDictionaryIds()) { + Dictionary originalDictionary = dictionaries.lookup(id); + Dictionary roundTripDictionary = roundTripDictionaries.lookup(id); + assertEquals( + originalDictionary.getVector().getValueCount(), + roundTripDictionary.getVector().getValueCount()); + for (int j = 0; j < originalDictionary.getVector().getValueCount(); j++) { + assertEquals( + originalDictionary.getVector().getObject(j), + roundTripDictionary.getVector().getObject(j)); + } + } + } } } @@ -141,7 +181,7 @@ private static void roundTripByteArrayTest( Schema schema = ArrowToAvroUtils.createAvroSchema(root.getSchema().getFields()); // Read back in and compare - try (VectorSchemaRoot roundTrip = readDataFile(schema, dataFile, allocator)) { + try (VectorSchemaRoot roundTrip = readDataFile(schema, dataFile, allocator, null)) { assertEquals(root.getSchema(), roundTrip.getSchema()); assertEquals(rowCount, roundTrip.getRowCount()); @@ -1603,4 +1643,58 @@ public void testRoundTripNullableStructs() throws Exception { roundTripTest(root, allocator, dataFile, rowCount); } } + + @Test + public void testRoundTripEnum() throws Exception { + + BufferAllocator allocator = new RootAllocator(); + + // Create a dictionary + FieldType dictionaryField = new FieldType(false, new ArrowType.Utf8(), null); + VarCharVector dictionaryVector = + new VarCharVector(new Field("dictionary", dictionaryField, null), allocator); + + dictionaryVector.allocateNew(3); + dictionaryVector.set(0, "apple".getBytes()); + dictionaryVector.set(1, "banana".getBytes()); + dictionaryVector.set(2, "cherry".getBytes()); + dictionaryVector.setValueCount(3); + + // For simplicity, ensure the index type matches what will be decoded during Avro enum decoding + Dictionary dictionary = + new Dictionary( + dictionaryVector, new DictionaryEncoding(0L, false, new ArrowType.Int(8, true))); + DictionaryProvider dictionaries = new DictionaryProvider.MapDictionaryProvider(dictionary); + + // Field definition + FieldType stringField = new FieldType(false, new ArrowType.Utf8(), null); + VarCharVector stringVector = + new VarCharVector(new Field("enumField", stringField, null), allocator); + stringVector.allocateNew(10); + stringVector.setSafe(0, "apple".getBytes()); + stringVector.setSafe(1, "banana".getBytes()); + stringVector.setSafe(2, "cherry".getBytes()); + stringVector.setSafe(3, "cherry".getBytes()); + stringVector.setSafe(4, "apple".getBytes()); + stringVector.setSafe(5, "banana".getBytes()); + stringVector.setSafe(6, "apple".getBytes()); + stringVector.setSafe(7, "cherry".getBytes()); + stringVector.setSafe(8, "banana".getBytes()); + stringVector.setSafe(9, "apple".getBytes()); + stringVector.setValueCount(10); + + TinyIntVector encodedVector = + (TinyIntVector) DictionaryEncoder.encode(stringVector, dictionary); + + // Set up VSR + List vectors = Arrays.asList(encodedVector); + int rowCount = 10; + + try (VectorSchemaRoot root = new VectorSchemaRoot(vectors)) { + + File dataFile = new File(TMP, "testRoundTripEnums.avro"); + + roundTripTest(root, allocator, dataFile, rowCount, dictionaries); + } + } } diff --git a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripSchemaTest.java b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripSchemaTest.java index 864e2c8b59..37c0b4d9fe 100644 --- a/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripSchemaTest.java +++ b/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/RoundTripSchemaTest.java @@ -21,27 +21,50 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.dictionary.Dictionary; +import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.types.DateUnit; import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.DictionaryEncoding; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.avro.Schema; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class RoundTripSchemaTest { private void doRoundTripTest(List fields) { + doRoundTripTest(fields, null); + } - AvroToArrowConfig config = new AvroToArrowConfig(null, 1, null, Collections.emptySet(), false); + private void doRoundTripTest(List fields, DictionaryProvider dictionaries) { - Schema avroSchema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord"); + DictionaryProvider.MapDictionaryProvider decodeDictionaries = + new DictionaryProvider.MapDictionaryProvider(); + AvroToArrowConfig decodeConfig = + new AvroToArrowConfig(null, 1, decodeDictionaries, Collections.emptySet(), false); + + Schema avroSchema = ArrowToAvroUtils.createAvroSchema(fields, "TestRecord", null, dictionaries); org.apache.arrow.vector.types.pojo.Schema arrowSchema = - AvroToArrowUtils.createArrowSchema(avroSchema, config); + AvroToArrowUtils.createArrowSchema(avroSchema, decodeConfig); // Compare string representations - equality not defined for logical types assertEquals(fields, arrowSchema.getFields()); + + for (int i = 0; i < fields.size(); i++) { + Field field = fields.get(i); + Field rtField = arrowSchema.getFields().get(i); + if (field.getDictionary() != null) { + // Dictionary content is not decoded until the data is consumed + Assertions.assertNotNull(rtField.getDictionary()); + } + } } // Schema round trip for primitive types, nullable and non-nullable @@ -440,4 +463,38 @@ public void testRoundTripStructType() { doRoundTripTest(fields); } + + @Test + public void testRoundTripEnumType() { + + BufferAllocator allocator = new RootAllocator(); + + FieldType dictionaryField = new FieldType(false, new ArrowType.Utf8(), null); + VarCharVector dictionaryVector = + new VarCharVector(new Field("dictionary", dictionaryField, null), allocator); + + dictionaryVector.allocateNew(3); + dictionaryVector.set(0, "apple".getBytes()); + dictionaryVector.set(1, "banana".getBytes()); + dictionaryVector.set(2, "cherry".getBytes()); + dictionaryVector.setValueCount(3); + + // For simplicity, ensure the index type matches what will be decoded during Avro enum decoding + Dictionary dictionary = + new Dictionary( + dictionaryVector, new DictionaryEncoding(0L, false, new ArrowType.Int(8, true))); + DictionaryProvider dictionaries = new DictionaryProvider.MapDictionaryProvider(dictionary); + + List fields = + Arrays.asList( + new Field( + "enumField", + new FieldType( + true, + new ArrowType.Int(8, true), + new DictionaryEncoding(0L, false, new ArrowType.Int(8, true))), + null)); + + doRoundTripTest(fields, dictionaries); + } } From d0e9d147a924f0cada7bfb477bd3e3d23b7f1216 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 08:05:58 +0200 Subject: [PATCH 049/271] MINOR: Bump com.gradle:common-custom-user-data-maven-extension from 2.0.1 to 2.0.3 (#785) Bumps [com.gradle:common-custom-user-data-maven-extension](https://github.com/gradle/common-custom-user-data-maven-extension) from 2.0.1 to 2.0.3.

Release notes

Sourced from com.gradle:common-custom-user-data-maven-extension's releases.

2.0.3

  • [NEW] Added tagging VS Code builds
  • [FIX] Redact user info from git urls when the user info contains a URL-encoded character

2.0.2

  • [FIX] Update to Groovy 4 to handle Java 24
Commits
  • c744e1a [maven-release-plugin] prepare release v2.0.3
  • 1aff439 Prepare changes.md for release
  • 1a81cf2 Redact URL-encoded characters in userinfo of URLs (#289)
  • e480f07 Combined PRs (#288)
  • feda61d Publish to Maven Central with central-publishing-maven-plugin (#285)
  • 9a96149 Bump org.junit.jupiter:junit-jupiter from 5.12.2 to 5.13.0 (#283)
  • dbc804b Merge pull request #282 from gradle/dependabot/maven/com.gradle-develocity-ma...
  • 328a79c Bump com.gradle:develocity-maven-extension from 2.0 to 2.0.1
  • 23680e3 Merge pull request #281 from gradle/dependabot/maven/org.apache.groovy-groovy...
  • 953c7aa Bump org.apache.groovy:groovy from 4.0.26 to 4.0.27
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.gradle:common-custom-user-data-maven-extension&package-manager=maven&previous-version=2.0.1&new-version=2.0.3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .mvn/extensions.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index 6fd036232f..943140738d 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -28,6 +28,6 @@ com.gradle common-custom-user-data-maven-extension - 2.0.1 + 2.0.3 From 7618274f9d537aa2e626f97ae5c1fa3cc0df8c99 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 08:07:29 +0200 Subject: [PATCH 050/271] MINOR: Bump io.grpc:grpc-bom from 1.71.0 to 1.73.0 (#781) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [io.grpc:grpc-bom](https://github.com/grpc/grpc-java) from 1.71.0 to 1.73.0.
Release notes

Sourced from io.grpc:grpc-bom's releases.

V1.72.0

API Changes

  • util: Remove deprecated method GracefulSwitchLb.switchTo() (f207be39a). It is rarely used outside of gRPC itself. The configuration is passed as lb policy configuration instead
  • xds: Add support for custom per-target credentials on the transport (#11951) (1958e4237)
  • xds: Explicitly set request hash key for the ring hash LB policy (892144dca)

Bug Fixes

  • core: Apply ManagedChannelImpl's updateBalancingState() immediately (ca4819ac6)
  • xds: Fix cluster selection races when updating config selector (d82613a74)
  • otel: Fix span names as per the A72 gRFC changes (#11974) (94f8e9369)
  • xds: ClusterResolverLoadBalancer handle update for both resolved addresses and errors via ResolutionResult (#11997) (868178651)

Improvements

  • netty: Avoid allocating an exception on transport shutdown. This reduces allocation rate for connection-heavy workloads/load testing (a57c14a51)
  • servlet: Set an explicit description for CANCELLED status (#11927) (fca1d3cf4)
  • xds: gRFC A74 xDS Config Tears implementation in the XdsNameResolver (e80c19745). While there is more remaining, users may already see reduced latency when resources are replaced. For example, if changing a route from one backend service to another, RPCs may see less latency during the transition
  • core: Log any exception during channel panic because of exception (3961a923a). This prevents the exception from propagating up the stack on an arbitrary thread. Such exceptions are rarely interesting. Instead, the exception that caused the channel panic is the important one, and RPCs will still fail with its details
  • util: Graceful switch to new LB when leaving CONNECTING (2e260a4bb). Previously when using xDS and the configuration changes the LB policy, the old LB policy is used until the new one is READY. Now the old LB policy is used until the new policy becomes READY, TRANSIENT_FAILURE, or IDLE
  • core: Use java.time.Time.getNano directly in InstantTimeProvider. Previously reflection was used which would confuse R8 full mode (#11977) (7507a9ec0)
  • core: Avoid cancellation exceptions when notifying watchers that already have their connections cancelled (#11934) (350f90e1a)
  • rls: allow maxAge in RLS config to exceed 5 minutes if staleAge is set. Previously, the limit was 5 minutes, which isn't enough for some gRPC clients (#11931) (c340f4a2f)
  • xds: avoid unnecessary dns lookup for CIDR addresses (#11932) (602aece08)
  • netty: Swap to UniformStreamByteDistributor (#11954) (2f52a0036). gRPC will no longer observe the HTTP/2 priorities, which were not used directly by gRPC and deprecated in RFC 9113
  • core: Avoid Set.removeAll() when passing a possibly-large List (#11994) (666136b4b)
  • stub: trailersFromThrowable() metadata should be copied (#11979) (a6e1c1f09)

New Features

  • xds: xDS-based HTTP CONNECT configuration (#11861) (12197065f)
  • netty: Per-rpc authority verification against peer cert subject names. Overriding transport authority at rpc time is only allowed when using TlsChannelCredentials. The per-rpc authority verification feature is guarded by the environment variable GRPC_ENABLE_PER_RPC_AUTHORITY_CHECK in this release. When this is false or not set, the rpc will not fail when the authority verification fails but a warning will be logged. In a subsequent release the usage of this environment variable will be removed and RPCs will start failing if the authority doesn't match the peer certificate names. The environment variable is temporary; if you are depending on the existing insecure behavior, please file an issue (#11724) (cdab410b8)

Thanks to

@​panchenko @​emmanuel-ferdman @​JoeCqupt

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=io.grpc:grpc-bom&package-manager=maven&previous-version=1.71.0&new-version=1.73.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d40f428b22..49e0c47c60 100644 --- a/pom.xml +++ b/pom.xml @@ -97,7 +97,7 @@ under the License. 2.0.17 33.4.8-jre 4.1.119.Final - 1.71.0 + 1.73.0 4.30.2 2.18.3 3.4.1 From e6da71e871a3678a737a88dbe79491b4111496b4 Mon Sep 17 00:00:00 2001 From: Ivan Chesnov Date: Tue, 5 Aug 2025 05:02:48 +0300 Subject: [PATCH 051/271] GH-725: Added ExtensionReader (#726) ## What's Changed ExtensionReader was added to support reading extension types from a complex vector. It contains **read(ExtensionHolder)** method for reading to the holder. And **readObject** - for reading the value explicitly. Closes #725. --- .../templates/AbstractFieldReader.java | 17 +++++ .../AbstractPromotableFieldWriter.java | 4 +- .../main/codegen/templates/BaseReader.java | 2 +- .../main/codegen/templates/NullReader.java | 4 ++ .../codegen/templates/PromotableWriter.java | 4 ++ .../codegen/templates/UnionListWriter.java | 11 ++- .../complex/reader/ExtensionReader.java | 44 ++++++++++++ .../apache/arrow/vector/TestListVector.java | 72 +++++++++++++++++++ .../org/apache/arrow/vector/UuidVector.java | 13 ++++ .../complex/impl/TestPromotableWriter.java | 25 +++++++ .../vector/complex/impl/UuidReaderImpl.java | 64 +++++++++++++++++ .../complex/writer/TestComplexWriter.java | 41 +++++++++++ .../complex/writer/TestSimpleWriter.java | 20 ++++++ 13 files changed, 315 insertions(+), 6 deletions(-) create mode 100644 vector/src/main/java/org/apache/arrow/vector/complex/reader/ExtensionReader.java create mode 100644 vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java diff --git a/vector/src/main/codegen/templates/AbstractFieldReader.java b/vector/src/main/codegen/templates/AbstractFieldReader.java index 25b071fab7..7e84323b64 100644 --- a/vector/src/main/codegen/templates/AbstractFieldReader.java +++ b/vector/src/main/codegen/templates/AbstractFieldReader.java @@ -108,6 +108,23 @@ public void copyAsField(String name, ${name}Writer writer) { } + + public void read(ExtensionHolder holder) { + fail("Extension"); + } + + public void read(int arrayIndex, ExtensionHolder holder) { + fail("RepeatedExtension"); + } + + public void copyAsValue(AbstractExtensionTypeWriter writer) { + fail("CopyAsValueExtension"); + } + + public void copyAsField(String name, AbstractExtensionTypeWriter writer) { + fail("CopyAsFieldExtension"); + } + public FieldReader reader(String name) { fail("reader(String name)"); return null; diff --git a/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java b/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java index 951edd5eee..2e7792fcfe 100644 --- a/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java +++ b/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java @@ -295,7 +295,7 @@ public MapWriter map(boolean keysSorted) { @Override public ExtensionWriter extension(ArrowType arrowType) { - return getWriter(MinorType.EXTENSIONTYPE).extension(arrowType); + return getWriter(MinorType.LIST).extension(arrowType); } @Override @@ -325,7 +325,7 @@ public MapWriter map(String name, boolean keysSorted) { @Override public ExtensionWriter extension(String name, ArrowType arrowType) { - return getWriter(MinorType.EXTENSIONTYPE).extension(name, arrowType); + return getWriter(MinorType.STRUCT).extension(name, arrowType); } <#list vv.types as type><#list type.minor as minor> diff --git a/vector/src/main/codegen/templates/BaseReader.java b/vector/src/main/codegen/templates/BaseReader.java index e75e8a2974..c52345af21 100644 --- a/vector/src/main/codegen/templates/BaseReader.java +++ b/vector/src/main/codegen/templates/BaseReader.java @@ -73,7 +73,7 @@ public interface RepeatedMapReader extends MapReader{ public interface ScalarReader extends <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> ${name}Reader, - BaseReader {} + ExtensionReader, BaseReader {} interface ComplexReader{ StructReader rootAsStruct(); diff --git a/vector/src/main/codegen/templates/NullReader.java b/vector/src/main/codegen/templates/NullReader.java index 1d77248e96..88e6ea98ea 100644 --- a/vector/src/main/codegen/templates/NullReader.java +++ b/vector/src/main/codegen/templates/NullReader.java @@ -86,6 +86,10 @@ public void read(int arrayIndex, Nullable${name}Holder holder){ } + public void read(ExtensionHolder holder) { + holder.isSet = 0; + } + public int size(){ return 0; } diff --git a/vector/src/main/codegen/templates/PromotableWriter.java b/vector/src/main/codegen/templates/PromotableWriter.java index 8d7d57bb9d..d22eb00b2c 100644 --- a/vector/src/main/codegen/templates/PromotableWriter.java +++ b/vector/src/main/codegen/templates/PromotableWriter.java @@ -550,6 +550,10 @@ public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory factory) { getWriter(MinorType.EXTENSIONTYPE).addExtensionTypeWriterFactory(factory); } + public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory factory, ArrowType arrowType) { + getWriter(MinorType.EXTENSIONTYPE, arrowType).addExtensionTypeWriterFactory(factory); + } + @Override public void allocate() { getWriter().allocate(); diff --git a/vector/src/main/codegen/templates/UnionListWriter.java b/vector/src/main/codegen/templates/UnionListWriter.java index 9424533f29..94723e6c9d 100644 --- a/vector/src/main/codegen/templates/UnionListWriter.java +++ b/vector/src/main/codegen/templates/UnionListWriter.java @@ -53,6 +53,7 @@ public class Union${listName}Writer extends AbstractFieldWriter { private boolean inStruct = false; private boolean listStarted = false; private String structName; + private ArrowType extensionType; <#if listName == "LargeList" || listName == "LargeListView"> private static final long OFFSET_WIDTH = 8; <#else> @@ -203,8 +204,8 @@ public MapWriter map(String name, boolean keysSorted) { @Override public ExtensionWriter extension(ArrowType arrowType) { - writer.extension(arrowType); - return writer; + this.extensionType = arrowType; + return this; } @Override public ExtensionWriter extension(String name, ArrowType arrowType) { @@ -337,13 +338,17 @@ public void writeNull() { @Override public void writeExtension(Object value) { writer.writeExtension(value); + writer.setPosition(writer.idx() + 1); } + @Override public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory var1) { - writer.addExtensionTypeWriterFactory(var1); + writer.addExtensionTypeWriterFactory(var1, extensionType); } + public void write(ExtensionHolder var1) { writer.write(var1); + writer.setPosition(writer.idx() + 1); } <#list vv.types as type> diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/reader/ExtensionReader.java b/vector/src/main/java/org/apache/arrow/vector/complex/reader/ExtensionReader.java new file mode 100644 index 0000000000..1ba7b27156 --- /dev/null +++ b/vector/src/main/java/org/apache/arrow/vector/complex/reader/ExtensionReader.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.complex.reader; + +import org.apache.arrow.vector.holders.ExtensionHolder; + +/** Interface for reading extension types. Extends the functionality of {@link BaseReader}. */ +public interface ExtensionReader extends BaseReader { + + /** + * Reads to the given extension holder. + * + * @param holder the {@link ExtensionHolder} to read + */ + void read(ExtensionHolder holder); + + /** + * Reads and returns an object representation of the extension type. + * + * @return the object representation of the extension type + */ + Object readObject(); + + /** + * Checks if the current value is set. + * + * @return true if the value is set, false otherwise + */ + boolean isSet(); +} diff --git a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index 5b2043a014..d58b7cc941 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -24,16 +24,22 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.UUID; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.util.AutoCloseables; import org.apache.arrow.vector.complex.BaseRepeatedValueVector; import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.impl.UnionListReader; import org.apache.arrow.vector.complex.impl.UnionListWriter; +import org.apache.arrow.vector.complex.impl.UuidWriterFactory; import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.complex.writer.BaseWriter.ExtensionWriter; +import org.apache.arrow.vector.holder.UuidHolder; import org.apache.arrow.vector.holders.DurationHolder; import org.apache.arrow.vector.holders.FixedSizeBinaryHolder; import org.apache.arrow.vector.holders.TimeStampMilliTZHolder; @@ -42,6 +48,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.UuidType; import org.apache.arrow.vector.util.TransferPair; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -1199,6 +1206,71 @@ public void testGetTransferPairWithField() { } } + @Test + public void testListVectorWithExtensionType() throws Exception { + final FieldType type = FieldType.nullable(new UuidType()); + try (final ListVector inVector = new ListVector("list", allocator, type, null)) { + UnionListWriter writer = inVector.getWriter(); + writer.allocate(); + writer.setPosition(0); + UUID u1 = UUID.randomUUID(); + UUID u2 = UUID.randomUUID(); + writer.startList(); + ExtensionWriter extensionWriter = writer.extension(new UuidType()); + extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); + extensionWriter.writeExtension(u1); + extensionWriter.writeExtension(u2); + writer.endList(); + + writer.setValueCount(1); + + FieldReader reader = inVector.getReader(); + assertTrue(reader.isSet(), "shouldn't be null"); + Object result = inVector.getObject(0); + ArrayList resultSet = (ArrayList) result; + assertEquals(2, resultSet.size()); + assertEquals(u1, resultSet.get(0)); + assertEquals(u2, resultSet.get(1)); + } + } + + @Test + public void testListVectorReaderForExtensionType() throws Exception { + final FieldType type = FieldType.nullable(new UuidType()); + try (final ListVector inVector = new ListVector("list", allocator, type, null)) { + UnionListWriter writer = inVector.getWriter(); + writer.allocate(); + writer.setPosition(0); + UUID u1 = UUID.randomUUID(); + UUID u2 = UUID.randomUUID(); + writer.startList(); + ExtensionWriter extensionWriter = writer.extension(new UuidType()); + extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); + extensionWriter.writeExtension(u1); + extensionWriter.writeExtension(u2); + writer.endList(); + + writer.setValueCount(1); + + UnionListReader reader = inVector.getReader(); + assertTrue(reader.isSet(), "shouldn't be null"); + reader.setPosition(0); + reader.next(); + FieldReader uuidReader = reader.reader(); + UuidHolder holder = new UuidHolder(); + uuidReader.read(holder); + ByteBuffer bb = ByteBuffer.wrap(holder.value); + UUID actualUuid = new UUID(bb.getLong(), bb.getLong()); + assertEquals(u1, actualUuid); + reader.next(); + uuidReader = reader.reader(); + uuidReader.read(holder); + bb = ByteBuffer.wrap(holder.value); + actualUuid = new UUID(bb.getLong(), bb.getLong()); + assertEquals(u2, actualUuid); + } + } + private void writeIntValues(UnionListWriter writer, int[] values) { writer.startList(); for (int v : values) { diff --git a/vector/src/test/java/org/apache/arrow/vector/UuidVector.java b/vector/src/test/java/org/apache/arrow/vector/UuidVector.java index 5c90d45f60..72ba4aa555 100644 --- a/vector/src/test/java/org/apache/arrow/vector/UuidVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/UuidVector.java @@ -20,6 +20,9 @@ import java.util.UUID; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.util.hash.ArrowBufHasher; +import org.apache.arrow.vector.complex.impl.UuidReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holder.UuidHolder; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.UuidType; @@ -79,11 +82,21 @@ public TransferPair makeTransferPair(ValueVector to) { return new TransferImpl((UuidVector) to); } + @Override + protected FieldReader getReaderImpl() { + return new UuidReaderImpl(this); + } + public void setSafe(int index, byte[] value) { getUnderlyingVector().setIndexDefined(index); getUnderlyingVector().setSafe(index, value); } + public void get(int index, UuidHolder holder) { + holder.value = getUnderlyingVector().get(index); + holder.isSet = 1; + } + public class TransferImpl implements TransferPair { UuidVector to; ValueVector targetUnderlyingVector; diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java index 1556852c5a..7b8b1f9ef9 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java @@ -805,4 +805,29 @@ public void testExtensionType() throws Exception { assertEquals(u2, uuidVector.getObject(1)); } } + + @Test + public void testExtensionTypeForList() throws Exception { + try (final ListVector container = ListVector.empty(EMPTY_SCHEMA_PATH, allocator); + final UuidVector v = + (UuidVector) container.addOrGetVector(FieldType.nullable(new UuidType())).getVector(); + final PromotableWriter writer = new PromotableWriter(v, container)) { + UUID u1 = UUID.randomUUID(); + UUID u2 = UUID.randomUUID(); + container.allocateNew(); + container.setValueCount(1); + writer.addExtensionTypeWriterFactory(new UuidWriterFactory()); + + writer.setPosition(0); + writer.writeExtension(u1); + writer.setPosition(1); + writer.writeExtension(u2); + + container.setValueCount(2); + + UuidVector uuidVector = (UuidVector) container.getDataVector(); + assertEquals(u1, uuidVector.getObject(0)); + assertEquals(u2, uuidVector.getObject(1)); + } + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java new file mode 100644 index 0000000000..16dd734de8 --- /dev/null +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.complex.impl; + +import org.apache.arrow.vector.UuidVector; +import org.apache.arrow.vector.holder.UuidHolder; +import org.apache.arrow.vector.holders.ExtensionHolder; +import org.apache.arrow.vector.types.Types.MinorType; +import org.apache.arrow.vector.types.pojo.Field; + +public class UuidReaderImpl extends AbstractFieldReader { + + private final UuidVector vector; + + public UuidReaderImpl(UuidVector vector) { + super(); + this.vector = vector; + } + + @Override + public MinorType getMinorType() { + return vector.getMinorType(); + } + + @Override + public Field getField() { + return vector.getField(); + } + + @Override + public boolean isSet() { + return !vector.isNull(idx()); + } + + @Override + public void read(ExtensionHolder holder) { + vector.get(idx(), (UuidHolder) holder); + } + + @Override + public void read(int arrayIndex, ExtensionHolder holder) { + vector.get(arrayIndex, (UuidHolder) holder); + } + + @Override + public void copyAsValue(AbstractExtensionTypeWriter writer) { + UuidWriterImpl impl = (UuidWriterImpl) writer; + impl.vector.copyFromSafe(idx(), impl.idx(), vector); + } +} diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index 2745386db4..f374eb41e4 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -19,6 +19,7 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -31,6 +32,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.UUID; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; @@ -64,6 +66,7 @@ import org.apache.arrow.vector.complex.impl.UnionMapReader; import org.apache.arrow.vector.complex.impl.UnionReader; import org.apache.arrow.vector.complex.impl.UnionWriter; +import org.apache.arrow.vector.complex.impl.UuidWriterFactory; import org.apache.arrow.vector.complex.reader.BaseReader.StructReader; import org.apache.arrow.vector.complex.reader.BigIntReader; import org.apache.arrow.vector.complex.reader.FieldReader; @@ -71,9 +74,11 @@ import org.apache.arrow.vector.complex.reader.Float8Reader; import org.apache.arrow.vector.complex.reader.IntReader; import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter; +import org.apache.arrow.vector.complex.writer.BaseWriter.ExtensionWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.StructWriter; +import org.apache.arrow.vector.holder.UuidHolder; import org.apache.arrow.vector.holders.DecimalHolder; import org.apache.arrow.vector.holders.DurationHolder; import org.apache.arrow.vector.holders.FixedSizeBinaryHolder; @@ -84,6 +89,7 @@ import org.apache.arrow.vector.holders.NullableTimeStampNanoTZHolder; import org.apache.arrow.vector.holders.TimeStampMilliTZHolder; import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID; import org.apache.arrow.vector.types.pojo.ArrowType.Int; @@ -93,6 +99,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType.Utf8; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.UuidType; import org.apache.arrow.vector.util.CallBack; import org.apache.arrow.vector.util.DecimalUtility; import org.apache.arrow.vector.util.JsonStringArrayList; @@ -2489,4 +2496,38 @@ public void unionWithVarCharAndBinaryHelpers() throws Exception { "row12", new String(vector.getLargeVarBinaryVector().get(11), StandardCharsets.UTF_8)); } } + + @Test + public void extensionWriterReader() throws Exception { + // test values + UUID u1 = UUID.randomUUID(); + + try (NonNullableStructVector parent = NonNullableStructVector.empty("parent", allocator)) { + // write + + ComplexWriter writer = new ComplexWriterImpl("root", parent); + StructWriter rootWriter = writer.rootAsStruct(); + + { + ExtensionWriter extensionWriter = rootWriter.extension("uuid1", new UuidType()); + extensionWriter.setPosition(0); + extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); + extensionWriter.writeExtension(u1); + } + // read + StructReader rootReader = new SingleStructReaderImpl(parent).reader("root"); + { + FieldReader uuidReader = rootReader.reader("uuid1"); + uuidReader.setPosition(0); + UuidHolder uuidHolder = new UuidHolder(); + uuidReader.read(uuidHolder); + final ByteBuffer bb = ByteBuffer.wrap(uuidHolder.value); + UUID actualUuid = new UUID(bb.getLong(), bb.getLong()); + assertEquals(u1, actualUuid); + assertTrue(uuidReader.isSet()); + assertEquals(uuidReader.getMinorType(), MinorType.EXTENSIONTYPE); + assertInstanceOf(UuidType.class, uuidReader.getField().getFieldType().getType()); + } + } + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java index bf1b9b0dfa..269cff0670 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java @@ -30,6 +30,7 @@ import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.complex.impl.LargeVarBinaryWriterImpl; import org.apache.arrow.vector.complex.impl.LargeVarCharWriterImpl; +import org.apache.arrow.vector.complex.impl.UuidReaderImpl; import org.apache.arrow.vector.complex.impl.UuidWriterImpl; import org.apache.arrow.vector.complex.impl.VarBinaryWriterImpl; import org.apache.arrow.vector.complex.impl.VarCharWriterImpl; @@ -204,4 +205,23 @@ public void testWriteToExtensionVector() throws Exception { assertEquals(uuid, result); } } + + @Test + public void testReaderCopyAsValueExtensionVector() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator); + UuidVector vectorForRead = new UuidVector("test2", allocator); + UuidWriterImpl writer = new UuidWriterImpl(vector)) { + UUID uuid = UUID.randomUUID(); + vectorForRead.setValueCount(1); + vectorForRead.set(0, uuid); + UuidReaderImpl reader = (UuidReaderImpl) vectorForRead.getReader(); + reader.copyAsValue(writer); + UuidReaderImpl reader2 = (UuidReaderImpl) vector.getReader(); + UuidHolder holder = new UuidHolder(); + reader2.read(0, holder); + final ByteBuffer bb = ByteBuffer.wrap(holder.value); + UUID actualUuid = new UUID(bb.getLong(), bb.getLong()); + assertEquals(uuid, actualUuid); + } + } } From e4f64269db0a08299fa25be491570be5ba71d623 Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 5 Aug 2025 12:37:33 +0900 Subject: [PATCH 052/271] MINOR: Fix format (#809) ## What's Changed Apply pre-commit since I forgot. --- vector/src/main/codegen/templates/UnionListWriter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vector/src/main/codegen/templates/UnionListWriter.java b/vector/src/main/codegen/templates/UnionListWriter.java index 94723e6c9d..3c41ac72b6 100644 --- a/vector/src/main/codegen/templates/UnionListWriter.java +++ b/vector/src/main/codegen/templates/UnionListWriter.java @@ -340,12 +340,12 @@ public void writeExtension(Object value) { writer.writeExtension(value); writer.setPosition(writer.idx() + 1); } - + @Override public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory var1) { writer.addExtensionTypeWriterFactory(var1, extensionType); } - + public void write(ExtensionHolder var1) { writer.write(var1); writer.setPosition(writer.idx() + 1); From a6245ab87c782e2b873526acbd876cdf96d1f0ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Thu, 7 Aug 2025 01:29:41 +0100 Subject: [PATCH 053/271] GH-804: Prepend JDBC FlightSQL version to user agent (#806) ## What's Changed * Driver version is passed on to NettyBuilderClient to append it to the user-agent header * NettyBuilderClient now prepends `JDBC Flight SQL Client ` to the header (e.g. `JDBC Flight SQL Client 19.0.0-SNAPSHOT grpc-java-netty/1.73.0`) Closes #804. --- .../driver/jdbc/ArrowFlightConnection.java | 9 +++- .../client/ArrowFlightSqlClientHandler.java | 27 +++++++++++ .../ArrowFlightJdbcConnectionCookieTest.java | 4 +- .../arrow/driver/jdbc/ConnectionTest.java | 46 +++++++++++++++++++ .../jdbc/FlightServerTestExtension.java | 33 +++++++++---- ...rrowFlightSqlClientHandlerBuilderTest.java | 1 + 6 files changed, 107 insertions(+), 13 deletions(-) diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java index 747287ed13..f6f17770f1 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java @@ -32,6 +32,7 @@ import org.apache.arrow.util.Preconditions; import org.apache.calcite.avatica.AvaticaConnection; import org.apache.calcite.avatica.AvaticaFactory; +import org.apache.calcite.avatica.DriverVersion; /** Connection to the Arrow Flight server. */ public final class ArrowFlightConnection extends AvaticaConnection { @@ -86,13 +87,16 @@ static ArrowFlightConnection createNewConnection( throws SQLException { url = replaceSemiColons(url); final ArrowFlightConnectionConfigImpl config = new ArrowFlightConnectionConfigImpl(properties); - final ArrowFlightSqlClientHandler clientHandler = createNewClientHandler(config, allocator); + final ArrowFlightSqlClientHandler clientHandler = + createNewClientHandler(config, allocator, driver.getDriverVersion()); return new ArrowFlightConnection( driver, factory, url, properties, config, allocator, clientHandler); } private static ArrowFlightSqlClientHandler createNewClientHandler( - final ArrowFlightConnectionConfigImpl config, final BufferAllocator allocator) + final ArrowFlightConnectionConfigImpl config, + final BufferAllocator allocator, + final DriverVersion driverVersion) throws SQLException { try { return new ArrowFlightSqlClientHandler.Builder() @@ -116,6 +120,7 @@ private static ArrowFlightSqlClientHandler createNewClientHandler( .withCatalog(config.getCatalog()) .withClientCache(config.useClientCache() ? new FlightClientCache() : null) .withConnectTimeout(config.getConnectTimeout()) + .withDriverVersion(driverVersion) .build(); } catch (final SQLException e) { try { diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java index 17c2c16ebf..a3f6900373 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java @@ -66,6 +66,7 @@ import org.apache.arrow.util.VisibleForTesting; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.calcite.avatica.DriverVersion; import org.apache.calcite.avatica.Meta.StatementType; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; @@ -548,6 +549,9 @@ public FlightInfo getCrossReference( /** Builder for {@link ArrowFlightSqlClientHandler}. */ public static final class Builder { + static final String USER_AGENT_TEMPLATE = "JDBC Flight SQL Driver %s"; + static final String DEFAULT_VERSION = "(unknown or development build)"; + private final Set middlewareFactories = new HashSet<>(); private final Set options = new HashSet<>(); private String host; @@ -597,6 +601,8 @@ public static final class Builder { @VisibleForTesting ClientCookieMiddleware.Factory cookieFactory = new ClientCookieMiddleware.Factory(); + DriverVersion driverVersion; + public Builder() {} /** @@ -631,6 +637,8 @@ public Builder() {} if (original.retainAuth) { this.authFactory = original.authFactory; } + + this.driverVersion = original.driverVersion; } /** @@ -879,6 +887,17 @@ public Builder withConnectTimeout(Duration connectTimeout) { return this; } + /** + * Sets the driver version for this handler. + * + * @param driverVersion the driver version to set + * @return this builder instance + */ + public Builder withDriverVersion(DriverVersion driverVersion) { + this.driverVersion = driverVersion; + return this; + } + public String getCacheKey() { return getLocation().toString(); } @@ -914,6 +933,11 @@ public ArrowFlightSqlClientHandler build() throws SQLException { final NettyClientBuilder clientBuilder = new NettyClientBuilder(); clientBuilder.allocator(allocator); + String userAgent = String.format(USER_AGENT_TEMPLATE, DEFAULT_VERSION); + if (driverVersion != null && driverVersion.versionString != null) { + userAgent = String.format(USER_AGENT_TEMPLATE, driverVersion.versionString); + } + buildTimeMiddlewareFactories.add(new ClientCookieMiddleware.Factory()); buildTimeMiddlewareFactories.forEach(clientBuilder::intercept); if (useEncryption) { @@ -948,6 +972,9 @@ public ArrowFlightSqlClientHandler build() throws SQLException { } NettyChannelBuilder channelBuilder = clientBuilder.build(); + + channelBuilder.userAgent(userAgent); + if (connectTimeout != null) { channelBuilder.withOption( ChannelOption.CONNECT_TIMEOUT_MILLIS, (int) connectTimeout.toMillis()); diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcConnectionCookieTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcConnectionCookieTest.java index 1977b61392..7127c7fc32 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcConnectionCookieTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcConnectionCookieTest.java @@ -39,11 +39,11 @@ public void testCookies() throws SQLException { Statement statement = connection.createStatement()) { // Expect client didn't receive cookies before any operation - assertNull(FLIGHT_SERVER_TEST_EXTENSION.getMiddlewareCookieFactory().getCookie()); + assertNull(FLIGHT_SERVER_TEST_EXTENSION.getInterceptorFactory().getCookie()); // Run another action for check if the cookies was sent by the server. statement.execute(CoreMockedSqlProducers.LEGACY_REGULAR_SQL_CMD); - assertEquals("k=v", FLIGHT_SERVER_TEST_EXTENSION.getMiddlewareCookieFactory().getCookie()); + assertEquals("k=v", FLIGHT_SERVER_TEST_EXTENSION.getInterceptorFactory().getCookie()); } } } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java index 8e872a1167..72e4b222a3 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java @@ -31,6 +31,7 @@ import org.apache.arrow.driver.jdbc.client.ArrowFlightSqlClientHandler; import org.apache.arrow.driver.jdbc.utils.ArrowFlightConnectionConfigImpl.ArrowFlightConnectionProperty; import org.apache.arrow.driver.jdbc.utils.MockFlightSqlProducer; +import org.apache.arrow.flight.FlightMethod; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.util.AutoCloseables; @@ -576,4 +577,49 @@ public void testPasswordConnectionPropertyIntegerCorrectCastUrlWithDriverManager assertTrue(connection.isValid(0)); } } + + /** + * Test that the JDBC driver properly integrates driver version into client handler. + * + * @throws Exception on error. + */ + @Test + public void testJdbcDriverVersionIntegration() throws Exception { + final Properties properties = new Properties(); + properties.put( + ArrowFlightConnectionProperty.HOST.camelName(), FLIGHT_SERVER_TEST_EXTENSION.getHost()); + properties.put( + ArrowFlightConnectionProperty.PORT.camelName(), FLIGHT_SERVER_TEST_EXTENSION.getPort()); + properties.put(ArrowFlightConnectionProperty.USER.camelName(), userTest); + properties.put(ArrowFlightConnectionProperty.PASSWORD.camelName(), passTest); + properties.put(ArrowFlightConnectionProperty.USE_ENCRYPTION.camelName(), false); + + // Create a driver instance and connect + ArrowFlightJdbcDriver driverVersion = new ArrowFlightJdbcDriver(); + + try (Connection connection = + ArrowFlightConnection.createNewConnection( + driverVersion, + new ArrowFlightJdbcFactory(), + "jdbc:arrow-flight-sql://localhost:" + FLIGHT_SERVER_TEST_EXTENSION.getPort(), + properties, + allocator)) { + + assertTrue(connection.isValid(0)); + + var actualUserAgent = + FLIGHT_SERVER_TEST_EXTENSION + .getInterceptorFactory() + .getHeader(FlightMethod.HANDSHAKE, "user-agent"); + + var expectedUserAgent = + "JDBC Flight SQL Driver " + driverVersion.getDriverVersion().versionString; + // Driver appends version to grpc user-agent header. Assert the header starts with the + // expected + // value and ignored grpc version. + assertTrue( + actualUserAgent.startsWith(expectedUserAgent), + "Expected: " + expectedUserAgent + " but found: " + actualUserAgent); + } + } } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/FlightServerTestExtension.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/FlightServerTestExtension.java index aa586651f5..db0438059f 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/FlightServerTestExtension.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/FlightServerTestExtension.java @@ -25,6 +25,8 @@ import java.sql.SQLException; import java.util.ArrayDeque; import java.util.Deque; +import java.util.HashMap; +import java.util.Map; import java.util.Properties; import org.apache.arrow.driver.jdbc.authentication.Authentication; import org.apache.arrow.driver.jdbc.authentication.TokenAuthentication; @@ -33,6 +35,7 @@ import org.apache.arrow.flight.CallHeaders; import org.apache.arrow.flight.CallInfo; import org.apache.arrow.flight.CallStatus; +import org.apache.arrow.flight.FlightMethod; import org.apache.arrow.flight.FlightServer; import org.apache.arrow.flight.FlightServerMiddleware; import org.apache.arrow.flight.Location; @@ -67,7 +70,8 @@ public class FlightServerTestExtension private final CertKeyPair certKeyPair; private final File mTlsCACert; - private final MiddlewareCookie.Factory middlewareCookieFactory = new MiddlewareCookie.Factory(); + private final InterceptorMiddleware.Factory interceptorFactory = + new InterceptorMiddleware.Factory(); private FlightServerTestExtension( final Properties properties, @@ -130,8 +134,8 @@ private void setUseEncryption(boolean useEncryption) { properties.put("useEncryption", useEncryption); } - public MiddlewareCookie.Factory getMiddlewareCookieFactory() { - return middlewareCookieFactory; + public InterceptorMiddleware.Factory getInterceptorFactory() { + return interceptorFactory; } @FunctionalInterface @@ -143,7 +147,7 @@ private FlightServer initiateServer(Location location) throws IOException { FlightServer.Builder builder = FlightServer.builder(allocator, location, producer) .headerAuthenticator(authentication.authenticate()) - .middleware(FlightServerMiddleware.Key.of("KEY"), middlewareCookieFactory); + .middleware(FlightServerMiddleware.Key.of("KEY"), interceptorFactory); if (certKeyPair != null) { builder.useTls(certKeyPair.cert, certKeyPair.key); } @@ -301,11 +305,11 @@ public FlightServerTestExtension build() { * A middleware to handle with the cookies in the server. It is used to test if cookies are being * sent properly. */ - static class MiddlewareCookie implements FlightServerMiddleware { + static class InterceptorMiddleware implements FlightServerMiddleware { private final Factory factory; - public MiddlewareCookie(Factory factory) { + public InterceptorMiddleware(Factory factory) { this.factory = factory; } @@ -323,22 +327,33 @@ public void onCallCompleted(CallStatus callStatus) {} public void onCallErrored(Throwable throwable) {} /** A factory for the MiddlewareCookie. */ - static class Factory implements FlightServerMiddleware.Factory { + static class Factory implements FlightServerMiddleware.Factory { + private final Map receivedCallHeaders = new HashMap<>(); private boolean receivedCookieHeader = false; private String cookie; @Override - public MiddlewareCookie onCallStarted( + public InterceptorMiddleware onCallStarted( CallInfo callInfo, CallHeaders callHeaders, RequestContext requestContext) { cookie = callHeaders.get("Cookie"); receivedCookieHeader = null != cookie; - return new MiddlewareCookie(this); + + receivedCallHeaders.put(callInfo.method(), callHeaders); + return new InterceptorMiddleware(this); } public String getCookie() { return cookie; } + + public String getHeader(FlightMethod method, String key) { + CallHeaders headers = receivedCallHeaders.get(method); + if (headers == null) { + return null; + } + return headers.get(key); + } } } } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerBuilderTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerBuilderTest.java index 6524eaf39a..a60a71f23d 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerBuilderTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerBuilderTest.java @@ -149,6 +149,7 @@ public void testDefaults() { assertEquals(Optional.empty(), builder.catalog); assertNull(builder.flightClientCache); assertNull(builder.connectTimeout); + assertNull(builder.driverVersion); } @Test From 199e47bc2649538ca091c17ffb4c77b8dfd5966e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 05:53:05 +0900 Subject: [PATCH 054/271] MINOR: [CI] Bump actions/cache from 4.2.3 to 4.2.4 (#813) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/cache](https://github.com/actions/cache) from 4.2.3 to 4.2.4.
Release notes

Sourced from actions/cache's releases.

v4.2.4

What's Changed

New Contributors

Full Changelog: https://github.com/actions/cache/compare/v4...v4.2.4

Changelog

Sourced from actions/cache's changelog.

Releases

4.2.4

  • Bump @actions/cache to v4.0.5

4.2.3

  • Bump @actions/cache to v4.0.3 (obfuscates SAS token in debug logs for cache entries)

4.2.2

  • Bump @actions/cache to v4.0.2

4.2.1

  • Bump @actions/cache to v4.0.1

4.2.0

TLDR; The cache backend service has been rewritten from the ground up for improved performance and reliability. actions/cache now integrates with the new cache service (v2) APIs.

The new service will gradually roll out as of February 1st, 2025. The legacy service will also be sunset on the same date. Changes in these release are fully backward compatible.

We are deprecating some versions of this action. We recommend upgrading to version v4 or v3 as soon as possible before February 1st, 2025. (Upgrade instructions below).

If you are using pinned SHAs, please use the SHAs of versions v4.2.0 or v3.4.0

If you do not upgrade, all workflow runs using any of the deprecated actions/cache will fail.

Upgrading to the recommended versions will not break your workflows.

4.1.2

  • Add GitHub Enterprise Cloud instances hostname filters to inform API endpoint choices - #1474
  • Security fix: Bump braces from 3.0.2 to 3.0.3 - #1475

4.1.1

  • Restore original behavior of cache-hit output - #1467

4.1.0

  • Ensure cache-hit output is set when a cache is missed - #1404
  • Deprecate save-always input - #1452

4.0.2

  • Fixed restore fail-on-cache-miss not working.

... (truncated)

Commits
  • 0400d5f Merge pull request #1636 from actions/Link-/release-4.2.4
  • 374a27f Prepare release 4.2.4
  • 358a730 Merge pull request #1634 from actions/Link-/optimise-deps
  • 2ee706e Fix with another approach
  • 94f7b5d Fix bundle exec
  • c36116c Fix the workflow to use licensed from source
  • 320fe7d Update the licensed workflow to use the latest version
  • d81cc47 Add licensed output
  • de24398 Add licensed output
  • e7b6a9c @​protobuf-ts/plugin to dev dependencies
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/cache&package-manager=github_actions&previous-version=4.2.3&new-version=4.2.4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/dev.yml | 2 +- .github/workflows/rc.yml | 8 ++++---- .github/workflows/test.yml | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 37320898a6..4242af55c3 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -42,7 +42,7 @@ jobs: with: python-version: '3.x' - name: pre-commit (cache) - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 with: path: ~/.cache/pre-commit key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 72456fa556..8836537b7f 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -139,7 +139,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Cache - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 with: path: .docker key: jni-linux-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} @@ -270,7 +270,7 @@ jobs: run: | echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} - name: Cache ccache - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 with: path: ccache key: jni-macos-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} @@ -346,7 +346,7 @@ jobs: run: | echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} - name: Cache ccache - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 with: path: ccache key: jni-windows-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} @@ -420,7 +420,7 @@ jobs: repository: apache/arrow-testing path: testing - name: Cache ~/.m2 - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 with: path: ~/.m2 key: binaries-build-${{ hashFiles('**/*.java', '**/pom.xml') }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f1028ce029..c461b3c19e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,7 +63,7 @@ jobs: fetch-depth: 0 submodules: recursive - name: Cache Docker Volumes - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 with: path: .docker key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('compose.yaml', '**/pom.xml', '**/*.java') }} @@ -185,7 +185,7 @@ jobs: run: | ci/scripts/util_free_space.sh - name: Cache Docker Volumes - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 with: path: .docker key: integration-conda-${{ hashFiles('cpp/**') }} From edf64236b1ee58ca1ff081c7dca31384e6b0e280 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 05:53:32 +0900 Subject: [PATCH 055/271] MINOR: [CI] Bump docker/login-action from 3.4.0 to 3.5.0 (#814) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [docker/login-action](https://github.com/docker/login-action) from 3.4.0 to 3.5.0.
Release notes

Sourced from docker/login-action's releases.

v3.5.0

Full Changelog: https://github.com/docker/login-action/compare/v3.4.0...v3.5.0

Commits
  • 184bdaa Merge pull request #878 from docker/dependabot/npm_and_yarn/aws-sdk-dependenc...
  • 5c6bc94 chore: update generated content
  • caf4058 build(deps): bump the aws-sdk-dependencies group with 2 updates
  • ef38ec3 Merge pull request #860 from docker/dependabot/npm_and_yarn/aws-sdk-dependenc...
  • d52e8ef chore: update generated content
  • 9644ab7 build(deps): bump the aws-sdk-dependencies group with 2 updates
  • 7abd1d5 Merge pull request #875 from docker/dependabot/npm_and_yarn/form-data-2.5.5
  • 1a81202 Merge pull request #876 from crazy-max/aws-public-dual-stack
  • d1ab30d chore: update generated content
  • f25ff28 support dual-stack for aws public ecr
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=docker/login-action&package-manager=github_actions&previous-version=3.4.0&new-version=3.5.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 8836537b7f..98778dd0d0 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -133,7 +133,7 @@ jobs: with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing - - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + - uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 with: registry: ghcr.io username: ${{ github.actor }} From 65caf509bdb33b96e576ce15cb734fe1689bdb37 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 05:54:58 +0900 Subject: [PATCH 056/271] MINOR: [CI] Bump actions/download-artifact from 4.3.0 to 5.0.0 (#815) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 4.3.0 to 5.0.0.
Release notes

Sourced from actions/download-artifact's releases.

v5.0.0

What's Changed

v5.0.0

🚨 Breaking Change

This release fixes an inconsistency in path behavior for single artifact downloads by ID. If you're downloading single artifacts by ID, the output path may change.

What Changed

Previously, single artifact downloads behaved differently depending on how you specified the artifact:

  • By name: name: my-artifact → extracted to path/ (direct)
  • By ID: artifact-ids: 12345 → extracted to path/my-artifact/ (nested)

Now both methods are consistent:

  • By name: name: my-artifact → extracted to path/ (unchanged)
  • By ID: artifact-ids: 12345 → extracted to path/ (fixed - now direct)

Migration Guide

✅ No Action Needed If:
  • You download artifacts by name
  • You download multiple artifacts by ID
  • You already use merge-multiple: true as a workaround
⚠️ Action Required If:

You download single artifacts by ID and your workflows expect the nested directory structure.

Before v5 (nested structure):

- uses: actions/download-artifact@v4
  with:
    artifact-ids: 12345
    path: dist
# Files were in: dist/my-artifact/

Where my-artifact is the name of the artifact you previously uploaded

To maintain old behavior (if needed):

</tr></table>

... (truncated)

Commits
  • 634f93c Merge pull request #416 from actions/single-artifact-id-download-path
  • b19ff43 refactor: resolve download path correctly in artifact download tests (mainly ...
  • e262cbe bundle dist
  • bff23f9 update docs
  • fff8c14 fix download path logic when downloading a single artifact by id
  • 448e3f8 Merge pull request #407 from actions/nebuk89-patch-1
  • 47225c4 Update README.md
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/download-artifact&package-manager=github_actions&previous-version=4.3.0&new-version=5.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 98778dd0d0..9a8d7d9563 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -101,7 +101,7 @@ jobs: packages: write steps: - name: Download source archive - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: name: release-source - name: Extract source archive @@ -174,7 +174,7 @@ jobs: MACOSX_DEPLOYMENT_TARGET: "14.0" steps: - name: Download source archive - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: name: release-source - name: Extract source archive @@ -302,7 +302,7 @@ jobs: arch: "x86_64" steps: - name: Download source archive - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: name: release-source - name: Extract source archive @@ -375,7 +375,7 @@ jobs: - jni-windows steps: - name: Download artifacts - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: path: artifacts - name: Decompress artifacts @@ -456,11 +456,11 @@ jobs: with: cache: 'pip' - name: Download source archive - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: name: release-source - name: Download Javadocs - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: name: reference - name: Extract source archive @@ -525,7 +525,7 @@ jobs: cp ../.asf.yaml ./ git add .nojekyll .asf.yaml - name: Download - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: name: release-html - name: Extract @@ -561,7 +561,7 @@ jobs: - ubuntu-latest steps: - name: Download release artifacts - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: pattern: release-* - name: Verify @@ -595,7 +595,7 @@ jobs: contents: write steps: - name: Download release artifacts - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: pattern: release-* path: artifacts From 52f7a86b0f315df8848dd68cb7457e13236b5fd5 Mon Sep 17 00:00:00 2001 From: David Schlosnagle Date: Wed, 13 Aug 2025 19:39:22 -0400 Subject: [PATCH 057/271] GH-816: Presize JsonStringArrayList vector results (#817) ## What's Changed Presize `JsonStringArrayList`s when constructing them as part of `ValueVector#getObject` conversions. `FixedSizeListVector#getObject` already performs this optimization; however, `ListVector`, `ListViewVector`, `LargeListVector`, and `LargeListViewVector` do not yet presize the result `JsonStringArrayList` requiring dynamic reallocations as elements are converted & added. This can become a scalability bottleneck when using these types. Closes #816. --- .../apache/arrow/adapter/avro/AvroToArrowUtils.java | 4 ++-- .../arrow/adapter/avro/AvroToArrowVectorIterator.java | 11 +++++++---- .../apache/arrow/vector/complex/LargeListVector.java | 4 +++- .../arrow/vector/complex/LargeListViewVector.java | 2 +- .../org/apache/arrow/vector/complex/ListVector.java | 2 +- .../apache/arrow/vector/complex/ListViewVector.java | 2 +- 6 files changed, 15 insertions(+), 10 deletions(-) diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java index aedef7732e..a6e77e4050 100644 --- a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java @@ -1071,8 +1071,8 @@ private static FieldType createFieldType( } private static String convertAliases(Set aliases) { - JsonStringArrayList jsonList = new JsonStringArrayList(); - aliases.stream().forEach(a -> jsonList.add(a)); + JsonStringArrayList jsonList = new JsonStringArrayList(aliases.size()); + jsonList.addAll(aliases); return jsonList.toString(); } } diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowVectorIterator.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowVectorIterator.java index 4123370061..e82fdc36fb 100644 --- a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowVectorIterator.java +++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowVectorIterator.java @@ -17,13 +17,14 @@ package org.apache.arrow.adapter.avro; import java.io.EOFException; -import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.stream.Collectors; import org.apache.arrow.adapter.avro.consumers.CompositeAvroConsumer; +import org.apache.arrow.adapter.avro.consumers.Consumer; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.util.ValueVectorUtility; @@ -75,9 +76,11 @@ public static AvroToArrowVectorIterator create( private void initialize() { // create consumers compositeConsumer = AvroToArrowUtils.createCompositeConsumer(schema, config); - List vectors = new ArrayList<>(); - compositeConsumer.getConsumers().forEach(c -> vectors.add(c.getVector())); - List fields = vectors.stream().map(t -> t.getField()).collect(Collectors.toList()); + List vectors = + compositeConsumer.getConsumers().stream() + .map(Consumer::getVector) + .collect(Collectors.toList()); + List fields = vectors.stream().map(ValueVector::getField).collect(Collectors.toList()); VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, 0); rootSchema = root.getSchema(); diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java index 835d3468f3..997b5a8b78 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java @@ -31,6 +31,7 @@ import org.apache.arrow.memory.util.ArrowBufPointer; import org.apache.arrow.memory.util.ByteFunctionHelpers; import org.apache.arrow.memory.util.CommonUtil; +import org.apache.arrow.memory.util.LargeMemoryUtil; import org.apache.arrow.memory.util.hash.ArrowBufHasher; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.AddOrGetResult; @@ -861,10 +862,11 @@ public List getObject(int index) { if (isSet(index) == 0) { return null; } - final List vals = new JsonStringArrayList<>(); final long start = offsetBuffer.getLong((long) index * OFFSET_WIDTH); final long end = offsetBuffer.getLong(((long) index + 1L) * OFFSET_WIDTH); final ValueVector vv = getDataVector(); + final List vals = + new JsonStringArrayList<>(LargeMemoryUtil.checkedCastToInt(end - start)); for (long i = start; i < end; i++) { vals.add(vv.getObject(checkedCastToInt(i))); } diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java index 394c3c67bb..2da7eb057e 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java @@ -672,10 +672,10 @@ public List getObject(int index) { if (isSet(index) == 0) { return null; } - final List vals = new JsonStringArrayList<>(); final int start = offsetBuffer.getInt(index * OFFSET_WIDTH); final int end = start + sizeBuffer.getInt((index) * SIZE_WIDTH); final ValueVector vv = getDataVector(); + final List vals = new JsonStringArrayList<>(end - start); for (int i = start; i < end; i++) { vals.add(vv.getObject(checkedCastToInt(i))); } diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 2b2817515f..93a313ef4f 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -719,10 +719,10 @@ public List getObject(int index) { if (isSet(index) == 0) { return null; } - final List vals = new JsonStringArrayList<>(); final int start = offsetBuffer.getInt(index * OFFSET_WIDTH); final int end = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH); final ValueVector vv = getDataVector(); + final List vals = new JsonStringArrayList<>(end - start); for (int i = start; i < end; i++) { vals.add(vv.getObject(i)); } diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java index 2b80101926..8711db5e0f 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java @@ -678,10 +678,10 @@ public List getObject(int index) { if (isSet(index) == 0) { return null; } - final List vals = new JsonStringArrayList<>(); final int start = offsetBuffer.getInt(index * OFFSET_WIDTH); final int end = start + sizeBuffer.getInt((index) * SIZE_WIDTH); final ValueVector vv = getDataVector(); + final List vals = new JsonStringArrayList<>(end - start); for (int i = start; i < end; i++) { vals.add(vv.getObject(i)); } From 156b465f7836dec3c3a109dbc101ee7270009ff7 Mon Sep 17 00:00:00 2001 From: Pedro Matias Date: Wed, 20 Aug 2025 11:36:12 +0100 Subject: [PATCH 058/271] GH-797: [JDBC] Fix PreparedStatement#execute for DML/DDL (#811) ## What's Changed Instead of always obtaining a result set for queries issued via PreparedStatement.execute(), we now inspect the dataset_schema returned in ActionCreatePreparedStatementResult. If the schema has no fields, we retrieve the update count instead. This aligns the return value with the expectations of the JDBC API. For such cases, the Arrow Flight SQL path now uses CommandPreparedStatementUpdate instead of CommandPreparedStatementQuery. This change mirrors the existing approach in Statement.execute() and Statement.executeUpdate(). ### Are these changes tested? Yes Closes #797. --- .../driver/jdbc/ArrowFlightMetaImpl.java | 10 +++++-- .../ArrowFlightPreparedStatementTest.java | 29 +++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java index 9c7112f1c3..21cc3e431f 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java @@ -62,14 +62,17 @@ static Signature newSignature(final String sql, Schema resultSetSchema, Schema p parameterSchema == null ? new ArrayList<>() : ConvertUtils.convertArrowFieldsToAvaticaParameters(parameterSchema.getFields()); - + StatementType statementType = + resultSetSchema == null || resultSetSchema.getFields().isEmpty() + ? StatementType.IS_DML + : StatementType.SELECT; return new Signature( columnMetaData, sql, parameters, Collections.emptyMap(), null, // unnecessary, as SQL requests use ArrowFlightJdbcCursor - StatementType.SELECT); + statementType); } @Override @@ -105,7 +108,8 @@ public ExecuteResult execute( preparedStatement, ((ArrowFlightConnection) connection).getBufferAllocator()) .bind(typedValues); - if (statementHandle.signature == null) { + if (statementHandle.signature == null + || statementHandle.signature.statementType == StatementType.IS_DML) { // Update query long updatedCount = preparedStatement.executeUpdate(); return new ExecuteResult( diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightPreparedStatementTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightPreparedStatementTest.java index 774ad0081e..0369c3a162 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightPreparedStatementTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightPreparedStatementTest.java @@ -20,6 +20,8 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.nio.charset.StandardCharsets; import java.sql.Connection; @@ -83,6 +85,19 @@ public void testSimpleQueryNoParameterBinding() throws SQLException { } } + @Test + public void testSimpleQueryNoParameterBindingWithExecute() throws SQLException { + final String query = CoreMockedSqlProducers.LEGACY_REGULAR_SQL_CMD; + try (final PreparedStatement preparedStatement = connection.prepareStatement(query)) { + boolean isResultSet = preparedStatement.execute(); + assertTrue(isResultSet); + final ResultSet resultSet = preparedStatement.getResultSet(); + CoreMockedSqlProducers.assertLegacyRegularSqlResultSet(resultSet); + assertFalse(preparedStatement.getMoreResults()); + assertEquals(-1, preparedStatement.getUpdateCount()); + } + } + @Test public void testQueryWithParameterBinding() throws SQLException { final String query = "Fake query with parameters"; @@ -174,6 +189,20 @@ public void testUpdateQuery() throws SQLException { } } + @Test + public void testUpdateQueryWithExecute() throws SQLException { + String query = "Fake update with execute"; + PRODUCER.addUpdateQuery(query, /*updatedRows*/ 42); + try (final PreparedStatement stmt = connection.prepareStatement(query)) { + boolean isResultSet = stmt.execute(); + assertFalse(isResultSet); + int updated = stmt.getUpdateCount(); + assertEquals(42, updated); + assertFalse(stmt.getMoreResults()); + assertEquals(-1, stmt.getUpdateCount()); + } + } + @Test public void testUpdateQueryWithParameters() throws SQLException { String query = "Fake update with parameters"; From c21b8a731d2d700a7edd28f8d7dba2983cbd4dae Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 8 Sep 2025 15:50:11 +0900 Subject: [PATCH 059/271] GH-841: Use apache/arrow-dotnet for integration test (#842) ## What's Changed Use apache/arrow-dotnet for the .NET implementation. Use `@vX` for `actions/*` because Dependabot sometimes doesn't work well for `@SHA512`. See also: https://github.com/apache/arrow-java/pull/820#discussion_r2283530810 Closes #841. --- .github/workflows/test.yml | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c461b3c19e..5d420d1f91 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,12 +58,12 @@ jobs: MAVEN: ${{ matrix.maven }} steps: - name: Checkout Arrow - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + uses: actions/checkout@v5 with: fetch-depth: 0 submodules: recursive - name: Cache Docker Volumes - uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + uses: actions/cache@v4 with: path: .docker key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('compose.yaml', '**/pom.xml', '**/*.java') }} @@ -95,12 +95,12 @@ jobs: macos: latest steps: - name: Set up Java - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: 'temurin' java-version: ${{ matrix.jdk }} - name: Checkout Arrow - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: fetch-depth: 0 submodules: recursive @@ -126,12 +126,12 @@ jobs: jdk: [11] steps: - name: Set up Java - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: ${{ matrix.jdk }} distribution: 'temurin' - name: Checkout Arrow - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: fetch-depth: 0 submodules: recursive @@ -152,32 +152,37 @@ jobs: timeout-minutes: 60 steps: - name: Checkout Arrow - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: fetch-depth: 0 repository: apache/arrow submodules: recursive - name: Checkout Arrow Rust - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: repository: apache/arrow-rs path: rust - name: Checkout Arrow nanoarrow - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: repository: apache/arrow-nanoarrow path: nanoarrow + - name: Checkout Arrow .NET + uses: actions/checkout@v5 + with: + repository: apache/arrow-dotnet + path: dotnet - name: Checkout Arrow Go - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: repository: apache/arrow-go path: go - name: Checkout Arrow Java - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: path: java - name: Checkout Arrow JavaScript - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: repository: apache/arrow-js path: js @@ -185,13 +190,13 @@ jobs: run: | ci/scripts/util_free_space.sh - name: Cache Docker Volumes - uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + uses: actions/cache@v4 with: path: .docker key: integration-conda-${{ hashFiles('cpp/**') }} restore-keys: integration-conda- - name: Setup Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + uses: actions/setup-python@v5 with: python-version: 3.12 - name: Setup Archery @@ -202,6 +207,7 @@ jobs: archery docker run \ -e ARCHERY_DEFAULT_BRANCH=main \ -e ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS=java \ + -e ARCHERY_INTEGRATION_WITH_DOTNET=1 \ -e ARCHERY_INTEGRATION_WITH_GO=1 \ -e ARCHERY_INTEGRATION_WITH_JAVA=1 \ -e ARCHERY_INTEGRATION_WITH_JS=1 \ From d51f300df79acf408c29dae68c6184835ecf91a0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Sep 2025 10:08:02 +0900 Subject: [PATCH 060/271] MINOR: [CI] Bump actions/setup-java from 4.6.0 to 5.0.0 (#847) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/setup-java](https://github.com/actions/setup-java) from 4.6.0 to 5.0.0.
Release notes

Sourced from actions/setup-java's releases.

v5.0.0

What's Changed

Breaking Changes

Make sure your runner is updated to this version or newer to use this release. v2.327.1 Release Notes

Dependency Upgrades

Bug Fixes

New Contributors

Full Changelog: https://github.com/actions/setup-java/compare/v4...v5.0.0

v4.7.1

What's Changed

Documentation changes

Dependency updates:

Full Changelog: https://github.com/actions/setup-java/compare/v4...v4.7.1

v4.7.0

What's Changed

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/setup-java&package-manager=github_actions&previous-version=4.6.0&new-version=5.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sutou Kouhei --- .github/workflows/rc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 9a8d7d9563..25c31da84a 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -321,7 +321,7 @@ jobs: repository: apache/arrow path: arrow - name: Set up Java - uses: actions/setup-java@7a6d8a8234af8eb26422e24e3006232cccaa061b # v4.6.0 + uses: actions/setup-java@v5 with: java-version: '11' distribution: 'temurin' From 8dd5ed8ab1b9655dbbb4fe315291e9d688f10b5b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Sep 2025 10:08:53 +0900 Subject: [PATCH 061/271] MINOR: [CI] Bump actions/setup-python from 5 to 6 (#843) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5 to 6.
Release notes

Sourced from actions/setup-python's releases.

v6.0.0

What's Changed

Breaking Changes

Make sure your runner is on version v2.327.1 or later to ensure compatibility with this release. See Release Notes

Enhancements:

Bug fixes:

Dependency updates:

New Contributors

Full Changelog: https://github.com/actions/setup-python/compare/v5...v6.0.0

v5.6.0

What's Changed

Full Changelog: https://github.com/actions/setup-python/compare/v5...v5.6.0

v5.5.0

What's Changed

Enhancements:

Bug fixes:

... (truncated)

Commits
  • e797f83 Upgrade to node 24 (#1164)
  • 3d1e2d2 Revert "Enhance cache-dependency-path handling to support files outside the w...
  • 65b0712 Clarify pythonLocation behavior for PyPy and GraalPy in environment variables...
  • 5b668cf Bump actions/checkout from 4 to 5 (#1181)
  • f62a0e2 Change missing cache directory error to warning (#1182)
  • 9322b3c Upgrade setuptools to 78.1.1 to fix path traversal vulnerability in PackageIn...
  • fbeb884 Bump form-data to fix critical vulnerabilities #182 & #183 (#1163)
  • 03bb615 Bump idna from 2.9 to 3.7 in /tests/data (#843)
  • 36da51d Add version parsing from Pipfile (#1067)
  • 3c6f142 update documentation (#1156)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/setup-python&package-manager=github_actions&previous-version=5&new-version=6)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sutou Kouhei --- .github/workflows/dev.yml | 2 +- .github/workflows/rc.yml | 4 ++-- .github/workflows/test.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 4242af55c3..12968bbcca 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -38,7 +38,7 @@ jobs: fetch-depth: 0 persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: '3.x' - name: pre-commit (cache) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 25c31da84a..eec3344097 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -201,7 +201,7 @@ jobs: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing - name: Set up Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + uses: actions/setup-python@v6 with: cache: 'pip' python-version: 3.12 @@ -452,7 +452,7 @@ jobs: contents: read packages: write steps: - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + - uses: actions/setup-python@v6 with: cache: 'pip' - name: Download source archive diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5d420d1f91..65fbc262fc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -196,7 +196,7 @@ jobs: key: integration-conda-${{ hashFiles('cpp/**') }} restore-keys: integration-conda- - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: 3.12 - name: Setup Archery From e026f3c29832577ee4f55dff8f942d7cab8301ae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Sep 2025 10:09:27 +0900 Subject: [PATCH 062/271] MINOR: [CI] Bump actions/github-script from 7.0.1 to 8.0.0 (#844) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/github-script](https://github.com/actions/github-script) from 7.0.1 to 8.0.0.
Release notes

Sourced from actions/github-script's releases.

v8.0.0

What's Changed

⚠️ Minimum Compatible Runner Version

v2.327.1
Release Notes

Make sure your runner is updated to this version or newer to use this release.

New Contributors

Full Changelog: https://github.com/actions/github-script/compare/v7.1.0...v8.0.0

v7.1.0

What's Changed

New Contributors

Full Changelog: https://github.com/actions/github-script/compare/v7...v7.1.0

Commits
  • ed59741 Merge pull request #653 from actions/sneha-krip/readme-for-v8
  • 2dc352e Bold minimum Actions Runner version in README
  • 01e118c Update README for Node 24 runtime requirements
  • 8b222ac Apply suggestion from @​salmanmkc
  • adc0eea README for updating actions/github-script from v7 to v8
  • 20fe497 Merge pull request #637 from actions/node24
  • e7b7f22 update licenses
  • 2c81ba0 Update Node.js version support to 24.x
  • f28e40c Merge pull request #610 from actions/nebuk89-patch-1
  • 1ae9958 Update README.md
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/github-script&package-manager=github_actions&previous-version=7.0.1&new-version=8.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sutou Kouhei --- .github/workflows/comment_bot.yml | 2 +- .github/workflows/dev_pr.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml index 5fbc858cc6..b4dbc92dfb 100644 --- a/.github/workflows/comment_bot.yml +++ b/.github/workflows/comment_bot.yml @@ -30,7 +30,7 @@ jobs: if: github.event.comment.body == 'take' runs-on: ubuntu-latest steps: - - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + - uses: actions/github-script@v8 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: |- diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml index 7352137b09..58f5f3d30a 100644 --- a/.github/workflows/dev_pr.yml +++ b/.github/workflows/dev_pr.yml @@ -49,28 +49,28 @@ jobs: - name: Ensure PR title format id: title-format - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + uses: actions/github-script@v8 with: script: | const scripts = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr.js`); return scripts.check_title_format({core, github, context}); - name: Label PR - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + uses: actions/github-script@v8 with: script: | const scripts = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr.js`); await scripts.apply_labels({core, github, context}); - name: Ensure PR is labeled - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + uses: actions/github-script@v8 with: script: | const scripts = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr.js`); await scripts.check_labels({core, github, context}); - name: Ensure PR is linked to an issue - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + uses: actions/github-script@v8 with: script: | const scripts = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr.js`); From ec3a424736ce00e7469cb6d983061c335bf4853a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Giraldo?= Date: Thu, 25 Sep 2025 01:09:42 -0600 Subject: [PATCH 063/271] GH-858: Fix error handling in CompositeJdbcConsumer (#857) ## What's Changed Turns out not all MinorTypes have a corresponding ArrowType, which can cause the following exception while handling the original exception: ``` Caused by: java.lang.UnsupportedOperationException: Cannot get simple type for type DECIMAL at org.apache.arrow.vector.types.Types$MinorType.getType(Types.java:815) at org.apache.arrow.adapter.jdbc.consumer.CompositeJdbcConsumer.consume(CompositeJdbcConsumer.java:49) ``` This PR changes the value we store in the exception to be the `MinorType` instead so we can still get useful info about the type but avoiding this possible exception. Closes #858. --- .../jdbc/consumer/CompositeJdbcConsumer.java | 5 ++--- .../consumer/exceptions/JdbcConsumerException.java | 14 +++++++------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/CompositeJdbcConsumer.java b/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/CompositeJdbcConsumer.java index 2366116fd0..b8389ee27c 100644 --- a/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/CompositeJdbcConsumer.java +++ b/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/CompositeJdbcConsumer.java @@ -24,7 +24,6 @@ import org.apache.arrow.util.AutoCloseables; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.types.pojo.ArrowType; /** Composite consumer which hold all consumers. It manages the consume and cleanup process. */ public class CompositeJdbcConsumer implements JdbcConsumer { @@ -46,9 +45,9 @@ public void consume(ResultSet rs) throws SQLException, IOException { BaseConsumer consumer = (BaseConsumer) consumers[i]; JdbcFieldInfo fieldInfo = new JdbcFieldInfo(rs.getMetaData(), consumer.columnIndexInResultSet); - ArrowType arrowType = consumer.vector.getMinorType().getType(); + throw new JdbcConsumerException( - "Exception while consuming JDBC value", e, fieldInfo, arrowType); + "Exception while consuming JDBC value", e, fieldInfo, consumer.vector.getField()); } else { throw e; } diff --git a/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/exceptions/JdbcConsumerException.java b/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/exceptions/JdbcConsumerException.java index 04e26d640c..98927f416c 100644 --- a/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/exceptions/JdbcConsumerException.java +++ b/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/exceptions/JdbcConsumerException.java @@ -17,7 +17,7 @@ package org.apache.arrow.adapter.jdbc.consumer.exceptions; import org.apache.arrow.adapter.jdbc.JdbcFieldInfo; -import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; /** * Exception while consuming JDBC data. This exception stores the JdbcFieldInfo for the column and @@ -25,7 +25,7 @@ */ public class JdbcConsumerException extends RuntimeException { final JdbcFieldInfo fieldInfo; - final ArrowType arrowType; + final Field field; /** * Construct JdbcConsumerException with all fields. @@ -33,17 +33,17 @@ public class JdbcConsumerException extends RuntimeException { * @param message error message * @param cause original exception * @param fieldInfo JdbcFieldInfo for the column - * @param arrowType ArrowType for the corresponding vector + * @param field ArrowType for the corresponding vector */ public JdbcConsumerException( - String message, Throwable cause, JdbcFieldInfo fieldInfo, ArrowType arrowType) { + String message, Throwable cause, JdbcFieldInfo fieldInfo, Field field) { super(message, cause); this.fieldInfo = fieldInfo; - this.arrowType = arrowType; + this.field = field; } - public ArrowType getArrowType() { - return this.arrowType; + public Field getField() { + return this.field; } public JdbcFieldInfo getFieldInfo() { From f38e72f5a46e30f44c28ff407eb522a5e5c266c0 Mon Sep 17 00:00:00 2001 From: Zhen Wang <643348094@qq.com> Date: Fri, 26 Sep 2025 09:21:33 +0800 Subject: [PATCH 064/271] GH-859: Fix ARROW_STRUCT_CONFLICT_POLICY env var (#860) ## What's Changed Do not specify default value when getting the `arrow.struct.conflict.policy` property Closes #859 --- .../arrow/vector/complex/AbstractStructVector.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/AbstractStructVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/AbstractStructVector.java index 2921e43cb6..a57fbe473f 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/AbstractStructVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/AbstractStructVector.java @@ -46,11 +46,13 @@ public abstract class AbstractStructVector extends AbstractContainerVector { private ConflictPolicy conflictPolicy; static { - String conflictPolicyStr = - System.getProperty(STRUCT_CONFLICT_POLICY_JVM, ConflictPolicy.CONFLICT_REPLACE.toString()); + String conflictPolicyStr = System.getProperty(STRUCT_CONFLICT_POLICY_JVM); if (conflictPolicyStr == null) { conflictPolicyStr = System.getenv(STRUCT_CONFLICT_POLICY_ENV); } + if (conflictPolicyStr == null) { + conflictPolicyStr = ConflictPolicy.CONFLICT_REPLACE.toString(); + } ConflictPolicy conflictPolicy; try { conflictPolicy = ConflictPolicy.valueOf(conflictPolicyStr.toUpperCase(Locale.ROOT)); @@ -62,11 +64,11 @@ public abstract class AbstractStructVector extends AbstractContainerVector { /** Policy to determine how to react when duplicate columns are encountered. */ public enum ConflictPolicy { - // Ignore the conflict and append the field. This is the default behaviour + // Ignore the conflict and append the field. CONFLICT_APPEND, // Keep the existing field and ignore the newer one. CONFLICT_IGNORE, - // Replace the existing field with the newer one. + // Replace the existing field with the newer one. This is the default behaviour CONFLICT_REPLACE, // Refuse the new field and error out. CONFLICT_ERROR From 34060eb491a870f5ede5d30e007060b8310dc64f Mon Sep 17 00:00:00 2001 From: Ivan Chesnov Date: Wed, 1 Oct 2025 15:43:26 +0300 Subject: [PATCH 065/271] GH-836: Added support of ExtensionType for ComplexCopier (#837) ## What's Changed Updated ComplexCopier to support ExtensionType - it contains two **copy** methods ``` public static void copy(FieldReader input, FieldWriter output) //for not breaking existing logic public static void copy(FieldReader input, FieldWriter output, ExtensionTypeWriterFactory extensionTypeWriterFactory) ``` Also updated ComplexCopier tests. Closes #836. --- .../src/main/codegen/includes/vv_imports.ftl | 1 + .../templates/AbstractFieldReader.java | 4 + .../main/codegen/templates/BaseReader.java | 3 + .../main/codegen/templates/ComplexCopier.java | 39 +++++- .../main/codegen/templates/NullReader.java | 1 + .../apache/arrow/vector/BaseValueVector.java | 13 ++ .../org/apache/arrow/vector/NullVector.java | 13 ++ .../org/apache/arrow/vector/ValueVector.java | 25 ++++ .../complex/AbstractContainerVector.java | 13 ++ .../arrow/vector/complex/LargeListVector.java | 33 ++++- .../vector/complex/LargeListViewVector.java | 15 +++ .../arrow/vector/complex/ListVector.java | 33 ++++- .../arrow/vector/complex/ListViewVector.java | 15 ++- .../complex/impl/AbstractBaseReader.java | 10 ++ .../complex/impl/UnionExtensionWriter.java | 5 + .../complex/impl/UnionLargeListReader.java | 4 + .../apache/arrow/vector/TestListVector.java | 43 +++++++ .../apache/arrow/vector/TestMapVector.java | 96 +++++++++++++++ .../complex/impl/TestComplexCopier.java | 114 ++++++++++++++++++ .../vector/complex/impl/UuidReaderImpl.java | 5 + 20 files changed, 476 insertions(+), 9 deletions(-) diff --git a/vector/src/main/codegen/includes/vv_imports.ftl b/vector/src/main/codegen/includes/vv_imports.ftl index 7f216a7b43..2bbcecc856 100644 --- a/vector/src/main/codegen/includes/vv_imports.ftl +++ b/vector/src/main/codegen/includes/vv_imports.ftl @@ -34,6 +34,7 @@ import org.apache.arrow.vector.complex.*; import org.apache.arrow.vector.complex.reader.*; import org.apache.arrow.vector.complex.impl.*; import org.apache.arrow.vector.complex.writer.*; +import org.apache.arrow.vector.complex.writer.BaseWriter.ExtensionWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.StructWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; diff --git a/vector/src/main/codegen/templates/AbstractFieldReader.java b/vector/src/main/codegen/templates/AbstractFieldReader.java index 7e84323b64..c7c5b4d78d 100644 --- a/vector/src/main/codegen/templates/AbstractFieldReader.java +++ b/vector/src/main/codegen/templates/AbstractFieldReader.java @@ -109,6 +109,10 @@ public void copyAsField(String name, ${name}Writer writer) { + public void copyAsValue(StructWriter writer, ExtensionTypeWriterFactory writerFactory) { + fail("CopyAsValue StructWriter"); + } + public void read(ExtensionHolder holder) { fail("Extension"); } diff --git a/vector/src/main/codegen/templates/BaseReader.java b/vector/src/main/codegen/templates/BaseReader.java index c52345af21..4c6f49ab9b 100644 --- a/vector/src/main/codegen/templates/BaseReader.java +++ b/vector/src/main/codegen/templates/BaseReader.java @@ -49,6 +49,7 @@ public interface RepeatedStructReader extends StructReader{ boolean next(); int size(); void copyAsValue(StructWriter writer); + void copyAsValue(StructWriter writer, ExtensionTypeWriterFactory writerFactory); } public interface ListReader extends BaseReader{ @@ -59,6 +60,7 @@ public interface RepeatedListReader extends ListReader{ boolean next(); int size(); void copyAsValue(ListWriter writer); + void copyAsValue(ListWriter writer, ExtensionTypeWriterFactory writerFactory); } public interface MapReader extends BaseReader{ @@ -69,6 +71,7 @@ public interface RepeatedMapReader extends MapReader{ boolean next(); int size(); void copyAsValue(MapWriter writer); + void copyAsValue(MapWriter writer, ExtensionTypeWriterFactory writerFactory); } public interface ScalarReader extends diff --git a/vector/src/main/codegen/templates/ComplexCopier.java b/vector/src/main/codegen/templates/ComplexCopier.java index 4fff7059a7..4df5478f48 100644 --- a/vector/src/main/codegen/templates/ComplexCopier.java +++ b/vector/src/main/codegen/templates/ComplexCopier.java @@ -42,10 +42,14 @@ public class ComplexCopier { * @param output field to write to */ public static void copy(FieldReader input, FieldWriter output) { - writeValue(input, output); + writeValue(input, output, null); } - private static void writeValue(FieldReader reader, FieldWriter writer) { + public static void copy(FieldReader input, FieldWriter output, ExtensionTypeWriterFactory extensionTypeWriterFactory) { + writeValue(input, output, extensionTypeWriterFactory); + } + + private static void writeValue(FieldReader reader, FieldWriter writer, ExtensionTypeWriterFactory extensionTypeWriterFactory) { final MinorType mt = reader.getMinorType(); switch (mt) { @@ -61,7 +65,7 @@ private static void writeValue(FieldReader reader, FieldWriter writer) { FieldReader childReader = reader.reader(); FieldWriter childWriter = getListWriterForReader(childReader, writer); if (childReader.isSet()) { - writeValue(childReader, childWriter); + writeValue(childReader, childWriter, extensionTypeWriterFactory); } else { childWriter.writeNull(); } @@ -79,8 +83,8 @@ private static void writeValue(FieldReader reader, FieldWriter writer) { FieldReader structReader = reader.reader(); if (structReader.isSet()) { writer.startEntry(); - writeValue(mapReader.key(), getMapWriterForReader(mapReader.key(), writer.key())); - writeValue(mapReader.value(), getMapWriterForReader(mapReader.value(), writer.value())); + writeValue(mapReader.key(), getMapWriterForReader(mapReader.key(), writer.key()), extensionTypeWriterFactory); + writeValue(mapReader.value(), getMapWriterForReader(mapReader.value(), writer.value()), extensionTypeWriterFactory); writer.endEntry(); } else { writer.writeNull(); @@ -99,7 +103,7 @@ private static void writeValue(FieldReader reader, FieldWriter writer) { if (childReader.getMinorType() != Types.MinorType.NULL) { FieldWriter childWriter = getStructWriterForReader(childReader, writer, name); if (childReader.isSet()) { - writeValue(childReader, childWriter); + writeValue(childReader, childWriter, extensionTypeWriterFactory); } else { childWriter.writeNull(); } @@ -110,6 +114,20 @@ private static void writeValue(FieldReader reader, FieldWriter writer) { writer.writeNull(); } break; + case EXTENSIONTYPE: + if (extensionTypeWriterFactory == null) { + throw new IllegalArgumentException("Must provide ExtensionTypeWriterFactory"); + } + if (reader.isSet()) { + Object value = reader.readObject(); + if (value != null) { + writer.addExtensionTypeWriterFactory(extensionTypeWriterFactory); + writer.writeExtension(value); + } + } else { + writer.writeNull(); + } + break; <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> @@ -162,6 +180,9 @@ private static FieldWriter getStructWriterForReader(FieldReader reader, StructWr return (FieldWriter) writer.map(name); case LISTVIEW: return (FieldWriter) writer.listView(name); + case EXTENSIONTYPE: + ExtensionWriter extensionWriter = writer.extension(name, reader.getField().getType()); + return (FieldWriter) extensionWriter; default: throw new UnsupportedOperationException(reader.getMinorType().toString()); } @@ -186,6 +207,9 @@ private static FieldWriter getListWriterForReader(FieldReader reader, ListWriter return (FieldWriter) writer.list(); case LISTVIEW: return (FieldWriter) writer.listView(); + case EXTENSIONTYPE: + ExtensionWriter extensionWriter = writer.extension(reader.getField().getType()); + return (FieldWriter) extensionWriter; default: throw new UnsupportedOperationException(reader.getMinorType().toString()); } @@ -211,6 +235,9 @@ private static FieldWriter getMapWriterForReader(FieldReader reader, MapWriter w return (FieldWriter) writer.listView(); case MAP: return (FieldWriter) writer.map(false); + case EXTENSIONTYPE: + ExtensionWriter extensionWriter = writer.extension(reader.getField().getType()); + return (FieldWriter) extensionWriter; default: throw new UnsupportedOperationException(reader.getMinorType().toString()); } diff --git a/vector/src/main/codegen/templates/NullReader.java b/vector/src/main/codegen/templates/NullReader.java index 88e6ea98ea..0529633478 100644 --- a/vector/src/main/codegen/templates/NullReader.java +++ b/vector/src/main/codegen/templates/NullReader.java @@ -86,6 +86,7 @@ public void read(int arrayIndex, Nullable${name}Holder holder){ } + public void copyAsValue(StructWriter writer, ExtensionTypeWriterFactory writerFactory){} public void read(ExtensionHolder holder) { holder.isSet = 0; } diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java index 37dfa20616..cc57cde29e 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java @@ -22,6 +22,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.ReferenceManager; import org.apache.arrow.util.Preconditions; +import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.util.DataSizeRoundingUtil; import org.apache.arrow.vector.util.TransferPair; @@ -260,6 +261,18 @@ public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { throw new UnsupportedOperationException(); } + @Override + public void copyFrom( + int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { + throw new UnsupportedOperationException(); + } + + @Override + public void copyFromSafe( + int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { + throw new UnsupportedOperationException(); + } + /** * Transfer the validity buffer from `validityBuffer` to the target vector's `validityBuffer`. * Start at `startIndex` and copy `length` number of elements. If the starting index is 8 byte diff --git a/vector/src/main/java/org/apache/arrow/vector/NullVector.java b/vector/src/main/java/org/apache/arrow/vector/NullVector.java index 6bfe540d23..0d6dab2837 100644 --- a/vector/src/main/java/org/apache/arrow/vector/NullVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/NullVector.java @@ -27,6 +27,7 @@ import org.apache.arrow.memory.util.hash.ArrowBufHasher; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.compare.VectorVisitor; +import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.impl.NullReader; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.ipc.message.ArrowFieldNode; @@ -329,6 +330,18 @@ public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { throw new UnsupportedOperationException(); } + @Override + public void copyFrom( + int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { + throw new UnsupportedOperationException(); + } + + @Override + public void copyFromSafe( + int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { + throw new UnsupportedOperationException(); + } + @Override public String getName() { return this.getField().getName(); diff --git a/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/vector/src/main/java/org/apache/arrow/vector/ValueVector.java index 3a5058256c..e0628c2ee1 100644 --- a/vector/src/main/java/org/apache/arrow/vector/ValueVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/ValueVector.java @@ -22,6 +22,7 @@ import org.apache.arrow.memory.OutOfMemoryException; import org.apache.arrow.memory.util.hash.ArrowBufHasher; import org.apache.arrow.vector.compare.VectorVisitor; +import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.Field; @@ -309,6 +310,30 @@ public interface ValueVector extends Closeable, Iterable { */ void copyFromSafe(int fromIndex, int thisIndex, ValueVector from); + /** + * Copy a cell value from a particular index in source vector to a particular position in this + * vector. + * + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + * @param writerFactory the extension type writer factory to use for copying extension type values + */ + void copyFrom( + int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory); + + /** + * Same as {@link #copyFrom(int, int, ValueVector)} except that it handles the case when the + * capacity of the vector needs to be expanded before copy. + * + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + * @param writerFactory the extension type writer factory to use for copying extension type values + */ + void copyFromSafe( + int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory); + /** * Accept a generic {@link VectorVisitor} and return the result. * diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java index a6a71cf1a4..429f9884bb 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java @@ -21,6 +21,7 @@ import org.apache.arrow.vector.DensityAwareVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeList; @@ -151,6 +152,18 @@ public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { throw new UnsupportedOperationException(); } + @Override + public void copyFrom( + int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { + throw new UnsupportedOperationException(); + } + + @Override + public void copyFromSafe( + int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { + throw new UnsupportedOperationException(); + } + @Override public String getName() { return name; diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java index 997b5a8b78..48c8127e23 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java @@ -49,6 +49,7 @@ import org.apache.arrow.vector.ZeroVector; import org.apache.arrow.vector.compare.VectorVisitor; import org.apache.arrow.vector.complex.impl.ComplexCopier; +import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.impl.UnionLargeListReader; import org.apache.arrow.vector.complex.impl.UnionLargeListWriter; import org.apache.arrow.vector.complex.reader.FieldReader; @@ -482,12 +483,42 @@ public void copyFromSafe(int inIndex, int outIndex, ValueVector from) { */ @Override public void copyFrom(int inIndex, int outIndex, ValueVector from) { + copyFrom(inIndex, outIndex, from, null); + } + + /** + * Copy a cell value from a particular index in source vector to a particular position in this + * vector. + * + * @param inIndex position to copy from in source vector + * @param outIndex position to copy to in this vector + * @param from source vector + * @param writerFactory the extension type writer factory to use for copying extension type values + */ + @Override + public void copyFrom( + int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { Preconditions.checkArgument(this.getMinorType() == from.getMinorType()); FieldReader in = from.getReader(); in.setPosition(inIndex); UnionLargeListWriter out = getWriter(); out.setPosition(outIndex); - ComplexCopier.copy(in, out); + ComplexCopier.copy(in, out, writerFactory); + } + + /** + * Same as {@link #copyFrom(int, int, ValueVector)} except that it handles the case when the + * capacity of the vector needs to be expanded before copy. + * + * @param inIndex position to copy from in source vector + * @param outIndex position to copy to in this vector + * @param from source vector + * @param writerFactory the extension type writer factory to use for copying extension type values + */ + @Override + public void copyFromSafe( + int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { + copyFrom(inIndex, outIndex, from, writerFactory); } /** diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java index 2da7eb057e..992a664449 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java @@ -41,6 +41,7 @@ import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.ZeroVector; import org.apache.arrow.vector.compare.VectorVisitor; +import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.impl.UnionLargeListViewReader; import org.apache.arrow.vector.complex.impl.UnionLargeListViewWriter; import org.apache.arrow.vector.complex.impl.UnionListReader; @@ -346,6 +347,20 @@ public void copyFrom(int inIndex, int outIndex, ValueVector from) { "LargeListViewVector does not support copyFrom operation yet."); } + @Override + public void copyFromSafe( + int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { + throw new UnsupportedOperationException( + "LargeListViewVector does not support copyFromSafe operation yet."); + } + + @Override + public void copyFrom( + int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { + throw new UnsupportedOperationException( + "LargeListViewVector does not support copyFrom operation yet."); + } + @Override public FieldVector getDataVector() { return vector; diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 93a313ef4f..89549257c4 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -42,6 +42,7 @@ import org.apache.arrow.vector.ZeroVector; import org.apache.arrow.vector.compare.VectorVisitor; import org.apache.arrow.vector.complex.impl.ComplexCopier; +import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.impl.UnionListReader; import org.apache.arrow.vector.complex.impl.UnionListWriter; import org.apache.arrow.vector.complex.reader.FieldReader; @@ -400,12 +401,42 @@ public void copyFromSafe(int inIndex, int outIndex, ValueVector from) { */ @Override public void copyFrom(int inIndex, int outIndex, ValueVector from) { + copyFrom(inIndex, outIndex, from, null); + } + + /** + * Same as {@link #copyFrom(int, int, ValueVector)} except that it handles the case when the + * capacity of the vector needs to be expanded before copy. + * + * @param inIndex position to copy from in source vector + * @param outIndex position to copy to in this vector + * @param from source vector + * @param writerFactory the extension type writer factory to use for copying extension type values + */ + @Override + public void copyFromSafe( + int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { + copyFrom(inIndex, outIndex, from, writerFactory); + } + + /** + * Copy a cell value from a particular index in source vector to a particular position in this + * vector. + * + * @param inIndex position to copy from in source vector + * @param outIndex position to copy to in this vector + * @param from source vector + * @param writerFactory the extension type writer factory to use for copying extension type values + */ + @Override + public void copyFrom( + int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { Preconditions.checkArgument(this.getMinorType() == from.getMinorType()); FieldReader in = from.getReader(); in.setPosition(inIndex); FieldWriter out = getWriter(); out.setPosition(outIndex); - ComplexCopier.copy(in, out); + ComplexCopier.copy(in, out, writerFactory); } /** diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java index 8711db5e0f..2784240429 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java @@ -42,6 +42,7 @@ import org.apache.arrow.vector.ZeroVector; import org.apache.arrow.vector.compare.VectorVisitor; import org.apache.arrow.vector.complex.impl.ComplexCopier; +import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.impl.UnionListViewReader; import org.apache.arrow.vector.complex.impl.UnionListViewWriter; import org.apache.arrow.vector.complex.reader.FieldReader; @@ -338,6 +339,12 @@ public void copyFromSafe(int inIndex, int outIndex, ValueVector from) { copyFrom(inIndex, outIndex, from); } + @Override + public void copyFromSafe( + int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { + copyFrom(inIndex, outIndex, from, writerFactory); + } + @Override public OUT accept(VectorVisitor visitor, IN value) { return visitor.visit(this, value); @@ -345,12 +352,18 @@ public OUT accept(VectorVisitor visitor, IN value) { @Override public void copyFrom(int inIndex, int outIndex, ValueVector from) { + copyFrom(inIndex, outIndex, from, null); + } + + @Override + public void copyFrom( + int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { Preconditions.checkArgument(this.getMinorType() == from.getMinorType()); FieldReader in = from.getReader(); in.setPosition(inIndex); FieldWriter out = getWriter(); out.setPosition(outIndex); - ComplexCopier.copy(in, out); + ComplexCopier.copy(in, out, writerFactory); } @Override diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java index b2e95663f7..bf074ecb90 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java @@ -115,4 +115,14 @@ public void copyAsValue(ListWriter writer) { public void copyAsValue(MapWriter writer) { ComplexCopier.copy(this, (FieldWriter) writer); } + + @Override + public void copyAsValue(ListWriter writer, ExtensionTypeWriterFactory writerFactory) { + ComplexCopier.copy(this, (FieldWriter) writer, writerFactory); + } + + @Override + public void copyAsValue(MapWriter writer, ExtensionTypeWriterFactory writerFactory) { + ComplexCopier.copy(this, (FieldWriter) writer, writerFactory); + } } diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionExtensionWriter.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionExtensionWriter.java index d341384bd9..4219069cba 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionExtensionWriter.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionExtensionWriter.java @@ -76,4 +76,9 @@ public void setPosition(int index) { this.writer.setPosition(index); } } + + @Override + public void writeNull() { + this.writer.writeNull(); + } } diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionLargeListReader.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionLargeListReader.java index be236c3166..a9104cb0d2 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionLargeListReader.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionLargeListReader.java @@ -105,4 +105,8 @@ public boolean next() { public void copyAsValue(UnionLargeListWriter writer) { ComplexCopier.copy(this, (FieldWriter) writer); } + + public void copyAsValue(UnionLargeListWriter writer, ExtensionTypeWriterFactory writerFactory) { + ComplexCopier.copy(this, (FieldWriter) writer, writerFactory); + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index d58b7cc941..c6c7c5c862 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -1271,6 +1271,49 @@ public void testListVectorReaderForExtensionType() throws Exception { } } + @Test + public void testCopyFromForExtensionType() throws Exception { + try (ListVector inVector = ListVector.empty("input", allocator); + ListVector outVector = ListVector.empty("output", allocator)) { + UnionListWriter writer = inVector.getWriter(); + writer.allocate(); + writer.setPosition(0); + UUID u1 = UUID.randomUUID(); + UUID u2 = UUID.randomUUID(); + writer.startList(); + ExtensionWriter extensionWriter = writer.extension(new UuidType()); + extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); + extensionWriter.writeExtension(u1); + extensionWriter.writeExtension(u2); + extensionWriter.writeNull(); + writer.endList(); + + writer.setValueCount(1); + + // copy values from input to output + outVector.allocateNew(); + outVector.copyFrom(0, 0, inVector, new UuidWriterFactory()); + outVector.setValueCount(1); + + UnionListReader reader = outVector.getReader(); + assertTrue(reader.isSet(), "shouldn't be null"); + reader.setPosition(0); + reader.next(); + FieldReader uuidReader = reader.reader(); + UuidHolder holder = new UuidHolder(); + uuidReader.read(holder); + ByteBuffer bb = ByteBuffer.wrap(holder.value); + UUID actualUuid = new UUID(bb.getLong(), bb.getLong()); + assertEquals(u1, actualUuid); + reader.next(); + uuidReader = reader.reader(); + uuidReader.read(holder); + bb = ByteBuffer.wrap(holder.value); + actualUuid = new UUID(bb.getLong(), bb.getLong()); + assertEquals(u2, actualUuid); + } + } + private void writeIntValues(UnionListWriter writer, int[] values) { writer.startList(); for (int v : values) { diff --git a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java index 313d83ec91..1a1810d0f7 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java @@ -22,24 +22,30 @@ import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.UUID; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.complex.impl.UnionMapReader; import org.apache.arrow.vector.complex.impl.UnionMapWriter; +import org.apache.arrow.vector.complex.impl.UuidWriterFactory; import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.complex.writer.BaseWriter.ExtensionWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; import org.apache.arrow.vector.complex.writer.FieldWriter; +import org.apache.arrow.vector.holder.UuidHolder; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.UuidType; import org.apache.arrow.vector.util.JsonStringArrayList; import org.apache.arrow.vector.util.TransferPair; import org.junit.jupiter.api.AfterEach; @@ -1263,4 +1269,94 @@ public void testMapTypeReturnsSupportedMapWriter() { assertEquals(11, getResultValue(resultStruct)); } } + + @Test + public void testMapVectorWithExtensionType() throws Exception { + try (final MapVector inVector = MapVector.empty("map", allocator, false)) { + inVector.allocateNew(); + UnionMapWriter writer = inVector.getWriter(); + writer.setPosition(0); + UUID u1 = UUID.randomUUID(); + UUID u2 = UUID.randomUUID(); + writer.startMap(); + writer.startEntry(); + writer.key().bigInt().writeBigInt(0); + ExtensionWriter extensionWriter = writer.value().extension(new UuidType()); + extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); + extensionWriter.writeExtension(u1); + writer.endEntry(); + writer.startEntry(); + writer.key().bigInt().writeBigInt(1); + extensionWriter = writer.value().extension(new UuidType()); + extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); + extensionWriter.writeExtension(u2); + writer.endEntry(); + writer.endMap(); + + writer.setValueCount(1); + + UnionMapReader mapReader = inVector.getReader(); + mapReader.setPosition(0); + mapReader.next(); + FieldReader uuidReader = mapReader.value(); + UuidHolder holder = new UuidHolder(); + uuidReader.read(holder); + ByteBuffer bb = ByteBuffer.wrap(holder.value); + UUID actualUuid = new UUID(bb.getLong(), bb.getLong()); + assertEquals(u1, actualUuid); + mapReader.next(); + uuidReader = mapReader.value(); + uuidReader.read(holder); + bb = ByteBuffer.wrap(holder.value); + actualUuid = new UUID(bb.getLong(), bb.getLong()); + assertEquals(u2, actualUuid); + } + } + + @Test + public void testCopyFromForExtensionType() throws Exception { + try (final MapVector inVector = MapVector.empty("in", allocator, false); + final MapVector outVector = MapVector.empty("out", allocator, false)) { + inVector.allocateNew(); + UnionMapWriter writer = inVector.getWriter(); + writer.setPosition(0); + UUID u1 = UUID.randomUUID(); + UUID u2 = UUID.randomUUID(); + writer.startMap(); + writer.startEntry(); + writer.key().bigInt().writeBigInt(0); + ExtensionWriter extensionWriter = writer.value().extension(new UuidType()); + extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); + extensionWriter.writeExtension(u1); + writer.endEntry(); + writer.startEntry(); + writer.key().bigInt().writeBigInt(1); + extensionWriter = writer.value().extension(new UuidType()); + extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); + extensionWriter.writeExtension(u2); + writer.endEntry(); + writer.endMap(); + + writer.setValueCount(1); + outVector.allocateNew(); + outVector.copyFrom(0, 0, inVector, new UuidWriterFactory()); + outVector.setValueCount(1); + + UnionMapReader mapReader = outVector.getReader(); + mapReader.setPosition(0); + mapReader.next(); + FieldReader uuidReader = mapReader.value(); + UuidHolder holder = new UuidHolder(); + uuidReader.read(holder); + ByteBuffer bb = ByteBuffer.wrap(holder.value); + UUID actualUuid = new UUID(bb.getLong(), bb.getLong()); + assertEquals(u1, actualUuid); + mapReader.next(); + uuidReader = mapReader.value(); + uuidReader.read(holder); + bb = ByteBuffer.wrap(holder.value); + actualUuid = new UUID(bb.getLong(), bb.getLong()); + assertEquals(u2, actualUuid); + } + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java index 3bc02c6029..738e8905e3 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java @@ -20,6 +20,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.math.BigDecimal; +import java.util.UUID; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.DecimalVector; @@ -30,12 +31,14 @@ import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.complex.writer.BaseWriter; +import org.apache.arrow.vector.complex.writer.BaseWriter.ExtensionWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.StructWriter; import org.apache.arrow.vector.complex.writer.FieldWriter; import org.apache.arrow.vector.holders.DecimalHolder; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.UuidType; import org.apache.arrow.vector.util.DecimalUtility; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -845,4 +848,115 @@ public void testCopyMapVectorWithMapValue() { assertTrue(VectorEqualsVisitor.vectorEquals(from, to)); } } + + @Test + public void testCopyListVectorWithExtensionType() { + try (ListVector from = ListVector.empty("v", allocator); + ListVector to = ListVector.empty("v", allocator)) { + + UnionListWriter listWriter = from.getWriter(); + listWriter.allocate(); + + for (int i = 0; i < COUNT; i++) { + listWriter.setPosition(i); + listWriter.startList(); + ExtensionWriter extensionWriter = listWriter.extension(new UuidType()); + extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); + extensionWriter.writeExtension(UUID.randomUUID()); + extensionWriter.writeExtension(UUID.randomUUID()); + listWriter.endList(); + } + from.setValueCount(COUNT); + + // copy values + FieldReader in = from.getReader(); + FieldWriter out = to.getWriter(); + for (int i = 0; i < COUNT; i++) { + in.setPosition(i); + out.setPosition(i); + ComplexCopier.copy(in, out, new UuidWriterFactory()); + } + + to.setValueCount(COUNT); + + // validate equals + assertTrue(VectorEqualsVisitor.vectorEquals(from, to)); + } + } + + @Test + public void testCopyMapVectorWithExtensionType() { + try (final MapVector from = MapVector.empty("v", allocator, false); + final MapVector to = MapVector.empty("v", allocator, false)) { + + from.allocateNew(); + + UnionMapWriter mapWriter = from.getWriter(); + for (int i = 0; i < COUNT; i++) { + mapWriter.setPosition(i); + mapWriter.startMap(); + mapWriter.startEntry(); + ExtensionWriter extensionKeyWriter = mapWriter.key().extension(new UuidType()); + extensionKeyWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); + extensionKeyWriter.writeExtension(UUID.randomUUID()); + ExtensionWriter extensionValueWriter = mapWriter.value().extension(new UuidType()); + extensionValueWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); + extensionValueWriter.writeExtension(UUID.randomUUID()); + mapWriter.endEntry(); + mapWriter.endMap(); + } + + from.setValueCount(COUNT); + + // copy values + FieldReader in = from.getReader(); + FieldWriter out = to.getWriter(); + for (int i = 0; i < COUNT; i++) { + in.setPosition(i); + out.setPosition(i); + ComplexCopier.copy(in, out, new UuidWriterFactory()); + } + to.setValueCount(COUNT); + + // validate equals + assertTrue(VectorEqualsVisitor.vectorEquals(from, to)); + } + } + + @Test + public void testCopyStructVectorWithExtensionType() { + try (final StructVector from = StructVector.empty("v", allocator); + final StructVector to = StructVector.empty("v", allocator)) { + + from.allocateNewSafe(); + + NullableStructWriter structWriter = from.getWriter(); + for (int i = 0; i < COUNT; i++) { + structWriter.setPosition(i); + structWriter.start(); + ExtensionWriter extensionWriter1 = structWriter.extension("timestamp1", new UuidType()); + extensionWriter1.addExtensionTypeWriterFactory(new UuidWriterFactory()); + extensionWriter1.writeExtension(UUID.randomUUID()); + ExtensionWriter extensionWriter2 = structWriter.extension("timestamp2", new UuidType()); + extensionWriter2.addExtensionTypeWriterFactory(new UuidWriterFactory()); + extensionWriter2.writeExtension(UUID.randomUUID()); + structWriter.end(); + } + + from.setValueCount(COUNT); + + // copy values + FieldReader in = from.getReader(); + FieldWriter out = to.getWriter(); + for (int i = 0; i < COUNT; i++) { + in.setPosition(i); + out.setPosition(i); + ComplexCopier.copy(in, out, new UuidWriterFactory()); + } + to.setValueCount(COUNT); + + // validate equals + assertTrue(VectorEqualsVisitor.vectorEquals(from, to)); + } + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java index 16dd734de8..6b98d3b340 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java @@ -61,4 +61,9 @@ public void copyAsValue(AbstractExtensionTypeWriter writer) { UuidWriterImpl impl = (UuidWriterImpl) writer; impl.vector.copyFromSafe(idx(), impl.idx(), vector); } + + @Override + public Object readObject() { + return vector.getObject(idx()); + } } From 9cfa6ffc314e148afd196a89f2d5b12ec35c1bc2 Mon Sep 17 00:00:00 2001 From: XenoAmess Date: Fri, 3 Oct 2025 16:06:40 +0800 Subject: [PATCH 066/271] GH-848: TypedValue should be treated as Nullable in bind function in AvaticaParameterBinder (#849) ## What's Changed Closes #848. --- .../arrow/driver/jdbc/utils/AvaticaParameterBinder.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java index 4c2a9b865f..0fd99de539 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java @@ -44,6 +44,7 @@ import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.calcite.avatica.remote.TypedValue; +import org.checkerframework.checker.nullness.qual.Nullable; /** * Convert Avatica PreparedStatement parameters from a list of TypedValue to Arrow and bind them to @@ -108,9 +109,9 @@ public void bind(List typedValues, int index) { * @param typedValue TypedValue to bind to the vector. * @param index Vector index to bind the value at. */ - private void bind(FieldVector vector, TypedValue typedValue, int index) { + private void bind(FieldVector vector, @Nullable TypedValue typedValue, int index) { try { - if (typedValue.value == null) { + if (typedValue == null || typedValue.value == null) { if (vector.getField().isNullable()) { vector.setNull(index); } else { @@ -127,7 +128,7 @@ private void bind(FieldVector vector, TypedValue typedValue, int index) { throw new UnsupportedOperationException( String.format( "Binding value of type %s is not yet supported for expected Arrow type %s", - typedValue.type, vector.getField().getType())); + typedValue == null ? "null" : typedValue.type, vector.getField().getType())); } } From d529557087c601ccbb18d704081f8711f0df0cfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Milenkovi=C4=87?= Date: Fri, 3 Oct 2025 09:13:20 +0100 Subject: [PATCH 067/271] fix: issue with class names in arrow-c jni calls (#867) ## What's Changed Arrow C JNI code used class names which are not according to specification, making code unusable in latest graalvm 25. This PR changes class names according to JNI specification. More details at #866 Closes #866 --- c/src/main/cpp/jni_wrapper.cc | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/c/src/main/cpp/jni_wrapper.cc b/c/src/main/cpp/jni_wrapper.cc index 436cbdc806..3d7a194563 100644 --- a/c/src/main/cpp/jni_wrapper.cc +++ b/c/src/main/cpp/jni_wrapper.cc @@ -327,19 +327,20 @@ void ArrowArrayStreamRelease(ArrowArrayStream* stream) { jint JNI_OnLoad(JavaVM* vm, void* reserved) { JNIEnv* env; - if (vm->GetEnv(reinterpret_cast(&env), JNI_VERSION) != JNI_OK) { - return JNI_ERR; + const int err_code = vm->GetEnv(reinterpret_cast(&env), JNI_VERSION); + if (err_code != JNI_OK) { + return err_code; } JNI_METHOD_START - kObjectClass = CreateGlobalClassReference(env, "Ljava/lang/Object;"); + kObjectClass = CreateGlobalClassReference(env, "java/lang/Object"); kRuntimeExceptionClass = - CreateGlobalClassReference(env, "Ljava/lang/RuntimeException;"); + CreateGlobalClassReference(env, "java/lang/RuntimeException"); kPrivateDataClass = - CreateGlobalClassReference(env, "Lorg/apache/arrow/c/jni/PrivateData;"); + CreateGlobalClassReference(env, "org/apache/arrow/c/jni/PrivateData"); kCDataExceptionClass = - CreateGlobalClassReference(env, "Lorg/apache/arrow/c/jni/CDataJniException;"); + CreateGlobalClassReference(env, "org/apache/arrow/c/jni/CDataJniException"); kStreamPrivateDataClass = CreateGlobalClassReference( - env, "Lorg/apache/arrow/c/ArrayStreamExporter$ExportedArrayStreamPrivateData;"); + env, "org/apache/arrow/c/ArrayStreamExporter$ExportedArrayStreamPrivateData"); kPrivateDataLastErrorField = GetFieldID(env, kStreamPrivateDataClass, "lastError", "[B"); From d31d9b0cac6de22ba6649c38486281ffe6346294 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Fern=C3=A1ndez=20Giraldo?= Date: Mon, 6 Oct 2025 19:56:35 -0600 Subject: [PATCH 068/271] GH-839: Fix support for ResultSet.getObject for TIMESTAMP_WITH_TIMEZONE (#840) ## What's Changed Turns out AvaticaSite.get does not account for TIMESTAMP_WITH_TIMEZONE types so we add support on an override function. Closes #818. Closes #839. --- ...owFlightJdbcVectorSchemaRootResultSet.java | 31 ++++ .../jdbc/FlightServerTestExtension.java | 6 + .../driver/jdbc/TimestampResultSetTest.java | 164 ++++++++++++++++++ 3 files changed, 201 insertions(+) create mode 100644 flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/TimestampResultSetTest.java diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java index 0dc2b07c97..622e5fe7f6 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java @@ -19,6 +19,7 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; +import java.sql.Types; import java.util.HashSet; import java.util.List; import java.util.Objects; @@ -28,14 +29,17 @@ import org.apache.arrow.util.AutoCloseables; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.calcite.avatica.AvaticaConnection; import org.apache.calcite.avatica.AvaticaResultSet; import org.apache.calcite.avatica.AvaticaResultSetMetaData; +import org.apache.calcite.avatica.AvaticaSite; import org.apache.calcite.avatica.AvaticaStatement; import org.apache.calcite.avatica.ColumnMetaData; import org.apache.calcite.avatica.Meta; import org.apache.calcite.avatica.Meta.Frame; import org.apache.calcite.avatica.Meta.Signature; import org.apache.calcite.avatica.QueryState; +import org.apache.calcite.avatica.util.Cursor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -102,6 +106,33 @@ void populateData(final VectorSchemaRoot vectorSchemaRoot, final Schema schema) execute2(new ArrowFlightJdbcCursor(vectorSchemaRoot), this.signature.columns); } + /** + * The default method in AvaticaResultSet does not properly handle TIMESTASMP_WITH_TIMEZONE, so we + * override here to add support. + * + * @param columnIndex the first column is 1, the second is 2, ... + * @return Object + * @throws SQLException if there is an underlying exception + */ + @Override + public Object getObject(int columnIndex) throws SQLException { + this.checkOpen(); + + Cursor.Accessor accessor; + try { + accessor = accessorList.get(columnIndex - 1); + } catch (IndexOutOfBoundsException e) { + throw AvaticaConnection.HELPER.createException("invalid column ordinal: " + columnIndex); + } + + ColumnMetaData metaData = columnMetaDataList.get(columnIndex - 1); + if (metaData.type.id == Types.TIMESTAMP_WITH_TIMEZONE) { + return accessor.getTimestamp(localCalendar); + } else { + return AvaticaSite.get(accessor, metaData.type.id, localCalendar); + } + } + @Override protected void cancel() { signature.columns.clear(); diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/FlightServerTestExtension.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/FlightServerTestExtension.java index db0438059f..f71114e1b5 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/FlightServerTestExtension.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/FlightServerTestExtension.java @@ -130,6 +130,12 @@ public Connection getConnection(boolean useEncryption) throws SQLException { return this.createDataSource().getConnection(); } + public Connection getConnection(String timezone) throws SQLException { + setUseEncryption(false); + properties.put("timezone", timezone); + return this.createDataSource().getConnection(); + } + private void setUseEncryption(boolean useEncryption) { properties.put("useEncryption", useEncryption); } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/TimestampResultSetTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/TimestampResultSetTest.java new file mode 100644 index 0000000000..0921ae2d38 --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/TimestampResultSetTest.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc; + +import com.google.common.collect.ImmutableList; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.Calendar; +import java.util.Collections; +import java.util.TimeZone; +import org.apache.arrow.driver.jdbc.utils.MockFlightSqlProducer; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.TimeStampVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * Timestamps have a lot of nuances in JDBC. This class is here to test that timestamp behavior is + * correct for different types of Timestamp vectors as well as different methods of retrieving the + * timestamps in JDBC. + */ +public class TimestampResultSetTest { + private static final MockFlightSqlProducer FLIGHT_SQL_PRODUCER = new MockFlightSqlProducer(); + + @RegisterExtension public static FlightServerTestExtension FLIGHT_SERVER_TEST_EXTENSION; + + static { + FLIGHT_SERVER_TEST_EXTENSION = + FlightServerTestExtension.createStandardTestExtension(FLIGHT_SQL_PRODUCER); + } + + private static final String QUERY_STRING = "SELECT * FROM TIMESTAMPS"; + private static final Schema QUERY_SCHEMA = + new Schema( + ImmutableList.of( + Field.nullable("no_tz", new ArrowType.Timestamp(TimeUnit.MILLISECOND, null)), + Field.nullable("utc", new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC")), + Field.nullable("utc+1", new ArrowType.Timestamp(TimeUnit.MILLISECOND, "GMT+1")), + Field.nullable("utc-1", new ArrowType.Timestamp(TimeUnit.MILLISECOND, "GMT-1")))); + + @BeforeAll + public static void setup() throws SQLException { + Instant firstDay2025 = OffsetDateTime.of(2025, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant(); + + FLIGHT_SQL_PRODUCER.addSelectQuery( + QUERY_STRING, + QUERY_SCHEMA, + Collections.singletonList( + listener -> { + try (final BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + final VectorSchemaRoot root = VectorSchemaRoot.create(QUERY_SCHEMA, allocator)) { + listener.start(root); + root.getFieldVectors() + .forEach(v -> ((TimeStampVector) v).setSafe(0, firstDay2025.toEpochMilli())); + root.setRowCount(1); + listener.putNext(); + } catch (final Throwable throwable) { + listener.error(throwable); + } finally { + listener.completed(); + } + })); + } + + /** + * This test doesn't yet test anything other than ensuring all ResultSet methods to retrieve a + * timestamp succeed. + * + *

This is a good starting point to add more tests to ensure the values are correct when we + * change the "local calendar" either through changing the JVM default or through the connection + * property. + */ + @Test + public void test() { + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + try (Connection connection = FLIGHT_SERVER_TEST_EXTENSION.getConnection("UTC")) { + try (PreparedStatement s = connection.prepareStatement(QUERY_STRING)) { + try (ResultSet rs = s.executeQuery()) { + int numCols = rs.getMetaData().getColumnCount(); + try { + rs.next(); + for (int i = 1; i <= numCols; i++) { + int type = rs.getMetaData().getColumnType(i); + String name = rs.getMetaData().getColumnName(i); + System.out.println(name); + System.out.print("- getDate:\t\t\t\t\t\t\t"); + System.out.print(rs.getDate(i)); + System.out.println(); + System.out.print("- getTimestamp:\t\t\t\t\t\t"); + System.out.print(rs.getTimestamp(i)); + System.out.println(); + System.out.print("- getString:\t\t\t\t\t\t"); + System.out.print(rs.getString(i)); + System.out.println(); + System.out.print("- getObject:\t\t\t\t\t\t"); + System.out.print(rs.getObject(i)); + System.out.println(); + System.out.print("- getObject(Timestamp.class):\t\t"); + System.out.print(rs.getObject(i, Timestamp.class)); + System.out.println(); + System.out.print("- getTimestamp(default Calendar):\t"); + System.out.print(rs.getTimestamp(i, Calendar.getInstance())); + System.out.println(); + System.out.print("- getTimestamp(UTC Calendar):\t\t"); + System.out.print( + rs.getTimestamp(i, Calendar.getInstance(TimeZone.getTimeZone("UTC")))); + System.out.println(); + System.out.print("- getObject(LocalDateTime.class):\t"); + System.out.print(rs.getObject(i, LocalDateTime.class)); + System.out.println(); + if (type == Types.TIMESTAMP_WITH_TIMEZONE) { + System.out.print("- getObject(Instant.class):\t\t\t"); + System.out.print(rs.getObject(i, Instant.class)); + System.out.println(); + System.out.print("- getObject(OffsetDateTime.class):\t"); + System.out.print(rs.getObject(i, OffsetDateTime.class)); + System.out.println(); + System.out.print("- getObject(ZonedDateTime.class):\t"); + System.out.print(rs.getObject(i, ZonedDateTime.class)); + System.out.println(); + } + System.out.println(); + } + System.out.println(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + } +} From 1e533821c19516a19d7c6dec58c657735816c0e5 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 8 Oct 2025 12:49:54 +0900 Subject: [PATCH 069/271] GH-880: [CI] Fix syntax error in `dev_pr.yml` (#881) ## What's Changed We can't write `permissions` in a step. Closes #880. --- .github/workflows/dev_pr.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml index 58f5f3d30a..2f4a48d572 100644 --- a/.github/workflows/dev_pr.yml +++ b/.github/workflows/dev_pr.yml @@ -35,6 +35,7 @@ concurrency: permissions: contents: read + issues: write pull-requests: write jobs: @@ -80,9 +81,5 @@ jobs: if: '! github.event.pull_request.draft' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - permissions: - contents: read - issues: write - pull-requests: write run: | ./.github/workflows/dev_pr_milestone.sh "${GITHUB_REPOSITORY}" ${{ github.event.number }} From 37e9ba6b52d89c14cec5ed706f96d7986551eb8c Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 8 Oct 2025 12:51:17 +0900 Subject: [PATCH 070/271] GH-592: [Release] Use relative path in .sha* (#879) ## What's Changed If we use absolute path, users need to create the same directory structure. It's inconvenient. We should use relative path. Closes #592. --- ci/scripts/jni_full_build.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/scripts/jni_full_build.sh b/ci/scripts/jni_full_build.sh index e9ad0ddbda..5d0aee0555 100755 --- a/ci/scripts/jni_full_build.sh +++ b/ci/scripts/jni_full_build.sh @@ -97,8 +97,10 @@ find ~/.m2/repository/org/apache/arrow \ -exec echo "{}" ";" \ -exec cp "{}" "${dist_dir}" ";" -for artifact in "${dist_dir}"/*; do +pushd "${dist_dir}" +for artifact in *; do sha256sum "${artifact}" >"${artifact}.sha256" sha512sum "${artifact}" >"${artifact}.sha512" done +popd github_actions_group_end From aee8a1070132ada6a9cd5fde1efe1af06c2b72e0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Oct 2025 13:42:16 +0900 Subject: [PATCH 071/271] MINOR: [CI] Bump docker/login-action from 3.5.0 to 3.6.0 (#870) Bumps [docker/login-action](https://github.com/docker/login-action) from 3.5.0 to 3.6.0. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index eec3344097..60fa12f528 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -133,7 +133,7 @@ jobs: with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing - - uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 + - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: registry: ghcr.io username: ${{ github.actor }} From f71a02c043a311a17059df7420673bf538384826 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Mon, 27 Oct 2025 10:30:03 +0100 Subject: [PATCH 072/271] GH-898: Upgrade to Apache POM 35 and identify fixes needed to have CI happy (#865) Closes #898. --------- Co-authored-by: Sutou Kouhei Co-authored-by: Sutou Kouhei Co-authored-by: David Li --- .env | 5 +- .github/workflows/rc.yml | 26 +++++---- ci/docker/vcpkg-jni.dockerfile | 16 +---- ci/scripts/jni_build.sh | 2 +- ci/scripts/jni_macos_build.sh | 67 ++++----------------- ci/scripts/jni_manylinux_build.sh | 97 ++++--------------------------- ci/scripts/jni_windows_build.sh | 2 +- compose.yaml | 2 +- pom.xml | 5 +- 9 files changed, 48 insertions(+), 174 deletions(-) diff --git a/.env b/.env index d3e1c1d63a..a7783537d0 100644 --- a/.env +++ b/.env @@ -40,7 +40,7 @@ ARCH_SHORT=amd64 # Default repository to pull and push images from REPO=ghcr.io/apache/arrow-java-dev -ARROW_REPO=apache/arrow-dev +ARROW_REPO=ghcr.io/apache/arrow-dev # The setup attempts to generate coredumps by default, in order to disable the # coredump generation set it to 0 @@ -53,5 +53,4 @@ MAVEN=3.9.9 # Versions for various dependencies used to build artifacts # Keep in sync with apache/arrow ARROW_REPO_ROOT=./arrow -PYTHON=3.9 -VCPKG="f7423ee180c4b7f40d43402c2feb3859161ef625" # 2024.06.15 Release +VCPKG="4334d8b4c8916018600212ab4dd4bbdc343065d1" # 2025.09.17 Release diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 60fa12f528..f71f8aeed1 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -168,7 +168,7 @@ jobs: fail-fast: false matrix: platform: - - { runs_on: macos-13, arch: "x86_64"} + - { runs_on: macos-15-intel, arch: "x86_64"} - { runs_on: macos-14, arch: "aarch_64" } env: MACOSX_DEPLOYMENT_TARGET: "14.0" @@ -222,7 +222,7 @@ jobs: brew uninstall llvm || : # We can remove this when we drop support for - # macos-13. because macos-14 or later uses /opt/homebrew/ + # macos-15-intel. because macos-14 or later with arm64 uses /opt/homebrew/ # not /usr/local/. # # Ensure updating python@XXX with the "--overwrite" option. @@ -298,7 +298,7 @@ jobs: fail-fast: false matrix: platform: - - runs_on: windows-2019 + - runs_on: windows-2022 arch: "x86_64" steps: - name: Download source archive @@ -309,13 +309,19 @@ jobs: shell: bash run: | tar -xf apache-arrow-java-*.tar.gz --strip-components=1 - - name: Download the latest Apache Arrow C++ - if: github.event_name != 'schedule' - shell: bash - run: | - ci/scripts/download_cpp.sh + # We always use the main branch for apache/arrow for now. + # Because we want to use + # https://github.com/apache/arrow/pull/47749 in + # apache/arrow-java. We can revert this workaround once Apache + # Arrow 22.0.0 that includes the change released. + # + # - name: Download the latest Apache Arrow C++ + # if: github.event_name != 'schedule' + # shell: bash + # run: | + # ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ - if: github.event_name == 'schedule' + # if: github.event_name == 'schedule' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: apache/arrow @@ -354,7 +360,7 @@ jobs: - name: Build shell: cmd run: | - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 REM For ORC set TZDIR=/c/msys64/usr/share/zoneinfo bash -c "ci/scripts/jni_windows_build.sh . arrow build jni" diff --git a/ci/docker/vcpkg-jni.dockerfile b/ci/docker/vcpkg-jni.dockerfile index 55fa35e0d1..d6bd322a39 100644 --- a/ci/docker/vcpkg-jni.dockerfile +++ b/ci/docker/vcpkg-jni.dockerfile @@ -18,24 +18,10 @@ ARG base FROM ${base} -# Install the libraries required by Gandiva to run -# Use enable llvm[enable-rtti] in the vcpkg.json to avoid link problems in Gandiva -RUN vcpkg install \ - --clean-after-build \ - --x-install-root=${VCPKG_ROOT}/installed \ - --x-manifest-root=/arrow/ci/vcpkg \ - --x-feature=dev \ - --x-feature=flight \ - --x-feature=gcs \ - --x-feature=json \ - --x-feature=parquet \ - --x-feature=gandiva \ - --x-feature=s3 - # Install Java # We need Java for JNI headers, but we don't invoke Maven in this build. ARG java=11 -RUN yum install -y java-$java-openjdk-devel && yum clean all +RUN dnf install -y java-$java-openjdk-devel && dnf clean all # For ci/scripts/{cpp,java}_*.sh ENV ARROW_HOME=/tmp/local \ diff --git a/ci/scripts/jni_build.sh b/ci/scripts/jni_build.sh index aec6fc325c..c000837987 100755 --- a/ci/scripts/jni_build.sh +++ b/ci/scripts/jni_build.sh @@ -66,7 +66,7 @@ cmake \ -DProtobuf_USE_STATIC_LIBS=ON \ -GNinja \ "${EXTRA_CMAKE_OPTIONS[@]}" -cmake --build "${build_dir}" +cmake --build "${build_dir}" --verbose if [ "${ARROW_JAVA_BUILD_TESTS}" = "ON" ]; then ctest \ --output-on-failure \ diff --git a/ci/scripts/jni_macos_build.sh b/ci/scripts/jni_macos_build.sh index f7543b6f7a..13c0675d38 100755 --- a/ci/scripts/jni_macos_build.sh +++ b/ci/scripts/jni_macos_build.sh @@ -59,72 +59,24 @@ fi github_actions_group_begin "Building Arrow C++ libraries" install_dir="${build_dir}/cpp-install" -: "${ARROW_ACERO:=ON}" -export ARROW_ACERO -: "${ARROW_BUILD_TESTS:=OFF}" -export ARROW_BUILD_TESTS -: "${ARROW_DATASET:=ON}" -export ARROW_DATASET -: "${ARROW_GANDIVA:=ON}" -export ARROW_GANDIVA -: "${ARROW_ORC:=ON}" -export ARROW_ORC -: "${ARROW_PARQUET:=ON}" -: "${ARROW_S3:=ON}" -: "${CMAKE_BUILD_TYPE:=Release}" -: "${CMAKE_UNITY_BUILD:=ON}" -export ARROW_TEST_DATA="${arrow_dir}/testing/data" -export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data" +export ARROW_BUILD_TESTS=OFF + +export ARROW_DATASET=ON +export ARROW_GANDIVA=ON +export ARROW_ORC=ON +export ARROW_PARQUET=ON + export AWS_EC2_METADATA_DISABLED=TRUE cmake \ -S "${arrow_dir}/cpp" \ -B "${build_dir}/cpp" \ - -DARROW_ACERO="${ARROW_ACERO}" \ - -DARROW_BUILD_SHARED=OFF \ - -DARROW_BUILD_TESTS="${ARROW_BUILD_TESTS}" \ - -DARROW_CSV="${ARROW_DATASET}" \ - -DARROW_DATASET="${ARROW_DATASET}" \ - -DARROW_SUBSTRAIT="${ARROW_DATASET}" \ - -DARROW_DEPENDENCY_USE_SHARED=OFF \ - -DARROW_GANDIVA="${ARROW_GANDIVA}" \ - -DARROW_GANDIVA_STATIC_LIBSTDCPP=ON \ - -DARROW_JSON="${ARROW_DATASET}" \ - -DARROW_ORC="${ARROW_ORC}" \ - -DARROW_PARQUET="${ARROW_PARQUET}" \ - -DARROW_S3="${ARROW_S3}" \ - -DARROW_USE_CCACHE="${ARROW_USE_CCACHE}" \ - -DAWSSDK_SOURCE=BUNDLED \ - -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" \ - -DCMAKE_INSTALL_PREFIX="${install_dir}" \ - -DCMAKE_UNITY_BUILD="${CMAKE_UNITY_BUILD}" \ - -DGTest_SOURCE=BUNDLED \ - -DPARQUET_BUILD_EXAMPLES=OFF \ - -DPARQUET_BUILD_EXECUTABLES=OFF \ - -DPARQUET_REQUIRE_ENCRYPTION=OFF \ - -Dre2_SOURCE=BUNDLED \ - -GNinja + --preset=ninja-release-jni-macos \ + -DCMAKE_INSTALL_PREFIX="${install_dir}" cmake --build "${build_dir}/cpp" --target install github_actions_group_end -if [ "${ARROW_RUN_TESTS:-}" == "ON" ]; then - github_actions_group_begin "Running Arrow C++ libraries tests" - # MinIO is required - exclude_tests="arrow-s3fs-test" - # unstable - exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test" - exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test" - ctest \ - --exclude-regex "${exclude_tests}" \ - --label-regex unittest \ - --output-on-failure \ - --parallel "$(sysctl -n hw.ncpu)" \ - --test-dir "${build_dir}/cpp" \ - --timeout 300 - github_actions_group_end -fi - export JAVA_JNI_CMAKE_ARGS="-DProtobuf_ROOT=${build_dir}/cpp/protobuf_ep-install" "${source_dir}/ci/scripts/jni_build.sh" \ "${source_dir}" \ @@ -142,6 +94,7 @@ github_actions_group_begin "Checking shared dependencies for libraries" pushd "${dist_dir}" archery linking check-dependencies \ --allow CoreFoundation \ + --allow Network \ --allow Security \ --allow libSystem \ --allow libarrow_cdata_jni \ diff --git a/ci/scripts/jni_manylinux_build.sh b/ci/scripts/jni_manylinux_build.sh index a34ec0f420..3577c37ab3 100755 --- a/ci/scripts/jni_manylinux_build.sh +++ b/ci/scripts/jni_manylinux_build.sh @@ -53,33 +53,19 @@ if [ "${ARROW_USE_CCACHE}" == "ON" ]; then fi github_actions_group_begin "Building Arrow C++ libraries" -devtoolset_version="$(rpm -qa "devtoolset-*-gcc" --queryformat '%{VERSION}' | grep -o "^[0-9]*")" -devtoolset_include_cpp="/opt/rh/devtoolset-${devtoolset_version}/root/usr/include/c++/${devtoolset_version}" -: "${ARROW_ACERO:=ON}" -export ARROW_ACERO -: "${ARROW_BUILD_TESTS:=OFF}" -export ARROW_BUILD_TESTS -: "${ARROW_DATASET:=ON}" -export ARROW_DATASET -: "${ARROW_GANDIVA:=ON}" -export ARROW_GANDIVA -: "${ARROW_GCS:=ON}" -: "${ARROW_JEMALLOC:=OFF}" -: "${ARROW_MIMALLOC:=ON}" -: "${ARROW_RPATH_ORIGIN:=ON}" -: "${ARROW_ORC:=ON}" -export ARROW_ORC -: "${ARROW_PARQUET:=ON}" -: "${ARROW_S3:=ON}" -: "${CMAKE_BUILD_TYPE:=release}" -: "${CMAKE_UNITY_BUILD:=ON}" + : "${VCPKG_ROOT:=/opt/vcpkg}" : "${VCPKG_FEATURE_FLAGS:=-manifests}" -: "${VCPKG_TARGET_TRIPLET:=${VCPKG_DEFAULT_TRIPLET:-x64-linux-static-${CMAKE_BUILD_TYPE}}}" -: "${GANDIVA_CXX_FLAGS:=-isystem;${devtoolset_include_cpp};-isystem;${devtoolset_include_cpp}/x86_64-redhat-linux;-lpthread}" +: "${VCPKG_TARGET_TRIPLET:=${VCPKG_DEFAULT_TRIPLET:-x64-linux-static-release}}" +export VCPKG_TARGET_TRIPLET + +export ARROW_BUILD_TESTS=OFF + +export ARROW_DATASET=ON +export ARROW_GANDIVA=ON +export ARROW_ORC=ON +export ARROW_PARQUET=ON -export ARROW_TEST_DATA="${arrow_dir}/testing/data" -export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data" export AWS_EC2_METADATA_DISABLED=TRUE install_dir="${build_dir}/cpp-install" @@ -87,71 +73,12 @@ install_dir="${build_dir}/cpp-install" cmake \ -S "${arrow_dir}/cpp" \ -B "${build_dir}/cpp" \ - -DARROW_ACERO="${ARROW_ACERO}" \ - -DARROW_BUILD_SHARED=OFF \ - -DARROW_BUILD_TESTS="${ARROW_BUILD_TESTS}" \ - -DARROW_CSV="${ARROW_DATASET}" \ - -DARROW_DATASET="${ARROW_DATASET}" \ - -DARROW_SUBSTRAIT="${ARROW_DATASET}" \ - -DARROW_DEPENDENCY_SOURCE="VCPKG" \ - -DARROW_DEPENDENCY_USE_SHARED=OFF \ - -DARROW_GANDIVA_PC_CXX_FLAGS="${GANDIVA_CXX_FLAGS}" \ - -DARROW_GANDIVA="${ARROW_GANDIVA}" \ - -DARROW_GCS="${ARROW_GCS}" \ - -DARROW_JEMALLOC="${ARROW_JEMALLOC}" \ - -DARROW_JSON="${ARROW_DATASET}" \ - -DARROW_MIMALLOC="${ARROW_MIMALLOC}" \ - -DARROW_ORC="${ARROW_ORC}" \ - -DARROW_PARQUET="${ARROW_PARQUET}" \ - -DARROW_RPATH_ORIGIN="${ARROW_RPATH_ORIGIN}" \ - -DARROW_S3="${ARROW_S3}" \ - -DARROW_USE_CCACHE="${ARROW_USE_CCACHE}" \ - -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" \ - -DCMAKE_INSTALL_PREFIX="${install_dir}" \ - -DCMAKE_UNITY_BUILD="${CMAKE_UNITY_BUILD}" \ - -DGTest_SOURCE=BUNDLED \ - -DORC_SOURCE=BUNDLED \ - -DORC_PROTOBUF_EXECUTABLE="${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/tools/protobuf/protoc" \ - -DPARQUET_BUILD_EXAMPLES=OFF \ - -DPARQUET_BUILD_EXECUTABLES=OFF \ - -DPARQUET_REQUIRE_ENCRYPTION=OFF \ - -DVCPKG_MANIFEST_MODE=OFF \ - -DVCPKG_TARGET_TRIPLET="${VCPKG_TARGET_TRIPLET}" \ - -GNinja + --preset=ninja-release-jni-linux \ + -DCMAKE_INSTALL_PREFIX="${install_dir}" cmake --build "${build_dir}/cpp" cmake --install "${build_dir}/cpp" github_actions_group_end -if [ "${ARROW_RUN_TESTS:-OFF}" = "ON" ]; then - github_actions_group_begin "Running Arrow C++ libraries tests" - # MinIO is required - exclude_tests="arrow-s3fs-test" - case $(arch) in - aarch64) - # GCS testbench is crashed on aarch64: - # ImportError: ../grpc/_cython/cygrpc.cpython-38-aarch64-linux-gnu.so: - # undefined symbol: vtable for std::__cxx11::basic_ostringstream< - # char, std::char_traits, std::allocator > - exclude_tests="${exclude_tests}|arrow-gcsfs-test" - ;; - esac - # unstable - exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test" - exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test" - # external dependency - exclude_tests="${exclude_tests}|arrow-gcsfs-test" - # strptime - exclude_tests="${exclude_tests}|arrow-utility-test" - ctest \ - --exclude-regex "${exclude_tests}" \ - --label-regex unittest \ - --output-on-failure \ - --parallel "$(nproc)" \ - --test-dir "${build_dir}/cpp" \ - --timeout 300 - github_actions_group_end -fi - JAVA_JNI_CMAKE_ARGS="-DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" JAVA_JNI_CMAKE_ARGS="${JAVA_JNI_CMAKE_ARGS} -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET}" export JAVA_JNI_CMAKE_ARGS diff --git a/ci/scripts/jni_windows_build.sh b/ci/scripts/jni_windows_build.sh index d01ef45f5a..6503ac63e5 100755 --- a/ci/scripts/jni_windows_build.sh +++ b/ci/scripts/jni_windows_build.sh @@ -68,7 +68,7 @@ cmake \ -B "${build_dir}/cpp" \ -DARROW_ACERO="${ARROW_ACERO}" \ -DARROW_BUILD_SHARED=OFF \ - -DARROW_BUILD_TESTS=ON \ + -DARROW_BUILD_TESTS="${ARROW_BUILD_TESTS}" \ -DARROW_CSV="${ARROW_DATASET}" \ -DARROW_DATASET="${ARROW_DATASET}" \ -DARROW_SUBSTRAIT="${ARROW_DATASET}" \ diff --git a/compose.yaml b/compose.yaml index b125c3c983..f5082a22aa 100644 --- a/compose.yaml +++ b/compose.yaml @@ -99,7 +99,7 @@ services: cache_from: - ${REPO}:${ARCH}-vcpkg-jni-${VCPKG} args: - base: ${ARROW_REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2014-vcpkg-${VCPKG} + base: ${ARROW_REPO}:${ARCH}-cpp-jni-${VCPKG} volumes: - .:/arrow-java:delegated - ${ARROW_REPO_ROOT}:/arrow:delegated diff --git a/pom.xml b/pom.xml index 49e0c47c60..f6c9053db9 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache apache - 34 + 35 org.apache.arrow @@ -91,6 +91,7 @@ under the License. + 1695310533 ${project.build.directory}/generated-sources 1.9.0 5.12.2 @@ -123,6 +124,8 @@ under the License. 3.2.2 From ed81e5981a2bee40584b3a411ed755cb4cc5b91f Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Tue, 28 Oct 2025 00:57:13 +0100 Subject: [PATCH 073/271] GH-882: Add support for loading native library from a user specified location (#883) ## What's Changed Add an opt-in code path that attempts to load the native library relative to the path specified by the `arrow.cdata.library.path` system property. Closes #882. --- .../org/apache/arrow/c/jni/JniLoader.java | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/c/src/main/java/org/apache/arrow/c/jni/JniLoader.java b/c/src/main/java/org/apache/arrow/c/jni/JniLoader.java index f712b400bf..46c93f5541 100644 --- a/c/src/main/java/org/apache/arrow/c/jni/JniLoader.java +++ b/c/src/main/java/org/apache/arrow/c/jni/JniLoader.java @@ -75,8 +75,23 @@ private synchronized void loadRemaining() { } private void load(String name) { - final String libraryToLoad = - name + "/" + getNormalizedArch() + "/" + System.mapLibraryName(name); + String libraryName = System.mapLibraryName(name); + + // If 'arrow.cdata.library.path' is defined, try to load the native library from there + String libraryPath = System.getProperty("arrow.cdata.library.path"); + if (libraryPath != null) { + try { + File libraryFile = new File(libraryPath, libraryName); + if (libraryFile.isFile()) { + System.load(libraryFile.getAbsolutePath()); + return; + } + } catch (UnsatisfiedLinkError e) { + // Ignore this error and fall back to extracting from the JAR file + } + } + + final String libraryToLoad = name + "/" + getNormalizedArch() + "/" + libraryName; try { File temp = File.createTempFile("jnilib-", ".tmp", new File(System.getProperty("java.io.tmpdir"))); From a4f3f3ef8eaca1328a3a6032397caf7a2f9ac22d Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 29 Oct 2025 13:55:21 +0900 Subject: [PATCH 074/271] GH-899: [Dataset] Initialize compute module (#893) ## What's Changed Depends on https://github.com/apache/arrow-java/pull/865 Closes #899. --- dataset/src/main/cpp/jni_wrapper.cc | 8 ++++++++ .../main/java/org/apache/arrow/dataset/jni/JniLoader.java | 1 + .../java/org/apache/arrow/dataset/jni/JniWrapper.java | 3 +++ docs/source/substrait.rst | 2 +- 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/dataset/src/main/cpp/jni_wrapper.cc b/dataset/src/main/cpp/jni_wrapper.cc index 49cc85251c..e8087648eb 100644 --- a/dataset/src/main/cpp/jni_wrapper.cc +++ b/dataset/src/main/cpp/jni_wrapper.cc @@ -23,6 +23,7 @@ #include "arrow/array/concatenate.h" #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" +#include "arrow/compute/initialize.h" #include "arrow/dataset/api.h" #include "arrow/dataset/file_base.h" #ifdef ARROW_CSV @@ -807,6 +808,13 @@ JNIEXPORT void JNICALL Java_org_apache_arrow_dataset_jni_JniWrapper_ensureS3Fina JNI_METHOD_END() } +JNIEXPORT void JNICALL Java_org_apache_arrow_dataset_jni_JniWrapper_initialize( + JNIEnv* env, jobject) { + JNI_METHOD_START + JniAssertOkOrThrow(arrow::compute::Initialize()); + JNI_METHOD_END() +} + /* * Class: org_apache_arrow_dataset_file_JniWrapper * Method: makeFileSystemDatasetFactory diff --git a/dataset/src/main/java/org/apache/arrow/dataset/jni/JniLoader.java b/dataset/src/main/java/org/apache/arrow/dataset/jni/JniLoader.java index 631b8b1bbe..5fb4816488 100644 --- a/dataset/src/main/java/org/apache/arrow/dataset/jni/JniLoader.java +++ b/dataset/src/main/java/org/apache/arrow/dataset/jni/JniLoader.java @@ -56,6 +56,7 @@ public void ensureLoaded() { } loadRemaining(); ensureS3FinalizedOnShutdown(); + JniWrapper.get().initialize(); } private synchronized void loadRemaining() { diff --git a/dataset/src/main/java/org/apache/arrow/dataset/jni/JniWrapper.java b/dataset/src/main/java/org/apache/arrow/dataset/jni/JniWrapper.java index 6637c113d9..cfef098ec4 100644 --- a/dataset/src/main/java/org/apache/arrow/dataset/jni/JniWrapper.java +++ b/dataset/src/main/java/org/apache/arrow/dataset/jni/JniWrapper.java @@ -124,4 +124,7 @@ public native long createScanner( * uninitialized, then this is a noop. */ public native void ensureS3Finalized(); + + /** Initialize Arrow Compute. */ + public native void initialize(); } diff --git a/docs/source/substrait.rst b/docs/source/substrait.rst index b3678ac815..5ec07f1658 100644 --- a/docs/source/substrait.rst +++ b/docs/source/substrait.rst @@ -19,7 +19,7 @@ Substrait ========= -The ``arrow-dataset`` module can execute Substrait_ plans via the :external+arrow:doc:`Acero ` +The ``arrow-dataset`` module can execute Substrait_ plans via the :external+arrow:doc:`Acero ` query engine. Executing Queries Using Substrait Plans From c4d3c9e2777f5d7b55c3f8fdb5e5155b71bbd87a Mon Sep 17 00:00:00 2001 From: ViggoC Date: Wed, 29 Oct 2025 20:42:45 +0800 Subject: [PATCH 075/271] GH-109: Implement Vector Validators for StringView (#886) ## What's Changed Implement Vector Validators for StringView. Closes #109. --- .../validate/ValidateVectorBufferVisitor.java | 29 ++++++++++++++----- .../validate/ValidateVectorDataVisitor.java | 3 +- .../validate/ValidateVectorTypeVisitor.java | 9 +++++- .../validate/ValidateVectorVisitor.java | 9 ++++-- .../vector/TestVariableWidthViewVector.java | 17 ++++++++++- 5 files changed, 54 insertions(+), 13 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorBufferVisitor.java b/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorBufferVisitor.java index 5c7215437f..5cfe64b14e 100644 --- a/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorBufferVisitor.java +++ b/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorBufferVisitor.java @@ -52,14 +52,22 @@ private void validateVectorCommon(ValueVector vector) { if (vector instanceof FieldVector) { FieldVector fieldVector = (FieldVector) vector; - // TODO: https://github.com/apache/arrow/issues/41734 int typeBufferCount = TypeLayout.getTypeBufferCount(arrowType); - validateOrThrow( - fieldVector.getFieldBuffers().size() == typeBufferCount, - "Expected %s buffers in vector of type %s, got %s.", - typeBufferCount, - vector.getField().getType().toString(), - fieldVector.getFieldBuffers().size()); + if (TypeLayout.getTypeLayout(arrowType).isFixedBufferCount()) { + validateOrThrow( + fieldVector.getFieldBuffers().size() == typeBufferCount, + "Expected %s buffers in vector of type %s, got %s.", + typeBufferCount, + vector.getField().getType().toString(), + fieldVector.getFieldBuffers().size()); + } else { + validateOrThrow( + fieldVector.getFieldBuffers().size() >= typeBufferCount, + "Expected at least %s buffers in vector of type %s, got %s.", + typeBufferCount, + vector.getField().getType().toString(), + fieldVector.getFieldBuffers().size()); + } } } @@ -158,7 +166,12 @@ public Void visit(BaseLargeVariableWidthVector vector, Void value) { @Override public Void visit(BaseVariableWidthViewVector vector, Void value) { - throw new UnsupportedOperationException("View vectors are not supported."); + final int valueCount = vector.getValueCount(); + validateVectorCommon(vector); + validateOrThrow(vector.getFieldBuffers().size() >= 2, "Expected at least 2 buffers."); + validateValidityBuffer(vector, valueCount); + validateDataBuffer(vector, (long) valueCount * BaseVariableWidthViewVector.ELEMENT_SIZE); + return null; } @Override diff --git a/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorDataVisitor.java b/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorDataVisitor.java index c62bff79f7..9da8cc813e 100644 --- a/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorDataVisitor.java +++ b/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorDataVisitor.java @@ -121,7 +121,8 @@ public Void visit(BaseLargeVariableWidthVector vector, Void value) { @Override public Void visit(BaseVariableWidthViewVector vector, Void value) { - throw new UnsupportedOperationException("View vectors are not supported."); + vector.validateScalars(); + return null; } @Override diff --git a/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorTypeVisitor.java b/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorTypeVisitor.java index daad41dbdc..395852ef79 100644 --- a/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorTypeVisitor.java +++ b/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorTypeVisitor.java @@ -61,6 +61,8 @@ import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.ViewVarBinaryVector; +import org.apache.arrow.vector.ViewVarCharVector; import org.apache.arrow.vector.compare.VectorVisitor; import org.apache.arrow.vector.complex.DenseUnionVector; import org.apache.arrow.vector.complex.FixedSizeListVector; @@ -380,7 +382,12 @@ public Void visit(BaseLargeVariableWidthVector vector, Void value) { @Override public Void visit(BaseVariableWidthViewVector vector, Void value) { - throw new UnsupportedOperationException("View vectors are not supported."); + if (vector instanceof ViewVarCharVector) { + validateVectorCommon(vector, ArrowType.Utf8View.class); + } else if (vector instanceof ViewVarBinaryVector) { + validateVectorCommon(vector, ArrowType.BinaryView.class); + } + return null; } @Override diff --git a/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorVisitor.java b/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorVisitor.java index 5004ba488c..2111410016 100644 --- a/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorVisitor.java +++ b/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorVisitor.java @@ -107,8 +107,13 @@ public Void visit(BaseLargeVariableWidthVector left, Void value) { } @Override - public Void visit(BaseVariableWidthViewVector left, Void value) { - throw new UnsupportedOperationException("View vectors are not supported."); + public Void visit(BaseVariableWidthViewVector vector, Void value) { + if (vector.getValueCount() > 0) { + if (vector.getDataBuffer() == null || vector.getDataBuffer().capacity() == 0) { + throw new IllegalArgumentException("valueBuffer is null or capacity is 0"); + } + } + return null; } @Override diff --git a/vector/src/test/java/org/apache/arrow/vector/TestVariableWidthViewVector.java b/vector/src/test/java/org/apache/arrow/vector/TestVariableWidthViewVector.java index f7c66a00be..baf5e672c8 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestVariableWidthViewVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestVariableWidthViewVector.java @@ -61,6 +61,7 @@ import org.apache.arrow.vector.util.ReusableByteArray; import org.apache.arrow.vector.util.Text; import org.apache.arrow.vector.util.TransferPair; +import org.apache.arrow.vector.validate.ValidateUtil; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -2445,7 +2446,7 @@ public void testSplitAndTransferWithLongStringsOnValiditySplit() { final ViewVarBinaryVector sourceVector = newViewVarBinaryVector(EMPTY_SCHEMA_PATH, allocator)) { testSplitAndTransferOnValiditySplitHelper( - targetVector, sourceVector, startIndex, length, data); + targetVector, sourceVector, startIndex, length, binaryData); } } @@ -2852,4 +2853,18 @@ public void testVectorLoadUnloadOnMixedTypes() { } } } + + @Test + public void testValidate() { + try (final ViewVarCharVector vector = new ViewVarCharVector("v", allocator)) { + vector.validateFull(); + setVector(vector, STR1, STR2, STR3); + vector.validateFull(); + + vector.getDataBuffer().capacity(0); + ValidateUtil.ValidateException e = + assertThrows(ValidateUtil.ValidateException.class, () -> vector.validate()); + assertTrue(e.getMessage().contains("Not enough capacity for data buffer")); + } + } } From b8a23ddd8940d5a804b0091cfd7a908af4d0f532 Mon Sep 17 00:00:00 2001 From: Christopher Lambert Date: Thu, 30 Oct 2025 10:55:10 +0100 Subject: [PATCH 076/271] GH-900: Fix gandiva groupId in arrow-bom (#901) Use correct groupId for the `arrow-gandiva` artifact. Closes #900. --- bom/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bom/pom.xml b/bom/pom.xml index 61b452b9c1..655c33b813 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -165,7 +165,7 @@ under the License. ${project.version} - org.apache.arrow + org.apache.arrow.gandiva arrow-gandiva ${project.version} From 9f68d08f0d67699c2887bac011238ee7e0b01f06 Mon Sep 17 00:00:00 2001 From: ViggoC Date: Fri, 31 Oct 2025 15:55:12 +0800 Subject: [PATCH 077/271] GH-762: Implement VectorAppender for RunEndEncodedVector (#884) ## What's Changed Implement VectorAppender for RunEndEncodedVector Closes #762. --- .../arrow/vector/util/VectorAppender.java | 99 +++++++++++++++++++ .../arrow/vector/util/TestVectorAppender.java | 67 +++++++++++++ 2 files changed, 166 insertions(+) diff --git a/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java b/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java index 0dc96a4d4b..e7c0d11cb9 100644 --- a/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java +++ b/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java @@ -24,13 +24,17 @@ import org.apache.arrow.memory.util.MemoryUtil; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.BaseFixedWidthVector; +import org.apache.arrow.vector.BaseIntVector; import org.apache.arrow.vector.BaseLargeVariableWidthVector; import org.apache.arrow.vector.BaseVariableWidthVector; import org.apache.arrow.vector.BaseVariableWidthViewVector; +import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.BitVectorHelper; import org.apache.arrow.vector.ExtensionTypeVector; +import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.NullVector; +import org.apache.arrow.vector.SmallIntVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.compare.TypeEqualsVisitor; import org.apache.arrow.vector.compare.VectorVisitor; @@ -39,6 +43,7 @@ import org.apache.arrow.vector.complex.LargeListVector; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.NonNullableStructVector; +import org.apache.arrow.vector.complex.RunEndEncodedVector; import org.apache.arrow.vector.complex.UnionVector; /** Utility to append two vectors together. */ @@ -698,4 +703,98 @@ public ValueVector visit(ExtensionTypeVector deltaVector, Void value) { deltaVector.getUnderlyingVector().accept(underlyingAppender, null); return targetVector; } + + @Override + public ValueVector visit(RunEndEncodedVector deltaVector, Void value) { + Preconditions.checkArgument( + typeVisitor.equals(deltaVector), + "The deltaVector to append must have the same type as the targetVector"); + + if (deltaVector.getValueCount() == 0) { + return targetVector; // optimization, nothing to append, return + } + + RunEndEncodedVector targetEncodedVector = (RunEndEncodedVector) targetVector; + + final int targetLogicalValueCount = targetEncodedVector.getValueCount(); + + // Append the values vector first. + VectorAppender valueAppender = new VectorAppender(targetEncodedVector.getValuesVector()); + deltaVector.getValuesVector().accept(valueAppender, null); + + // Then append the run-ends vector. + BaseIntVector targetRunEndsVector = (BaseIntVector) targetEncodedVector.getRunEndsVector(); + BaseIntVector deltaRunEndsVector = (BaseIntVector) deltaVector.getRunEndsVector(); + appendRunEndsVector(targetRunEndsVector, deltaRunEndsVector, targetLogicalValueCount); + + targetEncodedVector.setValueCount(targetLogicalValueCount + deltaVector.getValueCount()); + return targetVector; + } + + private void appendRunEndsVector( + BaseIntVector targetRunEndsVector, + BaseIntVector deltaRunEndsVector, + int targetLogicalValueCount) { + int targetPhysicalValueCount = targetRunEndsVector.getValueCount(); + int newPhysicalValueCount = targetPhysicalValueCount + deltaRunEndsVector.getValueCount(); + + // make sure there is enough capacity + while (targetVector.getValueCapacity() < newPhysicalValueCount) { + targetVector.reAlloc(); + } + + // append validity buffer + BitVectorHelper.concatBits( + targetRunEndsVector.getValidityBuffer(), + targetRunEndsVector.getValueCount(), + deltaRunEndsVector.getValidityBuffer(), + deltaRunEndsVector.getValueCount(), + targetRunEndsVector.getValidityBuffer()); + + // shift and append data buffer + shiftAndAppendRunEndsDataBuffer( + targetRunEndsVector, + targetPhysicalValueCount, + deltaRunEndsVector.getDataBuffer(), + targetLogicalValueCount, + deltaRunEndsVector.getValueCount()); + + targetRunEndsVector.setValueCount(newPhysicalValueCount); + } + + private void shiftAndAppendRunEndsDataBuffer( + BaseIntVector toRunEndVector, + int toIndex, + ArrowBuf fromRunEndBuffer, + int offset, + int physicalLength) { + ArrowBuf toRunEndBuffer = toRunEndVector.getDataBuffer(); + if (toRunEndVector instanceof SmallIntVector) { + byte typeWidth = SmallIntVector.TYPE_WIDTH; + for (int i = 0; i < physicalLength; i++) { + toRunEndBuffer.setShort( + (long) (i + toIndex) * typeWidth, + fromRunEndBuffer.getShort((long) (i) * typeWidth) + offset); + } + + } else if (toRunEndVector instanceof IntVector) { + byte typeWidth = IntVector.TYPE_WIDTH; + for (int i = 0; i < physicalLength; i++) { + toRunEndBuffer.setInt( + (long) (i + toIndex) * typeWidth, + fromRunEndBuffer.getInt((long) (i) * typeWidth) + offset); + } + + } else if (toRunEndVector instanceof BigIntVector) { + byte typeWidth = BigIntVector.TYPE_WIDTH; + for (int i = 0; i < physicalLength; i++) { + toRunEndBuffer.setLong( + (long) (i + toIndex) * typeWidth, + fromRunEndBuffer.getLong((long) (i) * typeWidth) + offset); + } + } else { + throw new IllegalArgumentException( + "Run-end vector and must be of type int with size 16, 32, or 64 bits."); + } + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java b/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java index 4ee9630a4d..df5521a1ad 100644 --- a/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java +++ b/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java @@ -47,6 +47,7 @@ import org.apache.arrow.vector.complex.FixedSizeListVector; import org.apache.arrow.vector.complex.LargeListVector; import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.RunEndEncodedVector; import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.complex.UnionVector; import org.apache.arrow.vector.holders.NullableBigIntHolder; @@ -1025,6 +1026,72 @@ public void testAppendDenseUnionVectorMismatch() { } } + @Test + public void testAppendRunEndEncodedVector() { + final FieldType reeFieldType = FieldType.notNullable(ArrowType.RunEndEncoded.INSTANCE); + final Field runEndsField = + new Field("runEnds", FieldType.notNullable(Types.MinorType.INT.getType()), null); + final Field valuesField = Field.nullable("values", Types.MinorType.INT.getType()); + final List children = Arrays.asList(runEndsField, valuesField); + + final Field targetField = new Field("target", reeFieldType, children); + final Field deltaField = new Field("delta", reeFieldType, children); + try (RunEndEncodedVector target = new RunEndEncodedVector(targetField, allocator, null); + RunEndEncodedVector delta = new RunEndEncodedVector(deltaField, allocator, null)) { + + // populate target + target.allocateNew(); + // data: [1, 1, 2, null, 3, 3, 3] (7 values) + // values: [1, 2, null, 3] + // runEnds: [2, 3, 4, 7] + ValueVectorDataPopulator.setVector((IntVector) target.getValuesVector(), 1, 2, null, 3); + ValueVectorDataPopulator.setVector((IntVector) target.getRunEndsVector(), 2, 3, 4, 7); + target.setValueCount(7); + + // populate delta + delta.allocateNew(); + // data: [3, 4, 4, 5, null, null] (6 values) + // values: [3, 4, 5, null] + // runEnds: [1, 3, 4, 6] + ValueVectorDataPopulator.setVector((IntVector) delta.getValuesVector(), 3, 4, 5, null); + ValueVectorDataPopulator.setVector((IntVector) delta.getRunEndsVector(), 1, 3, 4, 6); + delta.setValueCount(6); + + VectorAppender appender = new VectorAppender(target); + delta.accept(appender, null); + + assertEquals(13, target.getValueCount()); + + final Field expectedField = new Field("expected", reeFieldType, children); + try (RunEndEncodedVector expected = new RunEndEncodedVector(expectedField, allocator, null)) { + expected.allocateNew(); + // expected data: [1, 1, 2, null, 3, 3, 3, 3, 4, 4, 5, null, null] (13 values) + // expected values: [1, 2, null, 3, 3, 4, 5, null] + // expected runEnds: [2, 3, 4, 7, 8, 10, 11, 13] + ValueVectorDataPopulator.setVector( + (IntVector) expected.getValuesVector(), 1, 2, null, 3, 3, 4, 5, null); + ValueVectorDataPopulator.setVector( + (IntVector) expected.getRunEndsVector(), 2, 3, 4, 7, 8, 10, 11, 13); + expected.setValueCount(13); + + assertVectorsEqual(expected, target); + } + + // Check that delta is unchanged. + final Field expectedDeltaField = new Field("expectedDelta", reeFieldType, children); + try (RunEndEncodedVector expectedDelta = + new RunEndEncodedVector(expectedDeltaField, allocator, null)) { + expectedDelta.allocateNew(); + ValueVectorDataPopulator.setVector( + (IntVector) expectedDelta.getValuesVector(), 3, 4, 5, null); + ValueVectorDataPopulator.setVector( + (IntVector) expectedDelta.getRunEndsVector(), 1, 3, 4, 6); + expectedDelta.setValueCount(6); + assertVectorsEqual(expectedDelta, delta); + } + } + } + @Test public void testAppendVectorNegative() { final int vectorLength = 10; From 03e0d4da63e1de747e645b0053d737b263d7f26d Mon Sep 17 00:00:00 2001 From: Fabio Buso Date: Tue, 4 Nov 2025 12:45:43 +0100 Subject: [PATCH 078/271] MINOR: Bump io.netty:netty-bom from 4.1.119.Final to 4.1.127.Final (#855) ## What's Changed This PR bumps the netty version to 4.1.127.Final to address CVE-2025-55163 Based on https://github.com/apache/arrow-java/pull/740 - It looks like the team would prefer to stay on Netty 4.1.x for the time being. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f6c9053db9..5b683acd1b 100644 --- a/pom.xml +++ b/pom.xml @@ -97,7 +97,7 @@ under the License. 5.12.2 2.0.17 33.4.8-jre - 4.1.119.Final + 4.1.127.Final 1.73.0 4.30.2 2.18.3 From ba2f7d62ebbb964956ddebb5a250e5637f816083 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Nov 2025 13:56:33 +0100 Subject: [PATCH 079/271] MINOR: Bump logback.version from 1.5.18 to 1.5.20 (#897) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps `logback.version` from 1.5.18 to 1.5.20. Updates `ch.qos.logback:logback-classic` from 1.5.18 to 1.5.20

Release notes

Sourced from ch.qos.logback:logback-classic's releases.

Logback 1.5.19

2025-09-30 Release of logback version 1.5.19

• Disallow "new" operator in the condition attribute of <if> elements. This fixes an ACE vulnerability recorded as CVE-2025-11226.

• At initialization time, slightly better reporting about watched configuration files.

• Softer message regarding usage of ConsoleAppender and its potential impact on performance.

• In ViewStatusMessagesServlet, restrict processing of "Clear" button to POST method. This change was proposed by Ralf Wiebicke who also provided the relevant PR.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit e572d4f87f06674788eb3ca7148e8d1dffc615fa associated with the tag v_1.5.19. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits
  • 930fb15 prepare release 1.5.20
  • 0b4432a provide an alternative to Janino based conditional configuration processing -...
  • 258558f provide an alternative to Janino based conditional configuration processing -...
  • ee77a70 provide an alternative to Janino based conditional configuration processing -...
  • 5ca7ce8 provide an alternative to Janino based conditional configuration processing -...
  • 728803f fix typo
  • aa5eeb1 start work on version 1.5.20-SNAPSHOT
  • e572d4f skip deployment of blackbox and example modules, published as version 1.5.9
  • 4adae8b add plugin for Maven Central deployment
  • ee70cf4 prepare release 1.5.19
  • Additional commits viewable in compare view

Updates `ch.qos.logback:logback-core` from 1.5.18 to 1.5.20
Release notes

Sourced from ch.qos.logback:logback-core's releases.

Logback 1.5.19

2025-09-30 Release of logback version 1.5.19

• Disallow "new" operator in the condition attribute of <if> elements. This fixes an ACE vulnerability recorded as CVE-2025-11226.

• At initialization time, slightly better reporting about watched configuration files.

• Softer message regarding usage of ConsoleAppender and its potential impact on performance.

• In ViewStatusMessagesServlet, restrict processing of "Clear" button to POST method. This change was proposed by Ralf Wiebicke who also provided the relevant PR.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit e572d4f87f06674788eb3ca7148e8d1dffc615fa associated with the tag v_1.5.19. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits
  • 930fb15 prepare release 1.5.20
  • 0b4432a provide an alternative to Janino based conditional configuration processing -...
  • 258558f provide an alternative to Janino based conditional configuration processing -...
  • ee77a70 provide an alternative to Janino based conditional configuration processing -...
  • 5ca7ce8 provide an alternative to Janino based conditional configuration processing -...
  • 728803f fix typo
  • aa5eeb1 start work on version 1.5.20-SNAPSHOT
  • e572d4f skip deployment of blackbox and example modules, published as version 1.5.9
  • 4adae8b add plugin for Maven Central deployment
  • ee70cf4 prepare release 1.5.19
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5b683acd1b..41b82fabab 100644 --- a/pom.xml +++ b/pom.xml @@ -111,7 +111,7 @@ under the License. true 2.37.0 3.49.3 - 1.5.18 + 1.5.21 none -Xdoclint:none From 87d727a85a33f10f67e7f311170a282ef5dbae29 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Nov 2025 15:22:49 +0100 Subject: [PATCH 080/271] MINOR: Bump com.github.luben:zstd-jni from 1.5.7-2 to 1.5.7-6 (#896) Bumps [com.github.luben:zstd-jni](https://github.com/luben/zstd-jni) from 1.5.7-2 to 1.5.7-6.
Commits
  • c3cf908 v1.5.7-6
  • 1941118 Restore getContentSize behaviour
  • 97b5262 Update the README with more info about the "cloud" flavour
  • 7093a69 Add new flavour of artifacr
  • 9c3386d fix: compress & decompress DirectByteBufferStream error is always ZSTD_error_...
  • 7de4360 Add sbt-sonatype plugin to the build
  • 8250e09 Changes to migrate to the new Sonatype publishing infrastructure
  • 81e0d71 fix: decompressFrame(byte[], int) fixed, throwing exception on get frame cont...
  • 8532971 fix: decompress(byte[]) method now decompresses all frames, new tests added
  • 051e8de refactor: throwing an exception on findFrameCompressedSize0 error
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.github.luben:zstd-jni&package-manager=maven&previous-version=1.5.7-2&new-version=1.5.7-6)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- compression/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compression/pom.xml b/compression/pom.xml index 6f60eb7d0a..ba13156243 100644 --- a/compression/pom.xml +++ b/compression/pom.xml @@ -55,7 +55,7 @@ under the License. com.github.luben zstd-jni - 1.5.7-2 + 1.5.7-6 From edc6cf350abc08e91fb1009c73be42288207d074 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Nov 2025 16:16:16 +0100 Subject: [PATCH 081/271] MINOR: [CI] Bump actions/download-artifact from 5.0.0 to 6.0.0 (#895) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 5.0.0 to 6.0.0.
Release notes

Sourced from actions/download-artifact's releases.

v6.0.0

What's Changed

BREAKING CHANGE: this update supports Node v24.x. This is not a breaking change per-se but we're treating it as such.

New Contributors

Full Changelog: https://github.com/actions/download-artifact/compare/v5...v6.0.0

Commits
  • 018cc2c Merge pull request #438 from actions/danwkennedy/prepare-6.0.0
  • 815651c Revert "Remove github.dep.yml"
  • bb3a066 Remove github.dep.yml
  • fa1ce46 Prepare v6.0.0
  • 4a24838 Merge pull request #431 from danwkennedy/patch-1
  • 5e3251c Readme: spell out the first use of GHES
  • abefc31 Merge pull request #424 from actions/yacaovsnc/update_readme
  • ac43a60 Update README with artifact extraction details
  • de96f46 Merge pull request #417 from actions/yacaovsnc/update_readme
  • 7993cb4 Remove migration guide for artifact download changes
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/download-artifact&package-manager=github_actions&previous-version=5.0.0&new-version=6.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index f71f8aeed1..b53812ef5e 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -101,7 +101,7 @@ jobs: packages: write steps: - name: Download source archive - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: name: release-source - name: Extract source archive @@ -174,7 +174,7 @@ jobs: MACOSX_DEPLOYMENT_TARGET: "14.0" steps: - name: Download source archive - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: name: release-source - name: Extract source archive @@ -302,7 +302,7 @@ jobs: arch: "x86_64" steps: - name: Download source archive - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: name: release-source - name: Extract source archive @@ -381,7 +381,7 @@ jobs: - jni-windows steps: - name: Download artifacts - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: path: artifacts - name: Decompress artifacts @@ -462,11 +462,11 @@ jobs: with: cache: 'pip' - name: Download source archive - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: name: release-source - name: Download Javadocs - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: name: reference - name: Extract source archive @@ -531,7 +531,7 @@ jobs: cp ../.asf.yaml ./ git add .nojekyll .asf.yaml - name: Download - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: name: release-html - name: Extract @@ -567,7 +567,7 @@ jobs: - ubuntu-latest steps: - name: Download release artifacts - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: pattern: release-* - name: Verify @@ -601,7 +601,7 @@ jobs: contents: write steps: - name: Download release artifacts - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: pattern: release-* path: artifacts From 973e974e15d5997bb55102d71e8b3f59f3ba7eea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Nov 2025 21:50:56 +0100 Subject: [PATCH 082/271] MINOR: Bump commons-codec:commons-codec from 1.18.0 to 1.19.0 (#871) Bumps [commons-codec:commons-codec](https://github.com/apache/commons-codec) from 1.18.0 to 1.19.0.
Changelog

Sourced from commons-codec:commons-codec's changelog.

Apache Commons Codec 1.19.0 Release Notes

The Apache Commons Codec team is pleased to announce the release of Apache Commons Codec 1.19.0.

The Apache Commons Codec component contains encoders and decoders for formats such as Base16, Base32, Base64, digest, and Hexadecimal. In addition to these widely used encoders and decoders, the codec package also maintains a collection of phonetic encoding utilities.

This is a feature and maintenance release. Java 8 or later is required.

New features

  •         Add HmacUtils.hmac(Path). Thanks to Gary Gregory.

  •         Add HmacUtils.hmacHex(Path). Thanks to Gary Gregory.

  •  Add PMD check to the default Maven goal. Thanks to Gary
Gregory.

  •  Add SpotBugs check to the default Maven goal. Thanks to Gary
Gregory.

Fixed Bugs

  •  Remove -nouses directive from maven-bundle-plugin. OSGi
package imports now state 'uses' definitions for package imports, this
doesn't affect JPMS (from org.apache.commons:commons-parent:80). Thanks
to Gary Gregory.

  •  Refactor DigestUtils.updateDigest(MessageDigest, File) to
use NIO. Thanks to Gary Gregory.
  • CODEC-328: Clarify Javadoc for org.apache.commons.codec.digest.UnixCrypt.crypt(byte[],String). Thanks to Gary Gregory.
  •  Precompile regular expressions in
DaitchMokotoffSoundex.Rule. Thanks to Gary Gregory.

  •  Precompile regular expressions in
DaitchMokotoffSoundex.parseRules(Scanner, String, Map, Map). Thanks to
Gary Gregory.

  •  Precompile regular expressions in
Lang.loadFromResource(String, Languages). Thanks to Gary Gregory.

  •  Precompile regular expressions in
PhoneticEngine.encode(String, LanguageSet). Thanks to Gary Gregory.

  •  Precompile regular expressions in
org.apache.commons.codec.language.bm.Rule.parse*(*). Thanks to Gary
Gregory.

  •  Remove redundant checks for whitespace in
DaitchMokotoffSoundex.soundex(String, boolean). Thanks to Gary Gregory.

  •  Javadoc typo in Base16.java
[#380](https://github.com/apache/commons-codec/issues/380). Thanks to
Sebastian Baunsgaard.

  •  Deprecate unused constant
org.apache.commons.codec.language.bm.Rule.ALL. Thanks to Gary Gregory.
  • CODEC-331: org.apache.commons.codec.language.bm.Rule.parsePhonemeExpr(String) adds duplicate empty phoneme when input ends with |. Thanks to IlikeCode, Gary Gregory.
  • CODEC-331: org.apache.commons.codec.language.DaitchMokotoffSoundex.cleanup(String) does not remove special characters like punctuation. Thanks to IlikeCode, Gary Gregory.
  •  Fix PMD multiple UnnecessaryFullyQualifiedName in
org.apache.commons.codec.binary.StringUtils. Thanks to Gary Gregory.

  •  Fix PMD UnusedFormalParameter in private constructor in
org.apache.commons.codec.binary.Base16. Thanks to Gary Gregory.

  •  Fix PMD multiple UnnecessaryFullyQualifiedName in
org.apache.commons.codec.digest.Blake3. Thanks to Gary Gregory.

  •  Fix PMD UnnecessaryFullyQualifiedName in
org.apache.commons.codec.digest.Md5Crypt. Thanks to Gary Gregory.

  •  Fix PMD EmptyControlStatement in
org.apache.commons.codec.language.Metaphone. Thanks to Gary Gregory.

  •  Fix SpotBugs [ERROR] Medium:
org.apache.commons.codec.binary.BaseNCodec$AbstractBuilder.setEncodeTable(byte[])
may expose internal representation by storing an externally mutable
object into BaseNCodec$AbstractBuilder.encodeTable
[org.apache.commons.codec.binary.BaseNCodec$AbstractBuilder] At
BaseNCodec.java:[line 131] EI_EXPOSE_REP2. Thanks to Gary Gregory.

  •  The method
org.apache.commons.codec.binary.BaseNCodec.AbstractBuilder.setLineSeparator(byte...)
now makes a defensive copy. Thanks to Gary Gregory.

  •  Avoid unnecessary String conversion in
org.apache.commons.codec.language.bm.PhoneticEngine.applyFinalRules(PhonemeBuilder,
Map). Thanks to Gary Gregory.

  •  Fix SpotBugs [ERROR] High: Potentially dangerous use of
non-short-circuit logic in
org.apache.commons.codec.language.DaitchMokotoffSoundex.cleanup(String)
[org.apache.commons.codec.language.DaitchMokotoffSoundex] At
DaitchMokotoffSoundex.java:[line 350] NS_DANGEROUS_NON_SHORT_CIRCUIT.
Thanks to Gary Gregory.

Changes

... (truncated)

Commits
  • 351cb22 Prepare for the release candidate 1.19.0 RC1
  • 0d501b6 Prepare for the next release candidate
  • d6d4b82 Refactor duplicate code
  • 6d6456c No need to exclude abstract test classes from Surefire runs
  • 22d62e4 No need to specify the default value for linkXref
  • c4daf34 No longer need to override the version of the Jacoco Maven plugin
  • 8f2b673 Remove workaround for [SUREFIRE-2253]
  • 466a61d Fix Javadoc
  • ca27bd3 Fix Checkstyle
  • 1dfb4e5 Better internal method name
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=commons-codec:commons-codec&package-manager=maven&previous-version=1.18.0&new-version=1.19.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- vector/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vector/pom.xml b/vector/pom.xml index 52ad5105ea..89e9779008 100644 --- a/vector/pom.xml +++ b/vector/pom.xml @@ -60,7 +60,7 @@ under the License. commons-codec commons-codec - 1.18.0 + 1.20.0 org.apache.arrow From 11414f652d90b56a697d26a18c81457132f502cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Nov 2025 06:50:17 +0100 Subject: [PATCH 083/271] MINOR: [CI] Bump actions/checkout from 4 to 5 (#820) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 5.
Release notes

Sourced from actions/checkout's releases.

v5.0.0

What's Changed

⚠️ Minimum Compatible Runner Version

v2.327.1
Release Notes

Make sure your runner is updated to this version or newer to use this release.

Full Changelog: https://github.com/actions/checkout/compare/v4...v5.0.0

v4.3.0

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v4...v4.3.0

v4.2.2

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v4.2.1...v4.2.2

v4.2.1

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v4.2.0...v4.2.1

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/checkout&package-manager=github_actions&previous-version=4&new-version=5)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: JB Onofré --- .github/workflows/dev.yml | 2 +- .github/workflows/dev_pr.yml | 2 +- .github/workflows/rc.yml | 20 ++++++++++---------- .github/workflows/release.yml | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 12968bbcca..27b874ddb9 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -33,7 +33,7 @@ jobs: name: "pre-commit" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml index 2f4a48d572..759383d37b 100644 --- a/.github/workflows/dev_pr.yml +++ b/.github/workflows/dev_pr.yml @@ -43,7 +43,7 @@ jobs: name: "Ensure PR format" runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@v5 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index b53812ef5e..121d188d3a 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -38,7 +38,7 @@ jobs: timeout-minutes: 5 steps: - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: submodules: recursive - name: Prepare for tag @@ -119,17 +119,17 @@ jobs: # ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ # if: github.event_name == 'schedule' - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: repository: apache/arrow path: arrow - name: Checkout apache/arrow-testing - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: repository: apache/arrow-testing path: arrow/testing - name: Checkout apache/parquet-testing - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing @@ -186,17 +186,17 @@ jobs: # ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ # if: github.event_name == 'schedule' - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: repository: apache/arrow path: arrow - name: Checkout apache/arrow-testing - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: repository: apache/arrow-testing path: arrow/testing - name: Checkout apache/parquet-testing - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing @@ -322,7 +322,7 @@ jobs: # ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ # if: github.event_name == 'schedule' - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: repository: apache/arrow path: arrow @@ -421,7 +421,7 @@ jobs: test -f jni/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll test -f jni/arrow_orc_jni/x86_64/arrow_orc_jni.dll - name: Checkout apache/arrow-testing - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: repository: apache/arrow-testing path: testing @@ -509,7 +509,7 @@ jobs: contents: write steps: - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: path: site - name: Prepare branch diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a955964cd8..d7a148bbbf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -65,7 +65,7 @@ jobs: $artifact done - name: Checkout for publishing docs - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v5 with: path: site - name: Publish docs From af37687fe501396f44561aca1ff0524c6d581a59 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Nov 2025 06:55:38 +0100 Subject: [PATCH 084/271] MINOR: [CI] Bump actions/upload-artifact from 4.6.2 to 5.0.0 (#894) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 4.6.2 to 5.0.0.
Release notes

Sourced from actions/upload-artifact's releases.

v5.0.0

What's Changed

BREAKING CHANGE: this update supports Node v24.x. This is not a breaking change per-se but we're treating it as such.

New Contributors

Full Changelog: https://github.com/actions/upload-artifact/compare/v4...v5.0.0

Commits
  • 330a01c Merge pull request #734 from actions/danwkennedy/prepare-5.0.0
  • 03f2824 Update github.dep.yml
  • 905a1ec Prepare v5.0.0
  • 2d9f9cd Merge pull request #725 from patrikpolyak/patch-1
  • 9687587 Merge branch 'main' into patch-1
  • 2848b2c Merge pull request #727 from danwkennedy/patch-1
  • 9b51177 Spell out the first use of GHES
  • cd231ca Update GHES guidance to include reference to Node 20 version
  • de65e23 Merge pull request #712 from actions/nebuk89-patch-1
  • 8747d8c Update README.md
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/upload-artifact&package-manager=github_actions&previous-version=4.6.2&new-version=5.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 121d188d3a..e21cae2d0b 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -71,7 +71,7 @@ jobs: run: | dev/release/run_rat.sh "${TAR_GZ}" - name: Upload source archive - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: release-source path: | @@ -154,7 +154,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf jni-linux-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: jni-linux-${{ matrix.platform.arch }} path: jni-linux-${{ matrix.platform.arch }}.tar.gz @@ -284,7 +284,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf jni-macos-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: jni-macos-${{ matrix.platform.arch }} path: jni-macos-${{ matrix.platform.arch }}.tar.gz @@ -368,7 +368,7 @@ jobs: shell: bash run: tar -cvzf jni-windows-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: jni-windows-${{ matrix.platform.arch }} path: jni-windows-${{ matrix.platform.arch }}.tar.gz @@ -440,12 +440,12 @@ jobs: cp -a target/site/apidocs reference tar -cvzf reference.tar.gz reference - name: Upload binaries - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: release-binaries path: binaries/* - name: Upload docs - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: reference path: reference.tar.gz @@ -483,7 +483,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf html.tar.gz -C docs/build html - name: Upload artifacts - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: release-html path: html.tar.gz From e366510ab96bdb4d18c48b26132d08382d67d33f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Nov 2025 18:13:27 +0100 Subject: [PATCH 085/271] MINOR: Bump checker.framework.version from 3.49.3 to 3.49.5 (#800) Bumps `checker.framework.version` from 3.49.3 to 3.49.5. Updates `org.checkerframework:checker-qual` from 3.49.3 to 3.49.5
Release notes

Sourced from org.checkerframework:checker-qual's releases.

Checker Framework 3.49.5

Version 3.49.5 (June 30, 2025)

User-visible changes:

The Checker Framework runs under JDK 25 -- that is, it runs on a version 25 JVM.

Closed issues:

#7093.

Checker Framework 3.49.4

Version 3.49.4 (June 2, 2025)

Closed issues:

#6740, #7013, #7038, #7070, #7082.

Changelog

Sourced from org.checkerframework:checker-qual's changelog.

Version 3.49.5 (June 30, 2025)

User-visible changes:

The Checker Framework runs under JDK 25 -- that is, it runs on a version 25 JVM.

Closed issues:

#7093.

Version 3.49.4 (June 2, 2025)

Closed issues:

#6740, #7013, #7038, #7070, #7082.

Commits
  • 7f6e237 new release 3.49.5
  • 57eee7e Prep for release.
  • 759bead Don't run inference job on JDK 11
  • 0d200f2 Produce Java 11 bytecodes for the wpi-many projects
  • 04fd6b4 Produce Java 11 bytecodes for the owning-field project
  • 9b51e58 Test under Java 25 (#7113)
  • 2403df2 Remove file SKIP-REQUIRE-JAVADOC
  • bbf907e Suppress "this-escape" warnings; in the future, carefully examine (#7130)
  • 76d3e74 Inferring type arguments is slow (#7125)
  • 3d9448e Warn about slow type-checking of method invocations (#7124)
  • Additional commits viewable in compare view

Updates `org.checkerframework:checker` from 3.49.3 to 3.49.5
Release notes

Sourced from org.checkerframework:checker's releases.

Checker Framework 3.49.5

Version 3.49.5 (June 30, 2025)

User-visible changes:

The Checker Framework runs under JDK 25 -- that is, it runs on a version 25 JVM.

Closed issues:

#7093.

Checker Framework 3.49.4

Version 3.49.4 (June 2, 2025)

Closed issues:

#6740, #7013, #7038, #7070, #7082.

Changelog

Sourced from org.checkerframework:checker's changelog.

Version 3.49.5 (June 30, 2025)

User-visible changes:

The Checker Framework runs under JDK 25 -- that is, it runs on a version 25 JVM.

Closed issues:

#7093.

Version 3.49.4 (June 2, 2025)

Closed issues:

#6740, #7013, #7038, #7070, #7082.

Commits
  • 7f6e237 new release 3.49.5
  • 57eee7e Prep for release.
  • 759bead Don't run inference job on JDK 11
  • 0d200f2 Produce Java 11 bytecodes for the wpi-many projects
  • 04fd6b4 Produce Java 11 bytecodes for the owning-field project
  • 9b51e58 Test under Java 25 (#7113)
  • 2403df2 Remove file SKIP-REQUIRE-JAVADOC
  • bbf907e Suppress "this-escape" warnings; in the future, carefully examine (#7130)
  • 76d3e74 Inferring type arguments is slow (#7125)
  • 3d9448e Warn about slow type-checking of method invocations (#7124)
  • Additional commits viewable in compare view

You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 41b82fabab..3e09f982fe 100644 --- a/pom.xml +++ b/pom.xml @@ -110,7 +110,7 @@ under the License. 10.23.0 true 2.37.0 - 3.49.3 + 3.52.0 1.5.21 none -Xdoclint:none From ca5c06bbb94343c2bf69d2397c5beb08b66e3a14 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Nov 2025 13:03:56 +0100 Subject: [PATCH 086/271] MINOR: Bump error_prone_core.version from 2.37.0 to 2.42.0 (#749) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps `error_prone_core.version` from 2.37.0 to 2.38.0. Updates `com.google.errorprone:error_prone_annotations` from 2.37.0 to 2.38.0
Release notes

Sourced from com.google.errorprone:error_prone_annotations's releases.

Error Prone 2.38.0

New checks:

Closed issues: #4924, #4897, #4995

Full changelog: https://github.com/google/error-prone/compare/v2.37.0...v2.38.0

Commits
  • a07bd3e Release Error Prone 2.38.0
  • 09fd394 Fix typo in NullTernary.md
  • 4171fd7 FindIdentifiers: find binding variables declared by enclosing or earlier if...
  • d78f515 Audit each use of ElementKind.LOCAL_VARIABLE, and add BINDING_VARIABLE if app...
  • 6f94a97 Tolerate default cases in switches as being present to handle version skew
  • 0223abb Support @LenientFormatString in LenientFormatStringValidation.
  • cb7dfaf Remove the Side enum.
  • d64c9ce Promote error prone check TestExceptionChecker to ERROR within Google (blaze ...
  • c0ce475 Move TargetType to a top-level class alongside ASTHelpers.
  • 90b8efb Allow binding to BINDING_VARIABLEs in GuardedByBinder.
  • Additional commits viewable in compare view

Updates `com.google.errorprone:error_prone_core` from 2.37.0 to 2.38.0
Release notes

Sourced from com.google.errorprone:error_prone_core's releases.

Error Prone 2.38.0

New checks:

Closed issues: #4924, #4897, #4995

Full changelog: https://github.com/google/error-prone/compare/v2.37.0...v2.38.0

Commits
  • a07bd3e Release Error Prone 2.38.0
  • 09fd394 Fix typo in NullTernary.md
  • 4171fd7 FindIdentifiers: find binding variables declared by enclosing or earlier if...
  • d78f515 Audit each use of ElementKind.LOCAL_VARIABLE, and add BINDING_VARIABLE if app...
  • 6f94a97 Tolerate default cases in switches as being present to handle version skew
  • 0223abb Support @LenientFormatString in LenientFormatStringValidation.
  • cb7dfaf Remove the Side enum.
  • d64c9ce Promote error prone check TestExceptionChecker to ERROR within Google (blaze ...
  • c0ce475 Move TargetType to a top-level class alongside ASTHelpers.
  • 90b8efb Allow binding to BINDING_VARIABLEs in GuardedByBinder.
  • Additional commits viewable in compare view

You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: JB Onofré Co-authored-by: JB Onofré --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3e09f982fe..7f070a2f01 100644 --- a/pom.xml +++ b/pom.xml @@ -109,7 +109,7 @@ under the License. 2 10.23.0 true - 2.37.0 + 2.42.0 3.52.0 1.5.21 none From ba5057a6b857ff42958ff69c479b098e61672719 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Nov 2025 14:17:01 +0100 Subject: [PATCH 087/271] MINOR: Bump org.apache.orc:orc-core from 2.1.1 to 2.2.1 (#874) Bumps org.apache.orc:orc-core from 2.1.1 to 2.2.1. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.orc:orc-core&package-manager=maven&previous-version=2.1.1&new-version=2.2.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- adapter/orc/pom.xml | 2 +- dataset/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/adapter/orc/pom.xml b/adapter/orc/pom.xml index e3ae7d5163..ef72b2de65 100644 --- a/adapter/orc/pom.xml +++ b/adapter/orc/pom.xml @@ -61,7 +61,7 @@ under the License. org.apache.orc orc-core - 2.1.1 + 2.2.1 test diff --git a/dataset/pom.xml b/dataset/pom.xml index 6e56d555b7..66233c3970 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -130,7 +130,7 @@ under the License. org.apache.orc orc-core - 2.1.1 + 2.2.1 test From 8ecbea604628679140d08a9783491ac5a4c20087 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Nov 2025 15:03:25 +0100 Subject: [PATCH 088/271] MINOR: Bump com.google.protobuf:protobuf-bom from 4.30.2 to 4.33.0 (#888) Bumps [com.google.protobuf:protobuf-bom](https://github.com/protocolbuffers/protobuf) from 4.30.2 to 4.33.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.protobuf:protobuf-bom&package-manager=maven&previous-version=4.30.2&new-version=4.33.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7f070a2f01..e402f4efe7 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,7 @@ under the License. 33.4.8-jre 4.1.127.Final 1.73.0 - 4.30.2 + 4.33.1 2.18.3 3.4.1 25.2.10 From 266dfc12151a49dd49153890eb16e97937c5667c Mon Sep 17 00:00:00 2001 From: Aleksei Starikov Date: Mon, 17 Nov 2025 01:12:50 +0100 Subject: [PATCH 089/271] GH-586: Override fixedSizeBinary method for UnionMapWriter (#885) ## What's Changed `UnionMapWriter` will null out the entire map struct entry instead of setting the value to null in: ``` childWriter.value().fixedSizeBinary().writeNull(); ``` This PR overrides the `fixedSizeBinary()` method for the `UnionMapWriter`, resolving it. In addition, it introduces the `fixedSizeBinary(int byteWidth)` which needs to be used to initialize `key` and `value` writes in `UnionMapWriter` if they have not been initialized. Closes #586. --- .../codegen/templates/UnionMapWriter.java | 23 ++ .../apache/arrow/vector/TestMapVector.java | 221 ++++++++++++++++++ 2 files changed, 244 insertions(+) diff --git a/vector/src/main/codegen/templates/UnionMapWriter.java b/vector/src/main/codegen/templates/UnionMapWriter.java index 8b2f091215..8bbf6ae0a4 100644 --- a/vector/src/main/codegen/templates/UnionMapWriter.java +++ b/vector/src/main/codegen/templates/UnionMapWriter.java @@ -243,4 +243,27 @@ public ExtensionWriter extension(ArrowType type) { return super.extension(type); } } + + public FixedSizeBinaryWriter fixedSizeBinary(int byteWidth) { + switch (mode) { + case KEY: + return entryWriter.fixedSizeBinary(MapVector.KEY_NAME, byteWidth); + case VALUE: + return entryWriter.fixedSizeBinary(MapVector.VALUE_NAME, byteWidth); + default: + return this; + } + } + + @Override + public FixedSizeBinaryWriter fixedSizeBinary() { + switch (mode) { + case KEY: + return entryWriter.fixedSizeBinary(MapVector.KEY_NAME); + case VALUE: + return entryWriter.fixedSizeBinary(MapVector.VALUE_NAME); + default: + return this; + } + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java index 1a1810d0f7..8605d250fd 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java @@ -16,10 +16,12 @@ */ package org.apache.arrow.vector; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.nio.ByteBuffer; @@ -41,6 +43,7 @@ import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; import org.apache.arrow.vector.complex.writer.FieldWriter; import org.apache.arrow.vector.holder.UuidHolder; +import org.apache.arrow.vector.holders.FixedSizeBinaryHolder; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; @@ -1359,4 +1362,222 @@ public void testCopyFromForExtensionType() throws Exception { assertEquals(u2, actualUuid); } } + + private FixedSizeBinaryHolder getFixedSizeBinaryHolder(byte[] array) { + FixedSizeBinaryHolder holder = new FixedSizeBinaryHolder(); + holder.byteWidth = array.length; + holder.buffer = allocator.buffer(array.length); + for (int i = 0; i < array.length; i++) { + holder.buffer.setByte(i, array[i]); + } + + return holder; + } + + /** + * Regression test for GH-586: UnionMapWriter.fixedSizeBinary() should properly delegate to the + * entry writer for both key and value paths. + */ + @Test + public void testFixedSizeBinaryWriter() { + try (MapVector mapVector = MapVector.empty("map_vector", allocator, false)) { + UnionMapWriter writer = mapVector.getWriter(); + writer.allocate(); + + // populate input vector with the following records + // {[11, 22] -> [32, 21]} + // {1 -> [11, 22], 2 -> [32, 21]} + // null + // {[11, 22] -> 1, [32, 21] -> 2} + // {[11, 22] -> null} + // {null -> [32, 21]} - wrong "for a given entry, the "key" is non-nullable" - todo: it + // shouldn't work. Should it? + FixedSizeBinaryHolder holder1 = getFixedSizeBinaryHolder(new byte[] {11, 22}); + FixedSizeBinaryHolder holder2 = getFixedSizeBinaryHolder(new byte[] {32, 21}); + + writer.setPosition(0); // optional + writer.startMap(); + writer.startEntry(); + writer + .key() + .fixedSizeBinary(holder1.byteWidth) + .write(holder1); // need to initialize with byteWidth - NPE otherwise + writer.value().fixedSizeBinary(holder2.byteWidth).write(holder2); + writer.endEntry(); + holder1.buffer.close(); + holder2.buffer.close(); + writer.endMap(); + + // {1 -> [11, 22], 2 -> [32, 21]} + holder1 = getFixedSizeBinaryHolder(new byte[] {11, 22}); + holder2 = getFixedSizeBinaryHolder(new byte[] {32, 21}); + writer.setPosition(1); + writer.startMap(); + writer.startEntry(); + writer.key().bigInt().writeBigInt(1); + writer.value().fixedSizeBinary().write(holder1); + writer.endEntry(); + holder1.buffer.close(); + writer.startEntry(); + writer.key().bigInt().writeBigInt(2); + writer.value().fixedSizeBinary().write(holder2); + writer.endEntry(); + writer.endMap(); + holder2.buffer.close(); + + // {[11, 22] -> 1, [32, 21] -> 2} + holder1 = getFixedSizeBinaryHolder(new byte[] {11, 22}); + holder2 = getFixedSizeBinaryHolder(new byte[] {32, 21}); + writer.setPosition(3); + writer.startMap(); + writer.startEntry(); + writer.key().fixedSizeBinary().write(holder1); + writer.value().bigInt().writeBigInt(1); + writer.endEntry(); + holder1.buffer.close(); + writer.startEntry(); + writer.key().fixedSizeBinary().write(holder2); + writer.value().bigInt().writeBigInt(2); + writer.endEntry(); + writer.endMap(); + holder2.buffer.close(); + + // {[11, 22] -> null} + holder1 = getFixedSizeBinaryHolder(new byte[] {11, 22}); + writer.setPosition(4); + writer.startMap(); + writer.startEntry(); + writer.key().fixedSizeBinary().write(holder1); + writer.endEntry(); + writer.endMap(); + holder1.buffer.close(); + + // {null -> [32, 21]} + holder2 = getFixedSizeBinaryHolder(new byte[] {32, 21}); + writer.setPosition(5); + writer.startMap(); + writer.startEntry(); + writer.value().fixedSizeBinary().write(holder2); + writer.endEntry(); + writer.endMap(); + holder2.buffer.close(); + + writer.setValueCount(6); + + // assert the output vector is correct + FieldReader reader = mapVector.getReader(); + assertTrue(reader.isSet(), "shouldn't be null"); + reader.setPosition(1); + assertTrue(reader.isSet(), "shouldn't be null"); + reader.setPosition(2); + assertFalse(reader.isSet(), "should be null"); + reader.setPosition(3); + assertTrue(reader.isSet(), "shouldn't be null"); + reader.setPosition(4); + assertTrue(reader.isSet(), "shouldn't be null"); + reader.setPosition(5); + assertTrue(reader.isSet(), "shouldn't be null"); + + /* index 0 */ + Object result = mapVector.getObject(0); + ArrayList resultSet = (ArrayList) result; + assertEquals(1, resultSet.size()); + Map resultStruct = (Map) resultSet.get(0); + assertTrue(resultStruct.containsKey(MapVector.KEY_NAME)); + assertTrue(resultStruct.containsKey(MapVector.VALUE_NAME)); + assertArrayEquals(new byte[] {11, 22}, (byte[]) resultStruct.get(MapVector.KEY_NAME)); + assertArrayEquals(new byte[] {32, 21}, (byte[]) resultStruct.get(MapVector.VALUE_NAME)); + + /* index 1 */ + result = mapVector.getObject(1); + resultSet = (ArrayList) result; + assertEquals(2, resultSet.size()); + resultStruct = (Map) resultSet.get(0); + assertEquals(1L, getResultKey(resultStruct)); + assertTrue(resultStruct.containsKey(MapVector.VALUE_NAME)); + assertArrayEquals(new byte[] {11, 22}, (byte[]) resultStruct.get(MapVector.VALUE_NAME)); + resultStruct = (Map) resultSet.get(1); + assertEquals(2L, getResultKey(resultStruct)); + assertTrue(resultStruct.containsKey(MapVector.VALUE_NAME)); + assertArrayEquals(new byte[] {32, 21}, (byte[]) resultStruct.get(MapVector.VALUE_NAME)); + + /* index 2 */ + result = mapVector.getObject(2); + assertNull(result); + + /* index 3 */ + result = mapVector.getObject(3); + resultSet = (ArrayList) result; + assertEquals(2, resultSet.size()); + resultStruct = (Map) resultSet.get(0); + assertTrue(resultStruct.containsKey(MapVector.KEY_NAME)); + assertArrayEquals(new byte[] {11, 22}, (byte[]) resultStruct.get(MapVector.KEY_NAME)); + assertEquals(1L, getResultValue(resultStruct)); + resultStruct = (Map) resultSet.get(1); + assertTrue(resultStruct.containsKey(MapVector.KEY_NAME)); + assertArrayEquals(new byte[] {32, 21}, (byte[]) resultStruct.get(MapVector.KEY_NAME)); + assertEquals(2L, getResultValue(resultStruct)); + + /* index 4 */ + result = mapVector.getObject(4); + resultSet = (ArrayList) result; + assertEquals(1, resultSet.size()); + resultStruct = (Map) resultSet.get(0); + assertTrue(resultStruct.containsKey(MapVector.KEY_NAME)); + assertArrayEquals(new byte[] {11, 22}, (byte[]) resultStruct.get(MapVector.KEY_NAME)); + assertFalse(resultStruct.containsKey(MapVector.VALUE_NAME)); + + /* index 5 */ + result = mapVector.getObject(5); + resultSet = (ArrayList) result; + assertEquals(1, resultSet.size()); + resultStruct = (Map) resultSet.get(0); + assertFalse(resultStruct.containsKey(MapVector.KEY_NAME)); + assertTrue(resultStruct.containsKey(MapVector.VALUE_NAME)); + assertArrayEquals(new byte[] {32, 21}, (byte[]) resultStruct.get(MapVector.VALUE_NAME)); + } + } + + @Test + public void testFixedSizeBinaryFirstInitialization() { + try (MapVector mapVector = MapVector.empty("map_vector", allocator, false)) { + UnionMapWriter writer = mapVector.getWriter(); + writer.allocate(); + + // populate input vector with the following records + // {[11, 22] -> [32, 21]} + FixedSizeBinaryHolder holder1 = getFixedSizeBinaryHolder(new byte[] {11, 22}); + FixedSizeBinaryHolder holder2 = getFixedSizeBinaryHolder(new byte[] {32, 21}); + + writer.setPosition(0); // optional + writer.startMap(); + writer.startEntry(); + // require byteWidth parameter for first-time initialization of `key` or `value` writers + assertThrows(NullPointerException.class, () -> writer.key().fixedSizeBinary().write(holder1)); + assertThrows( + NullPointerException.class, () -> writer.value().fixedSizeBinary().write(holder2)); + writer.key().fixedSizeBinary(holder1.byteWidth).write(holder1); + writer.value().fixedSizeBinary(holder2.byteWidth).write(holder2); + writer.endEntry(); + holder1.buffer.close(); + holder2.buffer.close(); + writer.endMap(); + + writer.setValueCount(1); + + // assert the output vector is correct + FieldReader reader = mapVector.getReader(); + assertTrue(reader.isSet(), "shouldn't be null"); + + /* index 0 */ + Object result = mapVector.getObject(0); + ArrayList resultSet = (ArrayList) result; + assertEquals(1, resultSet.size()); + Map resultStruct = (Map) resultSet.get(0); + assertTrue(resultStruct.containsKey(MapVector.KEY_NAME)); + assertTrue(resultStruct.containsKey(MapVector.VALUE_NAME)); + assertArrayEquals(new byte[] {11, 22}, (byte[]) resultStruct.get(MapVector.KEY_NAME)); + assertArrayEquals(new byte[] {32, 21}, (byte[]) resultStruct.get(MapVector.VALUE_NAME)); + } + } } From 382790df634a9237403e6c0a2195dd0405766ef2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 15:32:54 +0900 Subject: [PATCH 090/271] MINOR: [CI] Bump actions/checkout from 5 to 6 (#911) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/checkout](https://github.com/actions/checkout) from 5 to 6.
Release notes

Sourced from actions/checkout's releases.

v6.0.0

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v5.0.0...v6.0.0

v6-beta

What's Changed

Updated persist-credentials to store the credentials under $RUNNER_TEMP instead of directly in the local git config.

This requires a minimum Actions Runner version of v2.329.0 to access the persisted credentials for Docker container action scenarios.

v5.0.1

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v5...v5.0.1

Changelog

Sourced from actions/checkout's changelog.

Changelog

V6.0.0

V5.0.1

V5.0.0

V4.3.1

V4.3.0

v4.2.2

v4.2.1

v4.2.0

v4.1.7

v4.1.6

v4.1.5

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/checkout&package-manager=github_actions&previous-version=5&new-version=6)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/dev.yml | 2 +- .github/workflows/dev_pr.yml | 2 +- .github/workflows/rc.yml | 20 ++++++++++---------- .github/workflows/release.yml | 2 +- .github/workflows/test.yml | 20 ++++++++++---------- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 27b874ddb9..57432d75f1 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -33,7 +33,7 @@ jobs: name: "pre-commit" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml index 759383d37b..84740628ee 100644 --- a/.github/workflows/dev_pr.yml +++ b/.github/workflows/dev_pr.yml @@ -43,7 +43,7 @@ jobs: name: "Ensure PR format" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index e21cae2d0b..0ae4399ef1 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -38,7 +38,7 @@ jobs: timeout-minutes: 5 steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: submodules: recursive - name: Prepare for tag @@ -119,17 +119,17 @@ jobs: # ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ # if: github.event_name == 'schedule' - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: repository: apache/arrow path: arrow - name: Checkout apache/arrow-testing - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: repository: apache/arrow-testing path: arrow/testing - name: Checkout apache/parquet-testing - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing @@ -186,17 +186,17 @@ jobs: # ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ # if: github.event_name == 'schedule' - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: repository: apache/arrow path: arrow - name: Checkout apache/arrow-testing - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: repository: apache/arrow-testing path: arrow/testing - name: Checkout apache/parquet-testing - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing @@ -322,7 +322,7 @@ jobs: # ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ # if: github.event_name == 'schedule' - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: repository: apache/arrow path: arrow @@ -421,7 +421,7 @@ jobs: test -f jni/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll test -f jni/arrow_orc_jni/x86_64/arrow_orc_jni.dll - name: Checkout apache/arrow-testing - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: repository: apache/arrow-testing path: testing @@ -509,7 +509,7 @@ jobs: contents: write steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: path: site - name: Prepare branch diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d7a148bbbf..5ca9cf9b73 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -65,7 +65,7 @@ jobs: $artifact done - name: Checkout for publishing docs - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: path: site - name: Publish docs diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 65fbc262fc..8d8fcb5052 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,7 +58,7 @@ jobs: MAVEN: ${{ matrix.maven }} steps: - name: Checkout Arrow - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: 0 submodules: recursive @@ -100,7 +100,7 @@ jobs: distribution: 'temurin' java-version: ${{ matrix.jdk }} - name: Checkout Arrow - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: 0 submodules: recursive @@ -131,7 +131,7 @@ jobs: java-version: ${{ matrix.jdk }} distribution: 'temurin' - name: Checkout Arrow - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: 0 submodules: recursive @@ -152,37 +152,37 @@ jobs: timeout-minutes: 60 steps: - name: Checkout Arrow - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: 0 repository: apache/arrow submodules: recursive - name: Checkout Arrow Rust - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: repository: apache/arrow-rs path: rust - name: Checkout Arrow nanoarrow - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: repository: apache/arrow-nanoarrow path: nanoarrow - name: Checkout Arrow .NET - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: repository: apache/arrow-dotnet path: dotnet - name: Checkout Arrow Go - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: repository: apache/arrow-go path: go - name: Checkout Arrow Java - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: path: java - name: Checkout Arrow JavaScript - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: repository: apache/arrow-js path: js From 4d7cf041d0abeae5a75d1c59cfdf59e12f9db686 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Nov 2025 18:20:37 +0100 Subject: [PATCH 091/271] MINOR: Bump org.codehaus.mojo:versions-maven-plugin from 2.18.0 to 2.20.0 (#912) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.codehaus.mojo:versions-maven-plugin](https://github.com/mojohaus/versions) from 2.18.0 to 2.20.0.
Release notes

Sourced from org.codehaus.mojo:versions-maven-plugin's releases.

2.20.0

🚀 New features and improvements

🐛 Bug Fixes

📝 Documentation updates

👻 Maintenance

📦 Dependency updates

2.19.1

🐛 Bug Fixes

... (truncated)

Commits
  • 2467d99 [maven-release-plugin] prepare release 2.20.0
  • 4c240e7 Bump org.apache.commons:commons-lang3 from 3.19.0 to 3.20.0
  • 6d64537 Bump byteBuddyVersion from 1.18.0 to 1.18.1
  • 7736ca6 Bump org.codehaus.plexus:plexus-archiver from 4.10.3 to 4.10.4
  • 37a5330 Bump byteBuddyVersion from 1.17.7 to 1.18.0
  • edeb5e7 Bump commons-codec:commons-codec from 1.19.0 to 1.20.0
  • 88874e0 Bump commons-io:commons-io from 2.20.0 to 2.21.0
  • 6769f03 Bump org.codehaus.plexus:plexus-i18n from 1.0.0 to 1.1.0
  • 9c122de Bump org.codehaus.plexus:plexus-interactivity-api from 1.4 to 1.5.1
  • 3f51967 Bump org.apache.maven.plugin-testing:maven-plugin-testing-harness
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.codehaus.mojo:versions-maven-plugin&package-manager=maven&previous-version=2.18.0&new-version=2.20.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- bom/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index 655c33b813..9efde53243 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -208,7 +208,7 @@ under the License. org.codehaus.mojo versions-maven-plugin - 2.18.0 + 2.20.0 diff --git a/pom.xml b/pom.xml index e402f4efe7..79b12dcb46 100644 --- a/pom.xml +++ b/pom.xml @@ -510,7 +510,7 @@ under the License. org.codehaus.mojo versions-maven-plugin - 2.18.0 + 2.20.0 pl.project13.maven From 8577cff9257afd9dc1561cbab6d250ee374dbcaf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Nov 2025 18:33:24 +0100 Subject: [PATCH 092/271] MINOR: Bump org.bouncycastle:bcpkix-jdk18on from 1.80 to 1.82 (#919) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.bouncycastle:bcpkix-jdk18on](https://github.com/bcgit/bc-java) from 1.80 to 1.82.
Changelog

Sourced from org.bouncycastle:bcpkix-jdk18on's changelog.

2.1.1 Version Release: 1.83 Date:      TBD

2.2.1 Version Release: 1.82 Date:      2025, 17th September.

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.bouncycastle:bcpkix-jdk18on&package-manager=maven&previous-version=1.80&new-version=1.82)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index d8e012101c..965e071e72 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -140,7 +140,7 @@ under the License. org.bouncycastle bcpkix-jdk18on - 1.80 + 1.82 From 13f1bd28b5a5fa9f6c2f8b52c108d6bf13bc0e63 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Nov 2025 19:05:52 +0100 Subject: [PATCH 093/271] MINOR: Bump org.apache.drill.tools:drill-fmpp-maven-plugin from 1.21.2 to 1.22.0 (#918) Bumps org.apache.drill.tools:drill-fmpp-maven-plugin from 1.21.2 to 1.22.0. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.drill.tools:drill-fmpp-maven-plugin&package-manager=maven&previous-version=1.21.2&new-version=1.22.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 79b12dcb46..5d2567beef 100644 --- a/pom.xml +++ b/pom.xml @@ -525,7 +525,7 @@ under the License. org.apache.drill.tools drill-fmpp-maven-plugin - 1.21.2 + 1.22.0 From 82baf1828df6e4ed40e8f2b62a3648235ffca76b Mon Sep 17 00:00:00 2001 From: Joana Hrotko Date: Fri, 28 Nov 2025 12:05:53 +0000 Subject: [PATCH 094/271] GH-825: Add UUID canonical extension type (#903) --- .../arrow/vector/BaseFixedWidthVector.java | 1 + .../apache/arrow/vector/FixedWidthVector.java | 3 + .../org/apache/arrow/vector/UuidVector.java | 481 ++++++++++++++++++ .../impl/ExtensionTypeWriterFactory.java | 4 +- .../vector/complex/impl/UuidReaderImpl.java | 35 +- .../complex/impl/UuidWriterFactory.java | 14 + .../vector/complex/impl/UuidWriterImpl.java | 41 +- .../arrow/vector/extension/UuidType.java | 109 ++++ .../vector/holders/NullableUuidHolder.java | 35 ++ .../arrow/vector/holders}/UuidHolder.java | 21 +- .../apache/arrow/vector/util/UuidUtility.java | 77 +++ .../apache/arrow/vector/TestListVector.java | 18 +- .../apache/arrow/vector/TestMapVector.java | 18 +- .../apache/arrow/vector/TestStructVector.java | 2 +- .../org/apache/arrow/vector/TestUtils.java | 11 + .../org/apache/arrow/vector/TestUuidType.java | 275 ++++++++++ .../apache/arrow/vector/TestUuidVector.java | 451 ++++++++++++++++ .../org/apache/arrow/vector/UuidVector.java | 127 ----- .../complex/impl/TestComplexCopier.java | 2 +- .../complex/impl/TestPromotableWriter.java | 2 +- .../complex/writer/TestComplexWriter.java | 8 +- .../complex/writer/TestSimpleWriter.java | 40 -- .../vector/types/pojo/TestExtensionType.java | 18 +- .../arrow/vector/types/pojo/UuidType.java | 60 --- 24 files changed, 1571 insertions(+), 282 deletions(-) create mode 100644 vector/src/main/java/org/apache/arrow/vector/UuidVector.java rename vector/src/{test => main}/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java (61%) rename vector/src/{test => main}/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java (72%) rename vector/src/{test => main}/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java (51%) create mode 100644 vector/src/main/java/org/apache/arrow/vector/extension/UuidType.java create mode 100644 vector/src/main/java/org/apache/arrow/vector/holders/NullableUuidHolder.java rename vector/src/{test/java/org/apache/arrow/vector/holder => main/java/org/apache/arrow/vector/holders}/UuidHolder.java (62%) create mode 100644 vector/src/main/java/org/apache/arrow/vector/util/UuidUtility.java create mode 100644 vector/src/test/java/org/apache/arrow/vector/TestUuidType.java create mode 100644 vector/src/test/java/org/apache/arrow/vector/TestUuidVector.java delete mode 100644 vector/src/test/java/org/apache/arrow/vector/UuidVector.java delete mode 100644 vector/src/test/java/org/apache/arrow/vector/types/pojo/UuidType.java diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java index f6e2a3b225..df1ac74f9b 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java @@ -70,6 +70,7 @@ public BaseFixedWidthVector(Field field, final BufferAllocator allocator, final refreshValueCapacity(); } + @Override public int getTypeWidth() { return typeWidth; } diff --git a/vector/src/main/java/org/apache/arrow/vector/FixedWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/FixedWidthVector.java index e22a973f3b..61a5574898 100644 --- a/vector/src/main/java/org/apache/arrow/vector/FixedWidthVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/FixedWidthVector.java @@ -31,4 +31,7 @@ public interface FixedWidthVector extends ElementAddressableVector { /** Zero out the underlying buffer backing this vector. */ void zeroVector(); + + /** Get the width of the type in bytes. */ + int getTypeWidth(); } diff --git a/vector/src/main/java/org/apache/arrow/vector/UuidVector.java b/vector/src/main/java/org/apache/arrow/vector/UuidVector.java new file mode 100644 index 0000000000..c662a6e064 --- /dev/null +++ b/vector/src/main/java/org/apache/arrow/vector/UuidVector.java @@ -0,0 +1,481 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector; + +import static org.apache.arrow.vector.extension.UuidType.UUID_BYTE_WIDTH; + +import java.nio.ByteBuffer; +import java.util.UUID; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.util.ArrowBufPointer; +import org.apache.arrow.memory.util.hash.ArrowBufHasher; +import org.apache.arrow.vector.complex.impl.UuidReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.extension.UuidType; +import org.apache.arrow.vector.holders.NullableUuidHolder; +import org.apache.arrow.vector.holders.UuidHolder; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.TransferPair; +import org.apache.arrow.vector.util.UuidUtility; + +/** + * Vector implementation for UUID values using {@link UuidType}. + * + *

Supports setting and retrieving UUIDs with efficient storage and nullable value handling. + * + *

Usage: + * + * 

{@code
+ * UuidVector vector = new UuidVector("uuid_col", allocator);
+ * vector.set(0, UUID.randomUUID());
+ * UUID value = vector.getObject(0);
+ * }
+ * + * @see UuidType + * @see UuidHolder + * @see NullableUuidHolder + */ +public class UuidVector extends ExtensionTypeVector + implements ValueIterableVector, FixedWidthVector { + private final Field field; + + /** The fixed byte width of UUID values (16 bytes). */ + public static final int TYPE_WIDTH = UUID_BYTE_WIDTH; + + /** + * Constructs a UUID vector with the given name, allocator, and underlying vector. + * + * @param name the name of the vector + * @param allocator the buffer allocator + * @param underlyingVector the underlying FixedSizeBinaryVector for storage + */ + public UuidVector( + String name, BufferAllocator allocator, FixedSizeBinaryVector underlyingVector) { + super(name, allocator, underlyingVector); + this.field = new Field(name, FieldType.nullable(new UuidType()), null); + } + + /** + * Constructs a UUID vector with the given name, field type, allocator, and underlying vector. + * + * @param name the name of the vector + * @param fieldType the field type (should contain UuidType) + * @param allocator the buffer allocator + * @param underlyingVector the underlying FixedSizeBinaryVector for storage + */ + public UuidVector( + String name, + FieldType fieldType, + BufferAllocator allocator, + FixedSizeBinaryVector underlyingVector) { + super(name, allocator, underlyingVector); + this.field = new Field(name, fieldType, null); + } + + /** + * Constructs a UUID vector with the given name and allocator. + * + *

Creates a new underlying FixedSizeBinaryVector with 16-byte width. + * + * @param name the name of the vector + * @param allocator the buffer allocator + */ + public UuidVector(String name, BufferAllocator allocator) { + super(name, allocator, new FixedSizeBinaryVector(name, allocator, UUID_BYTE_WIDTH)); + this.field = new Field(name, FieldType.nullable(new UuidType()), null); + } + + /** + * Constructs a UUID vector from a field and allocator. + * + * @param field the field definition (should contain UuidType) + * @param allocator the buffer allocator + */ + public UuidVector(Field field, BufferAllocator allocator) { + super( + field.getName(), + allocator, + new FixedSizeBinaryVector(field.getName(), allocator, UUID_BYTE_WIDTH)); + this.field = field; + } + + @Override + public UUID getObject(int index) { + if (isSet(index) == 0) { + return null; + } + final ByteBuffer bb = ByteBuffer.wrap(getUnderlyingVector().getObject(index)); + return new UUID(bb.getLong(), bb.getLong()); + } + + @Override + public int hashCode(int index) { + return hashCode(index, null); + } + + @Override + public int hashCode(int index, ArrowBufHasher hasher) { + return getUnderlyingVector().hashCode(index, hasher); + } + + /** + * Checks if the value at the given index is set (non-null). + * + * @param index the index to check + * @return 1 if the value is set, 0 if null + */ + public int isSet(int index) { + return getUnderlyingVector().isSet(index); + } + + /** + * Gets the UUID value at the given index as an ArrowBuf. + * + * @param index the index to retrieve + * @return a buffer slice containing the 16-byte UUID + * @throws IllegalStateException if the value at the index is null and null checking is enabled + */ + public ArrowBuf get(int index) throws IllegalStateException { + if (NullCheckingForGet.NULL_CHECKING_ENABLED && this.isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } else { + return getBufferSlicePostNullCheck(index); + } + } + + /** + * Reads the UUID value at the given index into a NullableUuidHolder. + * + * @param index the index to read from + * @param holder the holder to populate with the UUID data + */ + public void get(int index, NullableUuidHolder holder) { + if (NullCheckingForGet.NULL_CHECKING_ENABLED && this.isSet(index) == 0) { + holder.isSet = 0; + } else { + holder.isSet = 1; + holder.buffer = getBufferSlicePostNullCheck(index); + } + } + + /** + * Reads the UUID value at the given index into a UuidHolder. + * + * @param index the index to read from + * @param holder the holder to populate with the UUID data + */ + public void get(int index, UuidHolder holder) { + holder.isSet = 1; + holder.buffer = getBufferSlicePostNullCheck(index); + } + + /** + * Sets the UUID value at the given index. + * + * @param index the index to set + * @param value the UUID value to set, or null to set a null value + */ + public void set(int index, UUID value) { + if (value != null) { + set(index, UuidUtility.getBytesFromUUID(value)); + } else { + getUnderlyingVector().setNull(index); + } + } + + /** + * Sets the UUID value at the given index from a UuidHolder. + * + * @param index the index to set + * @param holder the holder containing the UUID data + */ + public void set(int index, UuidHolder holder) { + this.set(index, holder.isSet, holder.buffer); + } + + /** + * Sets the UUID value at the given index from a NullableUuidHolder. + * + * @param index the index to set + * @param holder the holder containing the UUID data + */ + public void set(int index, NullableUuidHolder holder) { + this.set(index, holder.isSet, holder.buffer); + } + + /** + * Sets the UUID value at the given index with explicit null flag. + * + * @param index the index to set + * @param isSet 1 if the value is set, 0 if null + * @param buffer the buffer containing the 16-byte UUID data + */ + public void set(int index, int isSet, ArrowBuf buffer) { + getUnderlyingVector().set(index, isSet, buffer); + } + + /** + * Sets the UUID value at the given index from an ArrowBuf. + * + * @param index the index to set + * @param value the buffer containing the 16-byte UUID data + */ + public void set(int index, ArrowBuf value) { + getUnderlyingVector().set(index, value); + } + + /** + * Sets the UUID value at the given index by copying from a source buffer. + * + * @param index the index to set + * @param source the source buffer to copy from + * @param sourceOffset the offset in the source buffer where the UUID data starts + */ + public void set(int index, ArrowBuf source, int sourceOffset) { + // Copy bytes from source buffer to target vector data buffer + ArrowBuf dataBuffer = getUnderlyingVector().getDataBuffer(); + dataBuffer.setBytes((long) index * UUID_BYTE_WIDTH, source, sourceOffset, UUID_BYTE_WIDTH); + getUnderlyingVector().setIndexDefined(index); + } + + /** + * Sets the UUID value at the given index from a byte array. + * + * @param index the index to set + * @param value the 16-byte array containing the UUID data + */ + public void set(int index, byte[] value) { + getUnderlyingVector().set(index, value); + } + + /** + * Sets the UUID value at the given index, expanding capacity if needed. + * + * @param index the index to set + * @param value the UUID value to set, or null to set a null value + */ + public void setSafe(int index, UUID value) { + if (value != null) { + setSafe(index, UuidUtility.getBytesFromUUID(value)); + } else { + getUnderlyingVector().setNull(index); + } + } + + /** + * Sets the UUID value at the given index from a NullableUuidHolder, expanding capacity if needed. + * + * @param index the index to set + * @param holder the holder containing the UUID data, or null to set a null value + */ + public void setSafe(int index, NullableUuidHolder holder) { + if (holder != null) { + getUnderlyingVector().setSafe(index, holder.isSet, holder.buffer); + } else { + getUnderlyingVector().setNull(index); + } + } + + /** + * Sets the UUID value at the given index from a UuidHolder, expanding capacity if needed. + * + * @param index the index to set + * @param holder the holder containing the UUID data, or null to set a null value + */ + public void setSafe(int index, UuidHolder holder) { + if (holder != null) { + getUnderlyingVector().setSafe(index, holder.isSet, holder.buffer); + } else { + getUnderlyingVector().setNull(index); + } + } + + /** + * Sets the UUID value at the given index from a byte array, expanding capacity if needed. + * + * @param index the index to set + * @param value the 16-byte array containing the UUID data + */ + public void setSafe(int index, byte[] value) { + getUnderlyingVector().setIndexDefined(index); + getUnderlyingVector().setSafe(index, value); + } + + /** + * Sets the UUID value at the given index from an ArrowBuf, expanding capacity if needed. + * + * @param index the index to set + * @param value the buffer containing the 16-byte UUID data + */ + public void setSafe(int index, ArrowBuf value) { + getUnderlyingVector().setSafe(index, value); + } + + @Override + public void copyFrom(int fromIndex, int thisIndex, ValueVector from) { + getUnderlyingVector() + .copyFromSafe(fromIndex, thisIndex, ((UuidVector) from).getUnderlyingVector()); + } + + @Override + public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { + getUnderlyingVector() + .copyFromSafe(fromIndex, thisIndex, ((UuidVector) from).getUnderlyingVector()); + } + + @Override + public Field getField() { + return field; + } + + @Override + public ArrowBufPointer getDataPointer(int i) { + return getUnderlyingVector().getDataPointer(i); + } + + @Override + public ArrowBufPointer getDataPointer(int i, ArrowBufPointer arrowBufPointer) { + return getUnderlyingVector().getDataPointer(i, arrowBufPointer); + } + + @Override + public void allocateNew(int valueCount) { + getUnderlyingVector().allocateNew(valueCount); + } + + @Override + public void zeroVector() { + getUnderlyingVector().zeroVector(); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((UuidVector) to); + } + + @Override + protected FieldReader getReaderImpl() { + return new UuidReaderImpl(this); + } + + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator) { + return new TransferImpl(field, allocator); + } + + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator, CallBack callBack) { + return getTransferPair(field, allocator); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return getTransferPair(ref, allocator); + } + + @Override + public TransferPair getTransferPair(BufferAllocator allocator) { + return getTransferPair(this.getField().getName(), allocator); + } + + private ArrowBuf getBufferSlicePostNullCheck(int index) { + return getUnderlyingVector() + .getDataBuffer() + .slice((long) index * UUID_BYTE_WIDTH, UUID_BYTE_WIDTH); + } + + @Override + public int getTypeWidth() { + return getUnderlyingVector().getTypeWidth(); + } + + /** {@link TransferPair} for {@link UuidVector}. */ + public class TransferImpl implements TransferPair { + UuidVector to; + + /** + * Constructs a transfer pair with the given target vector. + * + * @param to the target UUID vector + */ + public TransferImpl(UuidVector to) { + this.to = to; + } + + /** + * Constructs a transfer pair, creating a new target vector from the field and allocator. + * + * @param field the field definition for the target vector + * @param allocator the buffer allocator for the target vector + */ + public TransferImpl(Field field, BufferAllocator allocator) { + this.to = new UuidVector(field, allocator); + } + + /** + * Constructs a transfer pair, creating a new target vector with the given name and allocator. + * + * @param ref the name for the target vector + * @param allocator the buffer allocator for the target vector + */ + public TransferImpl(String ref, BufferAllocator allocator) { + this.to = new UuidVector(ref, allocator); + } + + /** + * Gets the target vector of this transfer pair. + * + * @return the target UUID vector + */ + public UuidVector getTo() { + return this.to; + } + + /** Transfers ownership of data from the source vector to the target vector. */ + public void transfer() { + getUnderlyingVector().transferTo(to.getUnderlyingVector()); + } + + /** + * Splits and transfers a range of values from the source vector to the target vector. + * + * @param startIndex the starting index in the source vector + * @param length the number of values to transfer + */ + public void splitAndTransfer(int startIndex, int length) { + getUnderlyingVector().splitAndTransferTo(startIndex, length, to.getUnderlyingVector()); + } + + /** + * Copies a value from the source vector to the target vector, expanding capacity if needed. + * + * @param fromIndex the index in the source vector + * @param toIndex the index in the target vector + */ + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, (ValueVector) UuidVector.this); + } + } +} diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/ExtensionTypeWriterFactory.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/ExtensionTypeWriterFactory.java index 09f0314c5f..a01d591555 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/impl/ExtensionTypeWriterFactory.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/ExtensionTypeWriterFactory.java @@ -20,8 +20,8 @@ import org.apache.arrow.vector.complex.writer.FieldWriter; /** - * A factory interface for creating instances of {@link ExtensionTypeWriter}. This factory allows - * configuring writer implementations for specific {@link ExtensionTypeVector}. + * A factory interface for creating instances of {@link AbstractExtensionTypeWriter}. This factory + * allows configuring writer implementations for specific {@link ExtensionTypeVector}. * * @param the type of writer implementation for a specific {@link ExtensionTypeVector}. */ diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java similarity index 61% rename from vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java rename to vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java index 6b98d3b340..bb35b960d3 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java @@ -17,15 +17,30 @@ package org.apache.arrow.vector.complex.impl; import org.apache.arrow.vector.UuidVector; -import org.apache.arrow.vector.holder.UuidHolder; import org.apache.arrow.vector.holders.ExtensionHolder; +import org.apache.arrow.vector.holders.NullableUuidHolder; +import org.apache.arrow.vector.holders.UuidHolder; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.Field; +/** + * Reader implementation for {@link UuidVector}. + * + *

Provides methods to read UUID values from a vector, including support for reading into {@link + * UuidHolder} and retrieving values as {@link java.util.UUID} objects. + * + * @see UuidVector + * @see org.apache.arrow.vector.extension.UuidType + */ public class UuidReaderImpl extends AbstractFieldReader { private final UuidVector vector; + /** + * Constructs a reader for the given UUID vector. + * + * @param vector the UUID vector to read from + */ public UuidReaderImpl(UuidVector vector) { super(); this.vector = vector; @@ -48,12 +63,26 @@ public boolean isSet() { @Override public void read(ExtensionHolder holder) { - vector.get(idx(), (UuidHolder) holder); + if (holder instanceof UuidHolder) { + vector.get(idx(), (UuidHolder) holder); + } else if (holder instanceof NullableUuidHolder) { + vector.get(idx(), (NullableUuidHolder) holder); + } else { + throw new IllegalArgumentException( + "Unsupported holder type for UuidReader: " + holder.getClass()); + } } @Override public void read(int arrayIndex, ExtensionHolder holder) { - vector.get(arrayIndex, (UuidHolder) holder); + if (holder instanceof UuidHolder) { + vector.get(arrayIndex, (UuidHolder) holder); + } else if (holder instanceof NullableUuidHolder) { + vector.get(arrayIndex, (NullableUuidHolder) holder); + } else { + throw new IllegalArgumentException( + "Unsupported holder type for UuidReader: " + holder.getClass()); + } } @Override diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java similarity index 72% rename from vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java rename to vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java index 1b1bf4e6e4..35988129cb 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java @@ -19,8 +19,22 @@ import org.apache.arrow.vector.ExtensionTypeVector; import org.apache.arrow.vector.UuidVector; +/** + * Factory for creating {@link UuidWriterImpl} instances. + * + *

This factory is used to create writers for UUID extension type vectors. + * + * @see UuidWriterImpl + * @see org.apache.arrow.vector.extension.UuidType + */ public class UuidWriterFactory implements ExtensionTypeWriterFactory { + /** + * Creates a writer implementation for the given extension type vector. + * + * @param extensionTypeVector the vector to create a writer for + * @return a {@link UuidWriterImpl} if the vector is a {@link UuidVector}, null otherwise + */ @Override public AbstractFieldWriter getWriterImpl(ExtensionTypeVector extensionTypeVector) { if (extensionTypeVector instanceof UuidVector) { diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java similarity index 51% rename from vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java rename to vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java index 68029b1df5..8a78add11c 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java @@ -16,32 +16,53 @@ */ package org.apache.arrow.vector.complex.impl; -import java.nio.ByteBuffer; -import java.util.UUID; +import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.vector.UuidVector; -import org.apache.arrow.vector.holder.UuidHolder; import org.apache.arrow.vector.holders.ExtensionHolder; +import org.apache.arrow.vector.holders.NullableUuidHolder; +import org.apache.arrow.vector.holders.UuidHolder; +/** + * Writer implementation for {@link UuidVector}. + * + *

Supports writing UUID values in multiple formats: {@link java.util.UUID}, byte arrays, and + * {@link ArrowBuf}. Also handles {@link UuidHolder} and {@link NullableUuidHolder}. + * + * @see UuidVector + * @see org.apache.arrow.vector.extension.UuidType + */ public class UuidWriterImpl extends AbstractExtensionTypeWriter { + /** + * Constructs a writer for the given UUID vector. + * + * @param vector the UUID vector to write to + */ public UuidWriterImpl(UuidVector vector) { super(vector); } @Override public void writeExtension(Object value) { - UUID uuid = (UUID) value; - ByteBuffer bb = ByteBuffer.allocate(16); - bb.putLong(uuid.getMostSignificantBits()); - bb.putLong(uuid.getLeastSignificantBits()); - vector.setSafe(getPosition(), bb.array()); + if (value instanceof byte[]) { + vector.setSafe(getPosition(), (byte[]) value); + } else if (value instanceof ArrowBuf) { + vector.setSafe(getPosition(), (ArrowBuf) value); + } else if (value instanceof java.util.UUID) { + vector.setSafe(getPosition(), (java.util.UUID) value); + } else { + throw new IllegalArgumentException("Unsupported value type for UUID: " + value.getClass()); + } vector.setValueCount(getPosition() + 1); } @Override public void write(ExtensionHolder holder) { - UuidHolder uuidHolder = (UuidHolder) holder; - vector.setSafe(getPosition(), uuidHolder.value); + if (holder instanceof UuidHolder) { + vector.setSafe(getPosition(), (UuidHolder) holder); + } else if (holder instanceof NullableUuidHolder) { + vector.setSafe(getPosition(), (NullableUuidHolder) holder); + } vector.setValueCount(getPosition() + 1); } } diff --git a/vector/src/main/java/org/apache/arrow/vector/extension/UuidType.java b/vector/src/main/java/org/apache/arrow/vector/extension/UuidType.java new file mode 100644 index 0000000000..f0f2636c82 --- /dev/null +++ b/vector/src/main/java/org/apache/arrow/vector/extension/UuidType.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.extension; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.FixedSizeBinaryVector; +import org.apache.arrow.vector.UuidVector; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ArrowType.ExtensionType; +import org.apache.arrow.vector.types.pojo.ExtensionTypeRegistry; +import org.apache.arrow.vector.types.pojo.FieldType; + +/** + * Extension type for UUID (Universally Unique Identifier) values. + * + *

UUIDs are stored as 16-byte fixed-size binary values. This extension type provides a + * standardized way to represent UUIDs in Arrow, making them interoperable across different systems + * and languages.π + * + *

The extension name is "arrow.uuid" and it uses {@link ArrowType.FixedSizeBinary} with 16 bytes + * as the storage type. + * + *

Usage: + * + * 

{@code
+ * UuidVector vector = new UuidVector("uuid_col", allocator);
+ * vector.set(0, UUID.randomUUID());
+ * UUID value = vector.getObject(0);
+ * }
+ * + * @see UuidVector + * @see org.apache.arrow.vector.holders.UuidHolder + * @see org.apache.arrow.vector.holders.NullableUuidHolder + */ +public class UuidType extends ExtensionType { + /** Singleton instance of UuidType. */ + public static final UuidType INSTANCE = new UuidType(); + + /** Extension name registered in the Arrow extension type registry. */ + public static final String EXTENSION_NAME = "arrow.uuid"; + + /** Number of bytes used to store a UUID (128 bits = 16 bytes). */ + public static final int UUID_BYTE_WIDTH = 16; + + /** Number of characters in the standard UUID string representation (with hyphens). */ + public static final int UUID_STRING_WIDTH = 36; + + /** Storage type for UUID: FixedSizeBinary(16). */ + public static final ArrowType STORAGE_TYPE = new ArrowType.FixedSizeBinary(UUID_BYTE_WIDTH); + + static { + ExtensionTypeRegistry.register(INSTANCE); + } + + @Override + public ArrowType storageType() { + return STORAGE_TYPE; + } + + @Override + public String extensionName() { + return EXTENSION_NAME; + } + + @Override + public boolean extensionEquals(ExtensionType other) { + return other instanceof UuidType; + } + + @Override + public ArrowType deserialize(ArrowType storageType, String serializedData) { + if (!storageType.equals(storageType())) { + throw new UnsupportedOperationException( + "Cannot construct UuidType from underlying type " + storageType); + } + return INSTANCE; + } + + @Override + public String serialize() { + return ""; + } + + @Override + public boolean isComplex() { + return false; + } + + @Override + public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator) { + return new UuidVector( + name, fieldType, allocator, new FixedSizeBinaryVector(name, allocator, UUID_BYTE_WIDTH)); + } +} diff --git a/vector/src/main/java/org/apache/arrow/vector/holders/NullableUuidHolder.java b/vector/src/main/java/org/apache/arrow/vector/holders/NullableUuidHolder.java new file mode 100644 index 0000000000..e5398d82cf --- /dev/null +++ b/vector/src/main/java/org/apache/arrow/vector/holders/NullableUuidHolder.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.holders; + +import org.apache.arrow.memory.ArrowBuf; + +/** + * Value holder for nullable UUID values. + * + *

The {@code isSet} field controls nullability: when {@code isSet = 1}, the holder contains a + * valid UUID in {@code buffer}; when {@code isSet = 0}, the holder represents a null value and + * {@code buffer} should not be accessed. + * + * @see UuidHolder + * @see org.apache.arrow.vector.UuidVector + * @see org.apache.arrow.vector.extension.UuidType + */ +public class NullableUuidHolder extends ExtensionHolder { + /** Buffer containing 16-byte UUID data. */ + public ArrowBuf buffer; +} diff --git a/vector/src/test/java/org/apache/arrow/vector/holder/UuidHolder.java b/vector/src/main/java/org/apache/arrow/vector/holders/UuidHolder.java similarity index 62% rename from vector/src/test/java/org/apache/arrow/vector/holder/UuidHolder.java rename to vector/src/main/java/org/apache/arrow/vector/holders/UuidHolder.java index 207b0951a7..484e05c24b 100644 --- a/vector/src/test/java/org/apache/arrow/vector/holder/UuidHolder.java +++ b/vector/src/main/java/org/apache/arrow/vector/holders/UuidHolder.java @@ -14,10 +14,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.arrow.vector.holder; +package org.apache.arrow.vector.holders; -import org.apache.arrow.vector.holders.ExtensionHolder; +import org.apache.arrow.memory.ArrowBuf; +/** + * Value holder for non-nullable UUID values. + * + *

Contains a 16-byte UUID in {@code buffer} with {@code isSet} always 1. + * + * @see NullableUuidHolder + * @see org.apache.arrow.vector.UuidVector + * @see org.apache.arrow.vector.extension.UuidType + */ public class UuidHolder extends ExtensionHolder { - public byte[] value; + /** Buffer containing 16-byte UUID data. */ + public ArrowBuf buffer; + + /** Constructs a UuidHolder with isSet = 1. */ + public UuidHolder() { + this.isSet = 1; + } } diff --git a/vector/src/main/java/org/apache/arrow/vector/util/UuidUtility.java b/vector/src/main/java/org/apache/arrow/vector/util/UuidUtility.java new file mode 100644 index 0000000000..a1b0b54579 --- /dev/null +++ b/vector/src/main/java/org/apache/arrow/vector/util/UuidUtility.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.util; + +import static org.apache.arrow.vector.extension.UuidType.UUID_BYTE_WIDTH; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.UUID; +import org.apache.arrow.memory.ArrowBuf; + +/** + * Utility class for UUID conversions and operations. + * + *

Provides methods to convert between {@link UUID} objects and byte representations used in + * Arrow vectors. + * + * @see org.apache.arrow.vector.UuidVector + * @see org.apache.arrow.vector.extension.UuidType + */ +public class UuidUtility { + /** + * Converts a UUID to a 16-byte array. + * + *

The UUID is stored in big-endian byte order, with the most significant bits first. + * + * @param uuid the UUID to convert + * @return a 16-byte array representing the UUID + */ + public static byte[] getBytesFromUUID(UUID uuid) { + byte[] result = new byte[16]; + long msb = uuid.getMostSignificantBits(); + long lsb = uuid.getLeastSignificantBits(); + for (int i = 15; i >= 8; i--) { + result[i] = (byte) (lsb & 0xFF); + lsb >>= 8; + } + for (int i = 7; i >= 0; i--) { + result[i] = (byte) (msb & 0xFF); + msb >>= 8; + } + return result; + } + + /** + * Constructs a UUID from bytes stored in an ArrowBuf at the specified index. + * + *

Reads 16 bytes from the buffer starting at the given index and interprets them as a UUID in + * big-endian byte order. + * + * @param buffer the buffer containing UUID data + * @param index the byte offset in the buffer where the UUID starts + * @return the UUID constructed from the buffer data + */ + public static UUID uuidFromArrowBuf(ArrowBuf buffer, long index) { + ByteBuffer buf = buffer.nioBuffer(index, UUID_BYTE_WIDTH); + + buf.order(ByteOrder.BIG_ENDIAN); + long mostSigBits = buf.getLong(0); + long leastSigBits = buf.getLong(Long.BYTES); + return new UUID(mostSigBits, leastSigBits); + } +} diff --git a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index c6c7c5c862..41a95a8d11 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -24,7 +24,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -39,17 +38,18 @@ import org.apache.arrow.vector.complex.impl.UuidWriterFactory; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.complex.writer.BaseWriter.ExtensionWriter; -import org.apache.arrow.vector.holder.UuidHolder; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.holders.DurationHolder; import org.apache.arrow.vector.holders.FixedSizeBinaryHolder; import org.apache.arrow.vector.holders.TimeStampMilliTZHolder; +import org.apache.arrow.vector.holders.UuidHolder; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.types.pojo.UuidType; import org.apache.arrow.vector.util.TransferPair; +import org.apache.arrow.vector.util.UuidUtility; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -1259,14 +1259,12 @@ public void testListVectorReaderForExtensionType() throws Exception { FieldReader uuidReader = reader.reader(); UuidHolder holder = new UuidHolder(); uuidReader.read(holder); - ByteBuffer bb = ByteBuffer.wrap(holder.value); - UUID actualUuid = new UUID(bb.getLong(), bb.getLong()); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); assertEquals(u1, actualUuid); reader.next(); uuidReader = reader.reader(); uuidReader.read(holder); - bb = ByteBuffer.wrap(holder.value); - actualUuid = new UUID(bb.getLong(), bb.getLong()); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); assertEquals(u2, actualUuid); } } @@ -1302,14 +1300,12 @@ public void testCopyFromForExtensionType() throws Exception { FieldReader uuidReader = reader.reader(); UuidHolder holder = new UuidHolder(); uuidReader.read(holder); - ByteBuffer bb = ByteBuffer.wrap(holder.value); - UUID actualUuid = new UUID(bb.getLong(), bb.getLong()); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); assertEquals(u1, actualUuid); reader.next(); uuidReader = reader.reader(); uuidReader.read(holder); - bb = ByteBuffer.wrap(holder.value); - actualUuid = new UUID(bb.getLong(), bb.getLong()); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); assertEquals(u2, actualUuid); } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java index 8605d250fd..df8f338f45 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java @@ -24,7 +24,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -42,15 +41,16 @@ import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; import org.apache.arrow.vector.complex.writer.FieldWriter; -import org.apache.arrow.vector.holder.UuidHolder; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.holders.FixedSizeBinaryHolder; +import org.apache.arrow.vector.holders.UuidHolder; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.types.pojo.UuidType; import org.apache.arrow.vector.util.JsonStringArrayList; import org.apache.arrow.vector.util.TransferPair; +import org.apache.arrow.vector.util.UuidUtility; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -1304,14 +1304,12 @@ public void testMapVectorWithExtensionType() throws Exception { FieldReader uuidReader = mapReader.value(); UuidHolder holder = new UuidHolder(); uuidReader.read(holder); - ByteBuffer bb = ByteBuffer.wrap(holder.value); - UUID actualUuid = new UUID(bb.getLong(), bb.getLong()); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); assertEquals(u1, actualUuid); mapReader.next(); uuidReader = mapReader.value(); uuidReader.read(holder); - bb = ByteBuffer.wrap(holder.value); - actualUuid = new UUID(bb.getLong(), bb.getLong()); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); assertEquals(u2, actualUuid); } } @@ -1351,14 +1349,12 @@ public void testCopyFromForExtensionType() throws Exception { FieldReader uuidReader = mapReader.value(); UuidHolder holder = new UuidHolder(); uuidReader.read(holder); - ByteBuffer bb = ByteBuffer.wrap(holder.value); - UUID actualUuid = new UUID(bb.getLong(), bb.getLong()); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); assertEquals(u1, actualUuid); mapReader.next(); uuidReader = mapReader.value(); uuidReader.read(holder); - bb = ByteBuffer.wrap(holder.value); - actualUuid = new UUID(bb.getLong(), bb.getLong()); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); assertEquals(u2, actualUuid); } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java b/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java index d40af9ae89..b8abfe1ef6 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java @@ -35,6 +35,7 @@ import org.apache.arrow.vector.complex.impl.NullableStructWriter; import org.apache.arrow.vector.complex.writer.Float8Writer; import org.apache.arrow.vector.complex.writer.IntWriter; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.holders.ComplexHolder; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.Types.MinorType; @@ -42,7 +43,6 @@ import org.apache.arrow.vector.types.pojo.ArrowType.Struct; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.types.pojo.UuidType; import org.apache.arrow.vector.util.TransferPair; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; diff --git a/vector/src/test/java/org/apache/arrow/vector/TestUtils.java b/vector/src/test/java/org/apache/arrow/vector/TestUtils.java index 82295f8037..c28751aa58 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestUtils.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestUtils.java @@ -20,6 +20,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ExtensionTypeRegistry; import org.apache.arrow.vector.types.pojo.FieldType; public class TestUtils { @@ -62,4 +63,14 @@ public static String generateRandomString(int length) { } return sb.toString(); } + + /* + * Ensure the extension type is registered, as there might other tests trying to unregister the + * type. ex.: TestExtensionType#readUnderlyingType + */ + public static void ensureRegistered(ArrowType.ExtensionType type) { + if (ExtensionTypeRegistry.lookup(type.extensionName()) == null) { + ExtensionTypeRegistry.register(type); + } + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestUuidType.java b/vector/src/test/java/org/apache/arrow/vector/TestUuidType.java new file mode 100644 index 0000000000..9f7c65b82b --- /dev/null +++ b/vector/src/test/java/org/apache/arrow/vector/TestUuidType.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector; + +import static org.apache.arrow.vector.TestUtils.ensureRegistered; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.UUID; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.dictionary.DictionaryProvider; +import org.apache.arrow.vector.extension.UuidType; +import org.apache.arrow.vector.ipc.ArrowStreamReader; +import org.apache.arrow.vector.ipc.ArrowStreamWriter; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.arrow.vector.util.UuidUtility; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class TestUuidType { + BufferAllocator allocator; + + @BeforeEach + void beforeEach() { + allocator = new RootAllocator(); + } + + @AfterEach + void afterEach() { + allocator.close(); + } + + @Test + void testConstants() { + assertEquals("arrow.uuid", UuidType.EXTENSION_NAME); + assertNotNull(UuidType.INSTANCE); + assertNotNull(UuidType.STORAGE_TYPE); + assertInstanceOf(ArrowType.FixedSizeBinary.class, UuidType.STORAGE_TYPE); + assertEquals( + UuidType.UUID_BYTE_WIDTH, + ((ArrowType.FixedSizeBinary) UuidType.STORAGE_TYPE).getByteWidth()); + } + + @Test + void testStorageType() { + UuidType type = new UuidType(); + assertEquals(UuidType.STORAGE_TYPE, type.storageType()); + assertInstanceOf(ArrowType.FixedSizeBinary.class, type.storageType()); + } + + @Test + void testExtensionName() { + UuidType type = new UuidType(); + assertEquals("arrow.uuid", type.extensionName()); + } + + @Test + void testExtensionEquals() { + UuidType type1 = new UuidType(); + UuidType type2 = new UuidType(); + UuidType type3 = UuidType.INSTANCE; + + assertTrue(type1.extensionEquals(type2)); + assertTrue(type1.extensionEquals(type3)); + assertTrue(type2.extensionEquals(type3)); + } + + @Test + void testIsComplex() { + UuidType type = new UuidType(); + assertFalse(type.isComplex()); + } + + @Test + void testSerialize() { + UuidType type = new UuidType(); + String serialized = type.serialize(); + assertEquals("", serialized); + } + + @Test + void testDeserializeValid() { + UuidType type = new UuidType(); + ArrowType storageType = new ArrowType.FixedSizeBinary(UuidType.UUID_BYTE_WIDTH); + + ArrowType deserialized = assertDoesNotThrow(() -> type.deserialize(storageType, "")); + assertInstanceOf(UuidType.class, deserialized); + assertEquals(UuidType.INSTANCE, deserialized); + } + + @Test + void testDeserializeInvalidStorageType() { + UuidType type = new UuidType(); + ArrowType wrongStorageType = new ArrowType.FixedSizeBinary(32); + + assertThrows(UnsupportedOperationException.class, () -> type.deserialize(wrongStorageType, "")); + } + + @Test + void testGetNewVector() { + UuidType type = new UuidType(); + try (FieldVector vector = + type.getNewVector("uuid_field", FieldType.nullable(type), allocator)) { + assertInstanceOf(UuidVector.class, vector); + assertEquals("uuid_field", vector.getField().getName()); + assertEquals(type, vector.getField().getType()); + } + } + + @Test + void testVectorOperations() { + UuidType type = new UuidType(); + try (FieldVector vector = + type.getNewVector("uuid_field", FieldType.nullable(type), allocator)) { + UuidVector uuidVector = (UuidVector) vector; + + UUID uuid1 = UUID.randomUUID(); + UUID uuid2 = UUID.randomUUID(); + + uuidVector.setSafe(0, uuid1); + uuidVector.setSafe(1, uuid2); + uuidVector.setNull(2); + uuidVector.setValueCount(3); + + assertEquals(uuid1, uuidVector.getObject(0)); + assertEquals(uuid2, uuidVector.getObject(1)); + assertNull(uuidVector.getObject(2)); + assertFalse(uuidVector.isNull(0)); + assertFalse(uuidVector.isNull(1)); + assertTrue(uuidVector.isNull(2)); + } + } + + @Test + void testIpcRoundTrip() { + UuidType type = UuidType.INSTANCE; + ensureRegistered(type); + + Schema schema = new Schema(Collections.singletonList(Field.nullable("uuid", type))); + byte[] serialized = schema.serializeAsMessage(); + Schema deserialized = Schema.deserializeMessage(ByteBuffer.wrap(serialized)); + assertEquals(schema, deserialized); + } + + @Test + void testVectorIpcRoundTrip() throws IOException { + UuidType type = UuidType.INSTANCE; + ensureRegistered(type); + + UUID uuid1 = UUID.randomUUID(); + UUID uuid2 = UUID.randomUUID(); + + try (FieldVector vector = type.getNewVector("field", FieldType.nullable(type), allocator)) { + UuidVector uuidVector = (UuidVector) vector; + uuidVector.setSafe(0, uuid1); + uuidVector.setNull(1); + uuidVector.setSafe(2, uuid2); + uuidVector.setValueCount(3); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (VectorSchemaRoot root = new VectorSchemaRoot(Collections.singletonList(uuidVector)); + ArrowStreamWriter writer = + new ArrowStreamWriter(root, new DictionaryProvider.MapDictionaryProvider(), baos)) { + writer.start(); + writer.writeBatch(); + } + + try (ArrowStreamReader reader = + new ArrowStreamReader(new ByteArrayInputStream(baos.toByteArray()), allocator)) { + assertTrue(reader.loadNextBatch()); + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + assertEquals(3, root.getRowCount()); + assertEquals( + new Schema(Collections.singletonList(uuidVector.getField())), root.getSchema()); + + UuidVector actual = assertInstanceOf(UuidVector.class, root.getVector("field")); + assertFalse(actual.isNull(0)); + assertTrue(actual.isNull(1)); + assertFalse(actual.isNull(2)); + assertEquals(uuid1, actual.getObject(0)); + assertNull(actual.getObject(1)); + assertEquals(uuid2, actual.getObject(2)); + } + } + } + + @Test + void testVectorByteArrayOperations() { + UuidType type = new UuidType(); + try (FieldVector vector = + type.getNewVector("uuid_field", FieldType.nullable(type), allocator)) { + UuidVector uuidVector = (UuidVector) vector; + + UUID uuid = UUID.randomUUID(); + byte[] uuidBytes = UuidUtility.getBytesFromUUID(uuid); + + uuidVector.setSafe(0, uuidBytes); + uuidVector.setValueCount(1); + + assertEquals(uuid, uuidVector.getObject(0)); + + // Verify the bytes match + byte[] actualBytes = new byte[UuidType.UUID_BYTE_WIDTH]; + uuidVector.get(0).getBytes(0, actualBytes); + assertArrayEquals(uuidBytes, actualBytes); + } + } + + @Test + void testGetNewVectorWithCustomFieldType() { + UuidType type = new UuidType(); + FieldType fieldType = new FieldType(false, type, null); + + try (FieldVector vector = type.getNewVector("non_nullable_uuid", fieldType, allocator)) { + assertInstanceOf(UuidVector.class, vector); + assertEquals("non_nullable_uuid", vector.getField().getName()); + assertFalse(vector.getField().isNullable()); + } + } + + @Test + void testSingleton() { + UuidType type1 = UuidType.INSTANCE; + UuidType type2 = UuidType.INSTANCE; + + // Same instance + assertSame(type1, type2); + assertTrue(type1.extensionEquals(type2)); + } + + @Test + void testUnderlyingVector() { + UuidType type = new UuidType(); + try (FieldVector vector = + type.getNewVector("uuid_field", FieldType.nullable(type), allocator)) { + UuidVector uuidVector = (UuidVector) vector; + FixedSizeBinaryVector underlying = uuidVector.getUnderlyingVector(); + + assertInstanceOf(FixedSizeBinaryVector.class, underlying); + assertEquals(UuidType.UUID_BYTE_WIDTH, underlying.getByteWidth()); + } + } +} diff --git a/vector/src/test/java/org/apache/arrow/vector/TestUuidVector.java b/vector/src/test/java/org/apache/arrow/vector/TestUuidVector.java new file mode 100644 index 0000000000..3d70238ece --- /dev/null +++ b/vector/src/test/java/org/apache/arrow/vector/TestUuidVector.java @@ -0,0 +1,451 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.ByteBuffer; +import java.util.UUID; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.complex.impl.UuidReaderImpl; +import org.apache.arrow.vector.complex.impl.UuidWriterImpl; +import org.apache.arrow.vector.extension.UuidType; +import org.apache.arrow.vector.holders.ExtensionHolder; +import org.apache.arrow.vector.holders.NullableUuidHolder; +import org.apache.arrow.vector.holders.UuidHolder; +import org.apache.arrow.vector.util.UuidUtility; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** Tests for UuidVector, UuidWriterImpl, and UuidReaderImpl. */ +class TestUuidVector { + + private BufferAllocator allocator; + + @BeforeEach + void beforeEach() { + allocator = new RootAllocator(); + } + + @AfterEach + void afterEach() { + allocator.close(); + } + + // ========== Writer Tests ========== + + @Test + void testWriteToExtensionVector() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator); + UuidWriterImpl writer = new UuidWriterImpl(vector)) { + UUID uuid = UUID.randomUUID(); + ByteBuffer bb = ByteBuffer.allocate(UuidType.UUID_BYTE_WIDTH); + bb.putLong(uuid.getMostSignificantBits()); + bb.putLong(uuid.getLeastSignificantBits()); + + // Allocate ArrowBuf for the holder + try (ArrowBuf buf = allocator.buffer(UuidType.UUID_BYTE_WIDTH)) { + buf.setBytes(0, bb.array()); + + UuidHolder holder = new UuidHolder(); + holder.buffer = buf; + + writer.write(holder); + UUID result = vector.getObject(0); + assertEquals(uuid, result); + } + } + } + + @Test + void testWriteExtensionWithUUID() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator); + UuidWriterImpl writer = new UuidWriterImpl(vector)) { + UUID uuid = UUID.randomUUID(); + writer.setPosition(0); + writer.writeExtension(uuid); + + UUID result = vector.getObject(0); + assertEquals(uuid, result); + assertEquals(1, vector.getValueCount()); + } + } + + @Test + void testWriteExtensionWithByteArray() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator); + UuidWriterImpl writer = new UuidWriterImpl(vector)) { + UUID uuid = UUID.randomUUID(); + byte[] uuidBytes = UuidUtility.getBytesFromUUID(uuid); + + writer.setPosition(0); + writer.writeExtension(uuidBytes); + + UUID result = vector.getObject(0); + assertEquals(uuid, result); + assertEquals(1, vector.getValueCount()); + } + } + + @Test + void testWriteExtensionWithArrowBuf() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator); + UuidWriterImpl writer = new UuidWriterImpl(vector); + ArrowBuf buf = allocator.buffer(UuidType.UUID_BYTE_WIDTH)) { + UUID uuid = UUID.randomUUID(); + byte[] uuidBytes = UuidUtility.getBytesFromUUID(uuid); + buf.setBytes(0, uuidBytes); + + writer.setPosition(0); + writer.writeExtension(buf); + + UUID result = vector.getObject(0); + assertEquals(uuid, result); + assertEquals(1, vector.getValueCount()); + } + } + + @Test + void testWriteExtensionWithUnsupportedType() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator); + UuidWriterImpl writer = new UuidWriterImpl(vector)) { + writer.setPosition(0); + + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> writer.writeExtension("invalid-type")); + + assertEquals( + "Unsupported value type for UUID: class java.lang.String", exception.getMessage()); + } + } + + @Test + void testWriteExtensionMultipleValues() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator); + UuidWriterImpl writer = new UuidWriterImpl(vector)) { + UUID uuid1 = UUID.randomUUID(); + UUID uuid2 = UUID.randomUUID(); + UUID uuid3 = UUID.randomUUID(); + + writer.setPosition(0); + writer.writeExtension(uuid1); + writer.setPosition(1); + writer.writeExtension(uuid2); + writer.setPosition(2); + writer.writeExtension(uuid3); + + assertEquals(uuid1, vector.getObject(0)); + assertEquals(uuid2, vector.getObject(1)); + assertEquals(uuid3, vector.getObject(2)); + assertEquals(3, vector.getValueCount()); + } + } + + @Test + void testWriteWithUuidHolder() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator); + UuidWriterImpl writer = new UuidWriterImpl(vector); + ArrowBuf buf = allocator.buffer(UuidType.UUID_BYTE_WIDTH)) { + UUID uuid = UUID.randomUUID(); + byte[] uuidBytes = UuidUtility.getBytesFromUUID(uuid); + buf.setBytes(0, uuidBytes); + + UuidHolder holder = new UuidHolder(); + holder.buffer = buf; + holder.isSet = 1; + + writer.setPosition(0); + writer.write(holder); + + UUID result = vector.getObject(0); + assertEquals(uuid, result); + assertEquals(1, vector.getValueCount()); + } + } + + @Test + void testWriteWithNullableUuidHolder() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator); + UuidWriterImpl writer = new UuidWriterImpl(vector); + ArrowBuf buf = allocator.buffer(UuidType.UUID_BYTE_WIDTH)) { + UUID uuid = UUID.randomUUID(); + byte[] uuidBytes = UuidUtility.getBytesFromUUID(uuid); + buf.setBytes(0, uuidBytes); + + NullableUuidHolder holder = new NullableUuidHolder(); + holder.buffer = buf; + holder.isSet = 1; + + writer.setPosition(0); + writer.write(holder); + + UUID result = vector.getObject(0); + assertEquals(uuid, result); + assertEquals(1, vector.getValueCount()); + } + } + + @Test + void testWriteWithNullableUuidHolderNull() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator); + UuidWriterImpl writer = new UuidWriterImpl(vector)) { + NullableUuidHolder holder = new NullableUuidHolder(); + holder.isSet = 0; + + writer.setPosition(0); + writer.write(holder); + + assertTrue(vector.isNull(0)); + assertEquals(1, vector.getValueCount()); + } + } + + // ========== Reader Tests ========== + + @Test + void testReaderCopyAsValueExtensionVector() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator); + UuidVector vectorForRead = new UuidVector("test2", allocator); + UuidWriterImpl writer = new UuidWriterImpl(vector)) { + UUID uuid = UUID.randomUUID(); + vectorForRead.setValueCount(1); + vectorForRead.set(0, uuid); + UuidReaderImpl reader = (UuidReaderImpl) vectorForRead.getReader(); + reader.copyAsValue(writer); + UuidReaderImpl reader2 = (UuidReaderImpl) vector.getReader(); + UuidHolder holder = new UuidHolder(); + reader2.read(0, holder); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + assertEquals(uuid, actualUuid); + } + } + + @Test + void testReaderReadWithUuidHolder() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid = UUID.randomUUID(); + vector.setSafe(0, uuid); + vector.setValueCount(1); + + UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); + reader.setPosition(0); + + UuidHolder holder = new UuidHolder(); + reader.read(holder); + + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + assertEquals(uuid, actualUuid); + assertEquals(1, holder.isSet); + } + } + + @Test + void testReaderReadWithNullableUuidHolder() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid = UUID.randomUUID(); + vector.setSafe(0, uuid); + vector.setValueCount(1); + + UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); + reader.setPosition(0); + + NullableUuidHolder holder = new NullableUuidHolder(); + reader.read(holder); + + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + assertEquals(uuid, actualUuid); + assertEquals(1, holder.isSet); + } + } + + @Test + void testReaderReadWithNullableUuidHolderNull() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + vector.setNull(0); + vector.setValueCount(1); + + UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); + reader.setPosition(0); + + NullableUuidHolder holder = new NullableUuidHolder(); + reader.read(holder); + + assertEquals(0, holder.isSet); + } + } + + @Test + void testReaderReadWithArrayIndexUuidHolder() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid1 = UUID.randomUUID(); + UUID uuid2 = UUID.randomUUID(); + UUID uuid3 = UUID.randomUUID(); + + vector.setSafe(0, uuid1); + vector.setSafe(1, uuid2); + vector.setSafe(2, uuid3); + vector.setValueCount(3); + + UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); + + UuidHolder holder = new UuidHolder(); + reader.read(1, holder); + + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + assertEquals(uuid2, actualUuid); + assertEquals(1, holder.isSet); + } + } + + @Test + void testReaderReadWithArrayIndexNullableUuidHolder() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid1 = UUID.randomUUID(); + UUID uuid2 = UUID.randomUUID(); + + vector.setSafe(0, uuid1); + vector.setNull(1); + vector.setSafe(2, uuid2); + vector.setValueCount(3); + + UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); + + NullableUuidHolder holder1 = new NullableUuidHolder(); + reader.read(0, holder1); + assertEquals(uuid1, UuidUtility.uuidFromArrowBuf(holder1.buffer, 0)); + assertEquals(1, holder1.isSet); + + NullableUuidHolder holder2 = new NullableUuidHolder(); + reader.read(1, holder2); + assertEquals(0, holder2.isSet); + + NullableUuidHolder holder3 = new NullableUuidHolder(); + reader.read(2, holder3); + assertEquals(uuid2, UuidUtility.uuidFromArrowBuf(holder3.buffer, 0)); + assertEquals(1, holder3.isSet); + } + } + + @Test + void testReaderReadWithUnsupportedHolder() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid = UUID.randomUUID(); + vector.setSafe(0, uuid); + vector.setValueCount(1); + + UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); + reader.setPosition(0); + + // Create a mock unsupported holder + ExtensionHolder unsupportedHolder = new ExtensionHolder() {}; + + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> reader.read(unsupportedHolder)); + + assertTrue(exception.getMessage().contains("Unsupported holder type for UuidReader")); + } + } + + @Test + void testReaderReadWithArrayIndexUnsupportedHolder() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid = UUID.randomUUID(); + vector.setSafe(0, uuid); + vector.setValueCount(1); + + UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); + + // Create a mock unsupported holder + ExtensionHolder unsupportedHolder = new ExtensionHolder() {}; + + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> reader.read(0, unsupportedHolder)); + + assertTrue(exception.getMessage().contains("Unsupported holder type for UuidReader")); + } + } + + @Test + void testReaderIsSet() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid = UUID.randomUUID(); + vector.setSafe(0, uuid); + vector.setNull(1); + vector.setSafe(2, uuid); + vector.setValueCount(3); + + UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); + + reader.setPosition(0); + assertTrue(reader.isSet()); + + reader.setPosition(1); + assertFalse(reader.isSet()); + + reader.setPosition(2); + assertTrue(reader.isSet()); + } + } + + @Test + void testReaderReadObject() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid1 = UUID.randomUUID(); + UUID uuid2 = UUID.randomUUID(); + + vector.setSafe(0, uuid1); + vector.setNull(1); + vector.setSafe(2, uuid2); + vector.setValueCount(3); + + UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); + + reader.setPosition(0); + assertEquals(uuid1, reader.readObject()); + + reader.setPosition(1); + assertNull(reader.readObject()); + + reader.setPosition(2); + assertEquals(uuid2, reader.readObject()); + } + } + + @Test + void testReaderGetMinorType() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); + assertEquals(vector.getMinorType(), reader.getMinorType()); + } + } + + @Test + void testReaderGetField() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); + assertEquals(vector.getField(), reader.getField()); + assertEquals("test", reader.getField().getName()); + } + } +} diff --git a/vector/src/test/java/org/apache/arrow/vector/UuidVector.java b/vector/src/test/java/org/apache/arrow/vector/UuidVector.java deleted file mode 100644 index 72ba4aa555..0000000000 --- a/vector/src/test/java/org/apache/arrow/vector/UuidVector.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.arrow.vector; - -import java.nio.ByteBuffer; -import java.util.UUID; -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.memory.util.hash.ArrowBufHasher; -import org.apache.arrow.vector.complex.impl.UuidReaderImpl; -import org.apache.arrow.vector.complex.reader.FieldReader; -import org.apache.arrow.vector.holder.UuidHolder; -import org.apache.arrow.vector.types.pojo.Field; -import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.types.pojo.UuidType; -import org.apache.arrow.vector.util.TransferPair; - -public class UuidVector extends ExtensionTypeVector - implements ValueIterableVector { - private final Field field; - - public UuidVector( - String name, BufferAllocator allocator, FixedSizeBinaryVector underlyingVector) { - super(name, allocator, underlyingVector); - this.field = new Field(name, FieldType.nullable(new UuidType()), null); - } - - public UuidVector(String name, BufferAllocator allocator) { - super(name, allocator, new FixedSizeBinaryVector(name, allocator, 16)); - this.field = new Field(name, FieldType.nullable(new UuidType()), null); - } - - @Override - public UUID getObject(int index) { - final ByteBuffer bb = ByteBuffer.wrap(getUnderlyingVector().getObject(index)); - return new UUID(bb.getLong(), bb.getLong()); - } - - @Override - public int hashCode(int index) { - return hashCode(index, null); - } - - @Override - public int hashCode(int index, ArrowBufHasher hasher) { - return getUnderlyingVector().hashCode(index, hasher); - } - - public void set(int index, UUID uuid) { - ByteBuffer bb = ByteBuffer.allocate(16); - bb.putLong(uuid.getMostSignificantBits()); - bb.putLong(uuid.getLeastSignificantBits()); - getUnderlyingVector().set(index, bb.array()); - } - - @Override - public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { - getUnderlyingVector() - .copyFromSafe(fromIndex, thisIndex, ((UuidVector) from).getUnderlyingVector()); - } - - @Override - public Field getField() { - return field; - } - - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((UuidVector) to); - } - - @Override - protected FieldReader getReaderImpl() { - return new UuidReaderImpl(this); - } - - public void setSafe(int index, byte[] value) { - getUnderlyingVector().setIndexDefined(index); - getUnderlyingVector().setSafe(index, value); - } - - public void get(int index, UuidHolder holder) { - holder.value = getUnderlyingVector().get(index); - holder.isSet = 1; - } - - public class TransferImpl implements TransferPair { - UuidVector to; - ValueVector targetUnderlyingVector; - TransferPair tp; - - public TransferImpl(UuidVector to) { - this.to = to; - targetUnderlyingVector = this.to.getUnderlyingVector(); - tp = getUnderlyingVector().makeTransferPair(targetUnderlyingVector); - } - - public UuidVector getTo() { - return this.to; - } - - public void transfer() { - tp.transfer(); - } - - public void splitAndTransfer(int startIndex, int length) { - tp.splitAndTransfer(startIndex, length); - } - - public void copyValueSafe(int fromIndex, int toIndex) { - tp.copyValueSafe(fromIndex, toIndex); - } - } -} diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java index 738e8905e3..493a4b26ab 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java @@ -34,11 +34,11 @@ import org.apache.arrow.vector.complex.writer.BaseWriter.ExtensionWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.StructWriter; import org.apache.arrow.vector.complex.writer.FieldWriter; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.holders.DecimalHolder; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.types.pojo.UuidType; import org.apache.arrow.vector.util.DecimalUtility; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java index 7b8b1f9ef9..a4594024fa 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java @@ -41,6 +41,7 @@ import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.complex.UnionVector; import org.apache.arrow.vector.complex.writer.BaseWriter.StructWriter; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.holders.DurationHolder; import org.apache.arrow.vector.holders.FixedSizeBinaryHolder; import org.apache.arrow.vector.holders.NullableDecimalHolder; @@ -54,7 +55,6 @@ import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.types.pojo.UuidType; import org.apache.arrow.vector.util.DecimalUtility; import org.apache.arrow.vector.util.Text; import org.junit.jupiter.api.AfterEach; diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index f374eb41e4..46c259bda0 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -78,7 +78,7 @@ import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.StructWriter; -import org.apache.arrow.vector.holder.UuidHolder; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.holders.DecimalHolder; import org.apache.arrow.vector.holders.DurationHolder; import org.apache.arrow.vector.holders.FixedSizeBinaryHolder; @@ -88,6 +88,7 @@ import org.apache.arrow.vector.holders.NullableTimeStampMilliTZHolder; import org.apache.arrow.vector.holders.NullableTimeStampNanoTZHolder; import org.apache.arrow.vector.holders.TimeStampMilliTZHolder; +import org.apache.arrow.vector.holders.UuidHolder; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -99,13 +100,13 @@ import org.apache.arrow.vector.types.pojo.ArrowType.Utf8; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.types.pojo.UuidType; import org.apache.arrow.vector.util.CallBack; import org.apache.arrow.vector.util.DecimalUtility; import org.apache.arrow.vector.util.JsonStringArrayList; import org.apache.arrow.vector.util.JsonStringHashMap; import org.apache.arrow.vector.util.Text; import org.apache.arrow.vector.util.TransferPair; +import org.apache.arrow.vector.util.UuidUtility; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -2521,8 +2522,7 @@ public void extensionWriterReader() throws Exception { uuidReader.setPosition(0); UuidHolder uuidHolder = new UuidHolder(); uuidReader.read(uuidHolder); - final ByteBuffer bb = ByteBuffer.wrap(uuidHolder.value); - UUID actualUuid = new UUID(bb.getLong(), bb.getLong()); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(uuidHolder.buffer, 0); assertEquals(u1, actualUuid); assertTrue(uuidReader.isSet()); assertEquals(uuidReader.getMinorType(), MinorType.EXTENSIONTYPE); diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java index 269cff0670..5bb5962704 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java @@ -20,21 +20,16 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import java.nio.ByteBuffer; -import java.util.UUID; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.LargeVarBinaryVector; import org.apache.arrow.vector.LargeVarCharVector; -import org.apache.arrow.vector.UuidVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.complex.impl.LargeVarBinaryWriterImpl; import org.apache.arrow.vector.complex.impl.LargeVarCharWriterImpl; -import org.apache.arrow.vector.complex.impl.UuidReaderImpl; -import org.apache.arrow.vector.complex.impl.UuidWriterImpl; import org.apache.arrow.vector.complex.impl.VarBinaryWriterImpl; import org.apache.arrow.vector.complex.impl.VarCharWriterImpl; -import org.apache.arrow.vector.holder.UuidHolder; import org.apache.arrow.vector.util.Text; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -189,39 +184,4 @@ public void testWriteTextToLargeVarChar() throws Exception { assertEquals(input, result); } } - - @Test - public void testWriteToExtensionVector() throws Exception { - try (UuidVector vector = new UuidVector("test", allocator); - UuidWriterImpl writer = new UuidWriterImpl(vector)) { - UUID uuid = UUID.randomUUID(); - ByteBuffer bb = ByteBuffer.allocate(16); - bb.putLong(uuid.getMostSignificantBits()); - bb.putLong(uuid.getLeastSignificantBits()); - UuidHolder holder = new UuidHolder(); - holder.value = bb.array(); - writer.write(holder); - UUID result = vector.getObject(0); - assertEquals(uuid, result); - } - } - - @Test - public void testReaderCopyAsValueExtensionVector() throws Exception { - try (UuidVector vector = new UuidVector("test", allocator); - UuidVector vectorForRead = new UuidVector("test2", allocator); - UuidWriterImpl writer = new UuidWriterImpl(vector)) { - UUID uuid = UUID.randomUUID(); - vectorForRead.setValueCount(1); - vectorForRead.set(0, uuid); - UuidReaderImpl reader = (UuidReaderImpl) vectorForRead.getReader(); - reader.copyAsValue(writer); - UuidReaderImpl reader2 = (UuidReaderImpl) vector.getReader(); - UuidHolder holder = new UuidHolder(); - reader2.read(0, holder); - final ByteBuffer bb = ByteBuffer.wrap(holder.value); - UUID actualUuid = new UUID(bb.getLong(), bb.getLong()); - assertEquals(uuid, actualUuid); - } - } } diff --git a/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java b/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java index d24708d66c..2ac4045aa2 100644 --- a/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java +++ b/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.vector.types.pojo; +import static org.apache.arrow.vector.TestUtils.ensureRegistered; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -47,6 +48,7 @@ import org.apache.arrow.vector.compare.Range; import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.complex.StructVector; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.ipc.ArrowFileReader; import org.apache.arrow.vector.ipc.ArrowFileWriter; import org.apache.arrow.vector.types.FloatingPointPrecision; @@ -59,9 +61,9 @@ public class TestExtensionType { /** Test that a custom UUID type can be round-tripped through a temporary file. */ @Test public void roundtripUuid() throws IOException { - ExtensionTypeRegistry.register(new UuidType()); + ensureRegistered(UuidType.INSTANCE); final Schema schema = - new Schema(Collections.singletonList(Field.nullable("a", new UuidType()))); + new Schema(Collections.singletonList(Field.nullable("a", UuidType.INSTANCE))); try (final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { UUID u1 = UUID.randomUUID(); @@ -89,7 +91,7 @@ public void roundtripUuid() throws IOException { assertEquals(root.getSchema(), readerRoot.getSchema()); final Field field = readerRoot.getSchema().getFields().get(0); - final UuidType expectedType = new UuidType(); + final UuidType expectedType = UuidType.INSTANCE; assertEquals( field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_NAME), expectedType.extensionName()); @@ -113,9 +115,9 @@ public void roundtripUuid() throws IOException { /** Test that a custom UUID type can be read as its underlying type. */ @Test public void readUnderlyingType() throws IOException { - ExtensionTypeRegistry.register(new UuidType()); + ensureRegistered(UuidType.INSTANCE); final Schema schema = - new Schema(Collections.singletonList(Field.nullable("a", new UuidType()))); + new Schema(Collections.singletonList(Field.nullable("a", UuidType.INSTANCE))); try (final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { UUID u1 = UUID.randomUUID(); @@ -135,7 +137,7 @@ public void readUnderlyingType() throws IOException { writer.end(); } - ExtensionTypeRegistry.unregister(new UuidType()); + ExtensionTypeRegistry.unregister(UuidType.INSTANCE); try (final SeekableByteChannel channel = Files.newByteChannel(Paths.get(file.getAbsolutePath())); @@ -153,7 +155,7 @@ public void readUnderlyingType() throws IOException { .getByteWidth()); final Field field = readerRoot.getSchema().getFields().get(0); - final UuidType expectedType = new UuidType(); + final UuidType expectedType = UuidType.INSTANCE; assertEquals( field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_NAME), expectedType.extensionName()); @@ -254,7 +256,7 @@ public void roundtripLocation() throws IOException { @Test public void testVectorCompare() { - UuidType uuidType = new UuidType(); + UuidType uuidType = UuidType.INSTANCE; ExtensionTypeRegistry.register(uuidType); try (final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); UuidVector a1 = diff --git a/vector/src/test/java/org/apache/arrow/vector/types/pojo/UuidType.java b/vector/src/test/java/org/apache/arrow/vector/types/pojo/UuidType.java deleted file mode 100644 index 5e2bd8881b..0000000000 --- a/vector/src/test/java/org/apache/arrow/vector/types/pojo/UuidType.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.arrow.vector.types.pojo; - -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.FixedSizeBinaryVector; -import org.apache.arrow.vector.UuidVector; -import org.apache.arrow.vector.types.pojo.ArrowType.ExtensionType; - -public class UuidType extends ExtensionType { - - @Override - public ArrowType storageType() { - return new ArrowType.FixedSizeBinary(16); - } - - @Override - public String extensionName() { - return "uuid"; - } - - @Override - public boolean extensionEquals(ExtensionType other) { - return other instanceof UuidType; - } - - @Override - public ArrowType deserialize(ArrowType storageType, String serializedData) { - if (!storageType.equals(storageType())) { - throw new UnsupportedOperationException( - "Cannot construct UuidType from underlying type " + storageType); - } - return new UuidType(); - } - - @Override - public String serialize() { - return ""; - } - - @Override - public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator) { - return new UuidVector(name, allocator, new FixedSizeBinaryVector(name, allocator, 16)); - } -} From 9dc1410a3f4cc1fcb28a54bbe0e463585af16d08 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Dec 2025 17:41:10 +0100 Subject: [PATCH 095/271] MINOR: Bump dep.hadoop.version from 3.4.1 to 3.4.2 (#915) Bumps `dep.hadoop.version` from 3.4.1 to 3.4.2. Updates `org.apache.hadoop:hadoop-client-runtime` from 3.4.1 to 3.4.2 Updates `org.apache.hadoop:hadoop-client-api` from 3.4.1 to 3.4.2 Updates `org.apache.hadoop:hadoop-common` from 3.4.1 to 3.4.2 Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---

Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5d2567beef..777f7d5cef 100644 --- a/pom.xml +++ b/pom.xml @@ -101,7 +101,7 @@ under the License. 1.73.0 4.33.1 2.18.3 - 3.4.1 + 3.4.2 25.2.10 1.12.0 5.17.0 From 033ecc3f101de35063fb5db83ae5cbcccaee9409 Mon Sep 17 00:00:00 2001 From: ViggoC Date: Thu, 4 Dec 2025 17:30:46 +0800 Subject: [PATCH 096/271] GH-110: bind StringView/BinaryView for Flight SQL JDBC (#905) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What's Changed Closes #110. --------- Co-authored-by: 张林伟 --- .../ArrowFlightJdbcAccessorFactory.java | 8 +++++ .../ArrowFlightJdbcBinaryVectorAccessor.java | 8 +++++ .../ArrowFlightJdbcVarCharVectorAccessor.java | 8 +++++ .../BinaryViewAvaticaParameterConverter.java | 8 ++++- .../Utf8ViewAvaticaParameterConverter.java | 9 ++++- .../jdbc/utils/AvaticaParameterBinder.java | 6 ++-- .../arrow/driver/jdbc/utils/SqlTypes.java | 2 ++ .../ArrowFlightJdbcAccessorFactoryTest.java | 26 ++++++++++++++ ...owFlightJdbcVarCharVectorAccessorTest.java | 26 ++++++++++++++ .../driver/jdbc/utils/ConvertUtilsTest.java | 34 +++++++++++++++++++ .../arrow/driver/jdbc/utils/SqlTypesTest.java | 4 +++ 11 files changed, 135 insertions(+), 4 deletions(-) diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactory.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactory.java index dad1fa5f73..bbfe88a78a 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactory.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactory.java @@ -68,6 +68,8 @@ import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.ViewVarBinaryVector; +import org.apache.arrow.vector.ViewVarCharVector; import org.apache.arrow.vector.complex.DenseUnionVector; import org.apache.arrow.vector.complex.FixedSizeListVector; import org.apache.arrow.vector.complex.LargeListVector; @@ -130,6 +132,9 @@ public static ArrowFlightJdbcAccessor createAccessor( } else if (vector instanceof VarBinaryVector) { return new ArrowFlightJdbcBinaryVectorAccessor( (VarBinaryVector) vector, getCurrentRow, setCursorWasNull); + } else if (vector instanceof ViewVarBinaryVector) { + return new ArrowFlightJdbcBinaryVectorAccessor( + (ViewVarBinaryVector) vector, getCurrentRow, setCursorWasNull); } else if (vector instanceof LargeVarBinaryVector) { return new ArrowFlightJdbcBinaryVectorAccessor( (LargeVarBinaryVector) vector, getCurrentRow, setCursorWasNull); @@ -163,6 +168,9 @@ public static ArrowFlightJdbcAccessor createAccessor( } else if (vector instanceof LargeVarCharVector) { return new ArrowFlightJdbcVarCharVectorAccessor( (LargeVarCharVector) vector, getCurrentRow, setCursorWasNull); + } else if (vector instanceof ViewVarCharVector) { + return new ArrowFlightJdbcVarCharVectorAccessor( + (ViewVarCharVector) vector, getCurrentRow, setCursorWasNull); } else if (vector instanceof DurationVector) { return new ArrowFlightJdbcDurationVectorAccessor( (DurationVector) vector, getCurrentRow, setCursorWasNull); diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/binary/ArrowFlightJdbcBinaryVectorAccessor.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/binary/ArrowFlightJdbcBinaryVectorAccessor.java index 30dfffce64..e71b6380a9 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/binary/ArrowFlightJdbcBinaryVectorAccessor.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/binary/ArrowFlightJdbcBinaryVectorAccessor.java @@ -27,6 +27,7 @@ import org.apache.arrow.vector.FixedSizeBinaryVector; import org.apache.arrow.vector.LargeVarBinaryVector; import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.ViewVarBinaryVector; /** * Accessor for the Arrow types: {@link FixedSizeBinaryVector}, {@link VarBinaryVector} and {@link @@ -61,6 +62,13 @@ public ArrowFlightJdbcBinaryVectorAccessor( this(vector::get, currentRowSupplier, setCursorWasNull); } + public ArrowFlightJdbcBinaryVectorAccessor( + ViewVarBinaryVector vector, + IntSupplier currentRowSupplier, + ArrowFlightJdbcAccessorFactory.WasNullConsumer setCursorWasNull) { + this(vector::get, currentRowSupplier, setCursorWasNull); + } + private ArrowFlightJdbcBinaryVectorAccessor( ByteArrayGetter getter, IntSupplier currentRowSupplier, diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/text/ArrowFlightJdbcVarCharVectorAccessor.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/text/ArrowFlightJdbcVarCharVectorAccessor.java index ebebf6ca74..7b04e89346 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/text/ArrowFlightJdbcVarCharVectorAccessor.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/text/ArrowFlightJdbcVarCharVectorAccessor.java @@ -35,6 +35,7 @@ import org.apache.arrow.driver.jdbc.utils.DateTimeUtils; import org.apache.arrow.vector.LargeVarCharVector; import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.ViewVarCharVector; import org.apache.arrow.vector.util.Text; /** Accessor for the Arrow types: {@link VarCharVector} and {@link LargeVarCharVector}. */ @@ -62,6 +63,13 @@ public ArrowFlightJdbcVarCharVectorAccessor( this(vector::get, currentRowSupplier, setCursorWasNull); } + public ArrowFlightJdbcVarCharVectorAccessor( + ViewVarCharVector vector, + IntSupplier currentRowSupplier, + ArrowFlightJdbcAccessorFactory.WasNullConsumer setCursorWasNull) { + this(vector::get, currentRowSupplier, setCursorWasNull); + } + ArrowFlightJdbcVarCharVectorAccessor( Getter getter, IntSupplier currentRowSupplier, diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/BinaryViewAvaticaParameterConverter.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/BinaryViewAvaticaParameterConverter.java index a035bbba49..d692f39372 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/BinaryViewAvaticaParameterConverter.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/BinaryViewAvaticaParameterConverter.java @@ -17,6 +17,7 @@ package org.apache.arrow.driver.jdbc.converter.impl; import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.ViewVarBinaryVector; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.calcite.avatica.AvaticaParameter; @@ -29,7 +30,12 @@ public BinaryViewAvaticaParameterConverter(ArrowType.BinaryView type) {} @Override public boolean bindParameter(FieldVector vector, TypedValue typedValue, int index) { - throw new UnsupportedOperationException("Not implemented"); + byte[] value = (byte[]) typedValue.toJdbc(null); + if (vector instanceof ViewVarBinaryVector) { + ((ViewVarBinaryVector) vector).setSafe(index, value); + return true; + } + return false; } @Override diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/Utf8ViewAvaticaParameterConverter.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/Utf8ViewAvaticaParameterConverter.java index 076fefc42a..c9d9f2926b 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/Utf8ViewAvaticaParameterConverter.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/Utf8ViewAvaticaParameterConverter.java @@ -17,8 +17,10 @@ package org.apache.arrow.driver.jdbc.converter.impl; import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.ViewVarCharVector; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.util.Text; import org.apache.calcite.avatica.AvaticaParameter; import org.apache.calcite.avatica.remote.TypedValue; @@ -29,7 +31,12 @@ public Utf8ViewAvaticaParameterConverter(ArrowType.Utf8View type) {} @Override public boolean bindParameter(FieldVector vector, TypedValue typedValue, int index) { - throw new UnsupportedOperationException("Utf8View not supported"); + String value = (String) typedValue.toLocal(); + if (vector instanceof ViewVarCharVector) { + ((ViewVarCharVector) vector).setSafe(index, new Text(value)); + return true; + } + return false; } @Override diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java index 0fd99de539..8c98ee4077 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java @@ -19,6 +19,7 @@ import java.util.List; import org.apache.arrow.driver.jdbc.client.ArrowFlightSqlClientHandler.PreparedStatement; import org.apache.arrow.driver.jdbc.converter.impl.BinaryAvaticaParameterConverter; +import org.apache.arrow.driver.jdbc.converter.impl.BinaryViewAvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.BoolAvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.DateAvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.DecimalAvaticaParameterConverter; @@ -39,6 +40,7 @@ import org.apache.arrow.driver.jdbc.converter.impl.TimestampAvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.UnionAvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.Utf8AvaticaParameterConverter; +import org.apache.arrow.driver.jdbc.converter.impl.Utf8ViewAvaticaParameterConverter; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.VectorSchemaRoot; @@ -208,7 +210,7 @@ public Boolean visit(ArrowType.Utf8 type) { @Override public Boolean visit(ArrowType.Utf8View type) { - throw new UnsupportedOperationException("Utf8View is unsupported"); + return new Utf8ViewAvaticaParameterConverter(type).bindParameter(vector, typedValue, index); } @Override @@ -223,7 +225,7 @@ public Boolean visit(ArrowType.Binary type) { @Override public Boolean visit(ArrowType.BinaryView type) { - throw new UnsupportedOperationException("BinaryView is unsupported"); + return new BinaryViewAvaticaParameterConverter(type).bindParameter(vector, typedValue, index); } @Override diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/SqlTypes.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/SqlTypes.java index 1b76ca0c95..5ba3957f8b 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/SqlTypes.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/SqlTypes.java @@ -107,12 +107,14 @@ public static int getSqlTypeIdFromArrowType(ArrowType arrowType) { } break; case Binary: + case BinaryView: return Types.VARBINARY; case FixedSizeBinary: return Types.BINARY; case LargeBinary: return Types.LONGVARBINARY; case Utf8: + case Utf8View: return Types.VARCHAR; case LargeUtf8: return Types.LONGVARCHAR; diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactoryTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactoryTest.java index b56bf3c63d..8b39041f0c 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactoryTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactoryTest.java @@ -46,6 +46,8 @@ import org.apache.arrow.vector.LargeVarCharVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.ViewVarBinaryVector; +import org.apache.arrow.vector.ViewVarCharVector; import org.apache.arrow.vector.complex.DenseUnionVector; import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.StructVector; @@ -239,6 +241,18 @@ public void createAccessorForFixedSizeBinaryVector() { } } + @Test + public void createAccessorForViewVarBinaryVector() { + try (ValueVector valueVector = + new ViewVarBinaryVector("", rootAllocatorTestExtension.getRootAllocator())) { + ArrowFlightJdbcAccessor accessor = + ArrowFlightJdbcAccessorFactory.createAccessor( + valueVector, GET_CURRENT_ROW, (boolean wasNull) -> {}); + + assertTrue(accessor instanceof ArrowFlightJdbcBinaryVectorAccessor); + } + } + @Test public void createAccessorForTimeStampVector() { try (ValueVector valueVector = rootAllocatorTestExtension.createTimeStampMilliVector()) { @@ -340,6 +354,18 @@ public void createAccessorForLargeVarCharVector() { } } + @Test + public void createAccessorForViewVarCharVector() { + try (ValueVector valueVector = + new ViewVarCharVector("", rootAllocatorTestExtension.getRootAllocator())) { + ArrowFlightJdbcAccessor accessor = + ArrowFlightJdbcAccessorFactory.createAccessor( + valueVector, GET_CURRENT_ROW, (boolean wasNull) -> {}); + + assertTrue(accessor instanceof ArrowFlightJdbcVarCharVectorAccessor); + } + } + @Test public void createAccessorForDurationVector() { try (ValueVector valueVector = diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/text/ArrowFlightJdbcVarCharVectorAccessorTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/text/ArrowFlightJdbcVarCharVectorAccessorTest.java index a2f6fd586f..82876f4aa1 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/text/ArrowFlightJdbcVarCharVectorAccessorTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/text/ArrowFlightJdbcVarCharVectorAccessorTest.java @@ -24,6 +24,8 @@ import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.when; @@ -46,6 +48,8 @@ import org.apache.arrow.vector.DateMilliVector; import org.apache.arrow.vector.TimeMilliVector; import org.apache.arrow.vector.TimeStampVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.ViewVarCharVector; import org.apache.arrow.vector.util.Text; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -695,4 +699,26 @@ public void testShouldGetObjectClassReturnString() { final Class clazz = accessor.getObjectClass(); assertThat(clazz, equalTo(String.class)); } + + @Test + public void testViewVarcharVector() throws Exception { + try (VarCharVector varCharVector = + new VarCharVector("", rootAllocatorTestExtension.getRootAllocator()); + ViewVarCharVector viewVarCharVector = + new ViewVarCharVector("", rootAllocatorTestExtension.getRootAllocator())) { + varCharVector.allocateNew(1); + viewVarCharVector.allocateNew(1); + + ArrowFlightJdbcVarCharVectorAccessor varCharVectorAccessor = + new ArrowFlightJdbcVarCharVectorAccessor(varCharVector, () -> 0, (boolean wasNull) -> {}); + ArrowFlightJdbcVarCharVectorAccessor viewVarcharVectorAccessor = + new ArrowFlightJdbcVarCharVectorAccessor( + viewVarCharVector, () -> 0, (boolean wasNull) -> {}); + assertNull(viewVarcharVectorAccessor.getString()); + + varCharVector.set(0, new Text("looooong_string")); + viewVarCharVector.set(0, new Text("looooong_string")); + assertEquals(varCharVectorAccessor.getString(), viewVarcharVectorAccessor.getString()); + } + } } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ConvertUtilsTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ConvertUtilsTest.java index b6fdc99694..f128ca7c73 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ConvertUtilsTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ConvertUtilsTest.java @@ -69,6 +69,19 @@ public void testShouldConvertArrowFieldsToColumnMetaDataList() { .tableName("table1") .build() .getMetadataMap()), + null), + new Field( + "col2", + new FieldType( + true, + ArrowType.Utf8View.INSTANCE, + null, + new FlightSqlColumnMetadata.Builder() + .catalogName("catalog1") + .schemaName("schema1") + .tableName("table1") + .build() + .getMetadataMap()), null)); final List expectedColumnMetaData = @@ -78,6 +91,25 @@ public void testShouldConvertArrowFieldsToColumnMetaDataList() { .setCatalogName("catalog1") .setSchemaName("schema1") .setTableName("table1") + .setColumnName("col1") + .setType( + Common.AvaticaType.newBuilder() + .setId(SqlTypes.getSqlTypeIdFromArrowType(ArrowType.Utf8.INSTANCE)) + .setName(SqlTypes.getSqlTypeNameFromArrowType(ArrowType.Utf8.INSTANCE)) + .build()) + .build()), + ColumnMetaData.fromProto( + Common.ColumnMetaData.newBuilder() + .setCatalogName("catalog1") + .setSchemaName("schema1") + .setTableName("table1") + .setColumnName("col2") + .setType( + Common.AvaticaType.newBuilder() + .setId(SqlTypes.getSqlTypeIdFromArrowType(ArrowType.Utf8View.INSTANCE)) + .setName( + SqlTypes.getSqlTypeNameFromArrowType(ArrowType.Utf8View.INSTANCE)) + .build()) .build())); final List actualColumnMetaData = @@ -95,6 +127,8 @@ private void assertColumnMetaData( assertThat(expectedColumnMetaData.catalogName, equalTo(actualColumnMetaData.catalogName)); assertThat(expectedColumnMetaData.schemaName, equalTo(actualColumnMetaData.schemaName)); assertThat(expectedColumnMetaData.tableName, equalTo(actualColumnMetaData.tableName)); + assertThat(expectedColumnMetaData.columnName, equalTo(actualColumnMetaData.columnName)); + assertThat(expectedColumnMetaData.type, equalTo(actualColumnMetaData.type)); assertThat(expectedColumnMetaData.readOnly, equalTo(actualColumnMetaData.readOnly)); assertThat(expectedColumnMetaData.autoIncrement, equalTo(actualColumnMetaData.autoIncrement)); assertThat(expectedColumnMetaData.precision, equalTo(actualColumnMetaData.precision)); diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/SqlTypesTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/SqlTypesTest.java index a6dd6b3275..d69c549296 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/SqlTypesTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/SqlTypesTest.java @@ -40,9 +40,11 @@ public void testGetSqlTypeIdFromArrowType() { assertEquals(Types.BINARY, getSqlTypeIdFromArrowType(new ArrowType.FixedSizeBinary(1024))); assertEquals(Types.VARBINARY, getSqlTypeIdFromArrowType(new ArrowType.Binary())); + assertEquals(Types.VARBINARY, getSqlTypeIdFromArrowType(new ArrowType.BinaryView())); assertEquals(Types.LONGVARBINARY, getSqlTypeIdFromArrowType(new ArrowType.LargeBinary())); assertEquals(Types.VARCHAR, getSqlTypeIdFromArrowType(new ArrowType.Utf8())); + assertEquals(Types.VARCHAR, getSqlTypeIdFromArrowType(new ArrowType.Utf8View())); assertEquals(Types.LONGVARCHAR, getSqlTypeIdFromArrowType(new ArrowType.LargeUtf8())); assertEquals(Types.DATE, getSqlTypeIdFromArrowType(new ArrowType.Date(DateUnit.MILLISECOND))); @@ -94,9 +96,11 @@ public void testGetSqlTypeNameFromArrowType() { assertEquals("BINARY", getSqlTypeNameFromArrowType(new ArrowType.FixedSizeBinary(1024))); assertEquals("VARBINARY", getSqlTypeNameFromArrowType(new ArrowType.Binary())); + assertEquals("VARBINARY", getSqlTypeNameFromArrowType(new ArrowType.BinaryView())); assertEquals("LONGVARBINARY", getSqlTypeNameFromArrowType(new ArrowType.LargeBinary())); assertEquals("VARCHAR", getSqlTypeNameFromArrowType(new ArrowType.Utf8())); + assertEquals("VARCHAR", getSqlTypeNameFromArrowType(new ArrowType.Utf8View())); assertEquals("LONGVARCHAR", getSqlTypeNameFromArrowType(new ArrowType.LargeUtf8())); assertEquals("DATE", getSqlTypeNameFromArrowType(new ArrowType.Date(DateUnit.MILLISECOND))); From 2329174e0bf35733b5ac99b7e36465a601ae20e8 Mon Sep 17 00:00:00 2001 From: Pedro Matias Date: Thu, 4 Dec 2025 09:33:49 +0000 Subject: [PATCH 097/271] GH-863: [JDBC] Suppress benign exceptions from gRPC layer on ArrowFlightSqlClientHandler#close (#910) ## What's Changed When using the Flight SQL JDBC driver with connection pooling and a catalog parameter, ArrowFlightSqlClientHandler.close() performs a CloseSession RPC that can fail during gRPC channel shutdown. These transient failures (UNAVAILABLE or INTERNAL with "Connection closed after GOAWAY") cause noisy errors in pooling frameworks like Apache Commons DBCP. With this PR these exceptions will instead be suppressed and logged, following the procedure that was used for [ARROW-17785](https://issues.apache.org/jira/browse/ARROW-17785) ### Are these changes tested? Yes Closes #863 --------- Co-authored-by: Vando Pereira --- .../client/ArrowFlightSqlClientHandler.java | 81 +++++++++++++++-- .../ArrowFlightSqlClientHandlerTest.java | 88 +++++++++++++++++++ 2 files changed, 160 insertions(+), 9 deletions(-) create mode 100644 flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerTest.java diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java index a3f6900373..5dc7e0e2e9 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java @@ -262,15 +262,85 @@ public FlightInfo getInfo(final String query) { @Override public void close() throws SQLException { if (catalog.isPresent()) { - sqlClient.closeSession(new CloseSessionRequest(), getOptions()); + try { + sqlClient.closeSession(new CloseSessionRequest(), getOptions()); + } catch (FlightRuntimeException fre) { + handleBenignCloseException( + fre, "Failed to close Flight SQL session.", "closing Flight SQL session"); + } } try { AutoCloseables.close(sqlClient); + } catch (FlightRuntimeException fre) { + handleBenignCloseException( + fre, "Failed to clean up client resources.", "closing Flight SQL client"); } catch (final Exception e) { throw new SQLException("Failed to clean up client resources.", e); } } + /** + * Handles FlightRuntimeException during close operations, suppressing benign gRPC shutdown errors + * while re-throwing genuine failures. + * + * @param fre the FlightRuntimeException to handle + * @param sqlErrorMessage the SQLException message to use for genuine failures + * @param operationDescription description of the operation for logging + * @throws SQLException if the exception represents a genuine failure + */ + private void handleBenignCloseException( + FlightRuntimeException fre, String sqlErrorMessage, String operationDescription) + throws SQLException { + if (isBenignCloseException(fre)) { + logSuppressedCloseException(fre, operationDescription); + } else { + throw new SQLException(sqlErrorMessage, fre); + } + } + + /** + * Handles FlightRuntimeException during close operations, suppressing benign gRPC shutdown errors + * while re-throwing genuine failures as FlightRuntimeException. + * + * @param fre the FlightRuntimeException to handle + * @param operationDescription description of the operation for logging + * @throws FlightRuntimeException if the exception represents a genuine failure + */ + private void handleBenignCloseException(FlightRuntimeException fre, String operationDescription) + throws FlightRuntimeException { + if (isBenignCloseException(fre)) { + logSuppressedCloseException(fre, operationDescription); + } else { + throw fre; + } + } + + /** + * Determines if a FlightRuntimeException represents a benign close operation error that should be + * suppressed. + * + * @param fre the FlightRuntimeException to check + * @return true if the exception should be suppressed, false otherwise + */ + private boolean isBenignCloseException(FlightRuntimeException fre) { + return fre.status().code().equals(FlightStatusCode.UNAVAILABLE) + || (fre.status().code().equals(FlightStatusCode.INTERNAL) + && fre.getMessage() != null + && fre.getMessage().contains("Connection closed after GOAWAY")); + } + + /** + * Logs a suppressed close exception with appropriate level based on debug settings. + * + * @param fre the FlightRuntimeException being suppressed + * @param operationDescription description of the operation for logging + */ + private void logSuppressedCloseException( + FlightRuntimeException fre, String operationDescription) { + // ARROW-17785 and GH-863: suppress exceptions caused by flaky gRPC layer during shutdown + LOGGER.debug("Suppressed error {}", operationDescription, fre); + } + /** A prepared statement handler. */ public interface PreparedStatement extends AutoCloseable { /** @@ -386,14 +456,7 @@ public void close() { try { preparedStatement.close(getOptions()); } catch (FlightRuntimeException fre) { - // ARROW-17785: suppress exceptions caused by flaky gRPC layer - if (fre.status().code().equals(FlightStatusCode.UNAVAILABLE) - || (fre.status().code().equals(FlightStatusCode.INTERNAL) - && fre.getMessage().contains("Connection closed after GOAWAY"))) { - LOGGER.warn("Supressed error closing PreparedStatement", fre); - return; - } - throw fre; + handleBenignCloseException(fre, "closing PreparedStatement"); } } }; diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerTest.java new file mode 100644 index 0000000000..d5973ab5d8 --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandlerTest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.client; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Optional; +import org.apache.arrow.flight.CallOption; +import org.apache.arrow.flight.CallStatus; +import org.apache.arrow.flight.CloseSessionRequest; +import org.apache.arrow.flight.FlightStatusCode; +import org.apache.arrow.flight.sql.FlightSqlClient; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +public class ArrowFlightSqlClientHandlerTest { + + @ParameterizedTest + @MethodSource + public void testCloseHandlesFlightRuntimeException( + boolean throwFromCloseSession, CallStatus callStatus, boolean shouldSuppress) + throws Exception { + FlightSqlClient sqlClient = mock(FlightSqlClient.class); + String cacheKey = "cacheKey"; + Optional catalog = + throwFromCloseSession ? Optional.of("test_catalog") : Optional.empty(); + final Collection credentialOptions = new ArrayList<>(); + ArrowFlightSqlClientHandler.Builder builder = new ArrowFlightSqlClientHandler.Builder(); + + if (throwFromCloseSession) { + doThrow(callStatus.toRuntimeException()) + .when(sqlClient) + .closeSession(any(CloseSessionRequest.class), any(CallOption[].class)); + } else { + doThrow(callStatus.toRuntimeException()).when(sqlClient).close(); + } + + ArrowFlightSqlClientHandler sqlClientHandler = + new ArrowFlightSqlClientHandler( + cacheKey, sqlClient, builder, credentialOptions, catalog, null); + + if (shouldSuppress) { + assertDoesNotThrow(sqlClientHandler::close); + } else { + assertThrows(SQLException.class, sqlClientHandler::close); + } + } + + private static Object[] testCloseHandlesFlightRuntimeException() { + CallStatus benignInternalError = + new CallStatus(FlightStatusCode.INTERNAL, null, "Connection closed after GOAWAY", null); + CallStatus notBenignInternalError = + new CallStatus(FlightStatusCode.INTERNAL, null, "Not a benign internal error", null); + CallStatus unavailableError = new CallStatus(FlightStatusCode.UNAVAILABLE, null, null, null); + CallStatus unknownError = new CallStatus(FlightStatusCode.UNKNOWN, null, null, null); + return new Object[] { + new Object[] {true, benignInternalError, true}, + new Object[] {false, benignInternalError, true}, + new Object[] {true, notBenignInternalError, false}, + new Object[] {false, notBenignInternalError, false}, + new Object[] {true, unavailableError, true}, + new Object[] {false, unavailableError, true}, + new Object[] {true, unknownError, false}, + new Object[] {false, unknownError, false}, + }; + } +} From 92816711d9a7da18ec2037395c71f9293a0f0301 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 08:48:02 +0100 Subject: [PATCH 098/271] MINOR: [CI] Bump actions/upload-artifact from 5.0.0 to 6.0.0 (#934) --- .github/workflows/rc.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 0ae4399ef1..b71d49b314 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -71,7 +71,7 @@ jobs: run: | dev/release/run_rat.sh "${TAR_GZ}" - name: Upload source archive - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: release-source path: | @@ -154,7 +154,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf jni-linux-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: jni-linux-${{ matrix.platform.arch }} path: jni-linux-${{ matrix.platform.arch }}.tar.gz @@ -284,7 +284,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf jni-macos-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: jni-macos-${{ matrix.platform.arch }} path: jni-macos-${{ matrix.platform.arch }}.tar.gz @@ -368,7 +368,7 @@ jobs: shell: bash run: tar -cvzf jni-windows-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: jni-windows-${{ matrix.platform.arch }} path: jni-windows-${{ matrix.platform.arch }}.tar.gz @@ -440,12 +440,12 @@ jobs: cp -a target/site/apidocs reference tar -cvzf reference.tar.gz reference - name: Upload binaries - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: release-binaries path: binaries/* - name: Upload docs - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: reference path: reference.tar.gz @@ -483,7 +483,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf html.tar.gz -C docs/build html - name: Upload artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: release-html path: html.tar.gz From 8f616f47483cd2d4a0ca169933da4d7565756928 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 08:48:26 +0100 Subject: [PATCH 099/271] MINOR: [CI] Bump actions/download-artifact from 6.0.0 to 7.0.0 (#935) --- .github/workflows/rc.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index b71d49b314..5dcbc664a7 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -101,7 +101,7 @@ jobs: packages: write steps: - name: Download source archive - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: release-source - name: Extract source archive @@ -174,7 +174,7 @@ jobs: MACOSX_DEPLOYMENT_TARGET: "14.0" steps: - name: Download source archive - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: release-source - name: Extract source archive @@ -302,7 +302,7 @@ jobs: arch: "x86_64" steps: - name: Download source archive - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: release-source - name: Extract source archive @@ -381,7 +381,7 @@ jobs: - jni-windows steps: - name: Download artifacts - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: path: artifacts - name: Decompress artifacts @@ -462,11 +462,11 @@ jobs: with: cache: 'pip' - name: Download source archive - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: release-source - name: Download Javadocs - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: reference - name: Extract source archive @@ -531,7 +531,7 @@ jobs: cp ../.asf.yaml ./ git add .nojekyll .asf.yaml - name: Download - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: release-html - name: Extract @@ -567,7 +567,7 @@ jobs: - ubuntu-latest steps: - name: Download release artifacts - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: pattern: release-* - name: Verify @@ -601,7 +601,7 @@ jobs: contents: write steps: - name: Download release artifacts - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: pattern: release-* path: artifacts From 96156ccc2bf933c75c852ca7c04418a61f87defd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 08:48:46 +0100 Subject: [PATCH 100/271] MINOR: [CI] Bump actions/cache from 4 to 5 (#936) --- .github/workflows/dev.yml | 2 +- .github/workflows/rc.yml | 8 ++++---- .github/workflows/test.yml | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 57432d75f1..2111f47254 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -42,7 +42,7 @@ jobs: with: python-version: '3.x' - name: pre-commit (cache) - uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 with: path: ~/.cache/pre-commit key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 5dcbc664a7..efa69533d3 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -139,7 +139,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Cache - uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 with: path: .docker key: jni-linux-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} @@ -270,7 +270,7 @@ jobs: run: | echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} - name: Cache ccache - uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 with: path: ccache key: jni-macos-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} @@ -352,7 +352,7 @@ jobs: run: | echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} - name: Cache ccache - uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 with: path: ccache key: jni-windows-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} @@ -426,7 +426,7 @@ jobs: repository: apache/arrow-testing path: testing - name: Cache ~/.m2 - uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 with: path: ~/.m2 key: binaries-build-${{ hashFiles('**/*.java', '**/pom.xml') }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8d8fcb5052..e1beea793e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,7 +63,7 @@ jobs: fetch-depth: 0 submodules: recursive - name: Cache Docker Volumes - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: .docker key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('compose.yaml', '**/pom.xml', '**/*.java') }} @@ -190,7 +190,7 @@ jobs: run: | ci/scripts/util_free_space.sh - name: Cache Docker Volumes - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: .docker key: integration-conda-${{ hashFiles('cpp/**') }} From 8d1802cef6bacd27d1cc8a0c2cf3efcddb6bf255 Mon Sep 17 00:00:00 2001 From: Kaustav Sarkar <70840177+Kaustav-Sarkar@users.noreply.github.com> Date: Mon, 29 Dec 2025 14:01:10 +0530 Subject: [PATCH 101/271] GH-399: Check for null writers in DenseUnionWriter#setPosition (#938) ## GH-399 Fix setPosition fails with NullPointerException Fixed a `NullPointerException` in `DenseUnionWriter#setPosition`. The issue was that `setPosition` tried to update all writers in its internal array, even if they hadn't been initialized yet. I added a null check so it only updates writers that actually exist. Also added a regression test (`TestDenseUnionWriterNPE`) to verify the fix and updated `.gitignore`. Closes #399. --- .gitignore | 2 + .../codegen/templates/DenseUnionWriter.java | 4 +- .../complex/writer/TestComplexWriter.java | 43 +++++++++++++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b57597af47..17d1d43ae1 100644 --- a/.gitignore +++ b/.gitignore @@ -7,10 +7,12 @@ .buildpath .checkstyle .classpath +.cursor/ .factorypath .idea/ .project .settings/ +.vscode/ /*-build/ /.mvn/.develocity/ /apache-arrow-java-* diff --git a/vector/src/main/codegen/templates/DenseUnionWriter.java b/vector/src/main/codegen/templates/DenseUnionWriter.java index 8515b759e6..9aeea5b054 100644 --- a/vector/src/main/codegen/templates/DenseUnionWriter.java +++ b/vector/src/main/codegen/templates/DenseUnionWriter.java @@ -55,7 +55,9 @@ public DenseUnionWriter(DenseUnionVector vector, NullableStructWriterFactory nul public void setPosition(int index) { super.setPosition(index); for (BaseWriter writer : writers) { - writer.setPosition(index); + if (writer != null) { + writer.setPosition(index); + } } } diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index 46c259bda0..871a3cc461 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -2530,4 +2530,47 @@ public void extensionWriterReader() throws Exception { } } } + + @Test + void testListOfDenseUnionWriterNPE() { + // Regression test for https://github.com/apache/arrow-java/issues/399 + try (ListVector listVector = ListVector.empty("list", allocator)) { + listVector.addOrGetVector(FieldType.nullable(MinorType.DENSEUNION.getType())); + UnionListWriter listWriter = listVector.getWriter(); + + listWriter.startList(); + listWriter.endList(); + } + } + + @Test + void testListOfDenseUnionWriterWithData() { + try (ListVector listVector = ListVector.empty("list", allocator)) { + listVector.addOrGetVector(FieldType.nullable(MinorType.DENSEUNION.getType())); + + UnionListWriter listWriter = listVector.getWriter(); + listWriter.startList(); + listWriter.writeInt(100); + listWriter.writeBigInt(200L); + listWriter.endList(); + + listWriter.startList(); + listWriter.writeFloat4(3.14f); + listWriter.endList(); + + listVector.setValueCount(2); + + assertEquals(2, listVector.getValueCount()); + + List value0 = (List) listVector.getObject(0); + List value1 = (List) listVector.getObject(1); + + assertEquals(2, value0.size()); + assertEquals(100, value0.get(0)); + assertEquals(200L, value0.get(1)); + + assertEquals(1, value1.size()); + assertEquals(3.14f, value1.get(0)); + } + } } From 385b51eb5bf001e34d71d63b5980a5eabaa38294 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 29 Dec 2025 19:44:18 +0900 Subject: [PATCH 102/271] MINOR: Update macos amd64 runner (#940) ## What's Changed Replace the macOS 13 runner. --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e1beea793e..2602592799 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -89,7 +89,7 @@ jobs: include: - arch: AMD64 jdk: 11 - macos: 13 + macos: 15-intel - arch: AArch64 jdk: 11 macos: latest From b9e40fa0a90143eef36ca53d2f937f2fef494402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Eickler?= <797483+eickler@users.noreply.github.com> Date: Mon, 5 Jan 2026 07:39:15 +0100 Subject: [PATCH 103/271] GH-942: Fix JDBC Connection.setCatalog() (#943) ## What's Changed Connection.setCatalog() is not silently ignored anymore (through the default implementation in Calcite) but instead it updates the catalog session option in the same way as during the initial connection. Closes #942. --- .../driver/jdbc/ArrowFlightMetaImpl.java | 23 +++++- .../client/ArrowFlightSqlClientHandler.java | 80 ++++++++++++------- .../arrow/driver/jdbc/ConnectionTest.java | 40 +++++++++- .../jdbc/utils/MockFlightSqlProducer.java | 19 +++++ 4 files changed, 131 insertions(+), 31 deletions(-) diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java index 21cc3e431f..64529b50c8 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java @@ -79,7 +79,8 @@ static Signature newSignature(final String sql, Schema resultSetSchema, Schema p public void closeStatement(final StatementHandle statementHandle) { PreparedStatement preparedStatement = statementHandlePreparedStatementMap.remove(new StatementHandleKey(statementHandle)); - // Testing if the prepared statement was created because the statement can be not created until + // Testing if the prepared statement was created because the statement can be + // not created until // this moment if (preparedStatement != null) { preparedStatement.close(); @@ -224,7 +225,8 @@ public ExecuteResult prepareAndExecute( MetaResultSet.create(handle.connectionId, handle.id, false, handle.signature, null); return new ExecuteResult(Collections.singletonList(metaResultSet)); } catch (SQLTimeoutException e) { - // So far AvaticaStatement(executeInternal) only handles NoSuchStatement and Runtime + // So far AvaticaStatement(executeInternal) only handles NoSuchStatement and + // Runtime // Exceptions. throw new RuntimeException(e); } catch (SQLException e) { @@ -253,6 +255,20 @@ public boolean syncResults( return false; } + @Override + public ConnectionProperties connectionSync(ConnectionHandle ch, ConnectionProperties connProps) { + final ConnectionProperties result = super.connectionSync(ch, connProps); + final String newCatalog = this.connProps.getCatalog(); + if (newCatalog != null) { + try { + ((ArrowFlightConnection) connection).getClientHandler().setCatalog(newCatalog); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + return result; + } + void setDefaultConnectionProperties() { // TODO Double-check this. connProps @@ -268,7 +284,8 @@ PreparedStatement getPreparedStatement(StatementHandle statementHandle) { return statementHandlePreparedStatementMap.get(new StatementHandleKey(statementHandle)); } - // Helper used to look up prepared statement instances later. Avatica doesn't give us the + // Helper used to look up prepared statement instances later. Avatica doesn't + // give us the // signature in // an UPDATE code path so we can't directly use StatementHandle as a map key. private static final class StatementHandleKey { diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java index 5dc7e0e2e9..666996cd95 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java @@ -47,7 +47,6 @@ import org.apache.arrow.flight.FlightStatusCode; import org.apache.arrow.flight.Location; import org.apache.arrow.flight.LocationSchemes; -import org.apache.arrow.flight.SessionOptionValue; import org.apache.arrow.flight.SessionOptionValueFactory; import org.apache.arrow.flight.SetSessionOptionsRequest; import org.apache.arrow.flight.SetSessionOptionsResult; @@ -147,20 +146,26 @@ public List getStreams(final FlightInfo flightInfo) try { for (FlightEndpoint endpoint : flightInfo.getEndpoints()) { if (endpoint.getLocations().isEmpty()) { - // Create a stream using the current client only and do not close the client at the end. + // Create a stream using the current client only and do not close the client at + // the end. endpoints.add( new CloseableEndpointStreamPair( sqlClient.getStream(endpoint.getTicket(), getOptions()), null)); } else { // Clone the builder and then set the new endpoint on it. - // GH-38574: Currently a new FlightClient will be made for each partition that returns a - // non-empty Location then disposed of. It may be better to cache clients because a server - // may report the same Locations. It would also be good to identify when the reported + // GH-38574: Currently a new FlightClient will be made for each partition that + // returns a + // non-empty Location then disposed of. It may be better to cache clients + // because a server + // may report the same Locations. It would also be good to identify when the + // reported // location - // is the same as the original connection's Location and skip creating a FlightClient in + // is the same as the original connection's Location and skip creating a + // FlightClient in // that scenario. - // Also copy the cache to the client so we can share a cache. Cache needs to cache + // Also copy the cache to the client so we can share a cache. Cache needs to + // cache // negative attempts too. List exceptions = new ArrayList<>(); CloseableEndpointStreamPair stream = null; @@ -337,7 +342,8 @@ private boolean isBenignCloseException(FlightRuntimeException fre) { */ private void logSuppressedCloseException( FlightRuntimeException fre, String operationDescription) { - // ARROW-17785 and GH-863: suppress exceptions caused by flaky gRPC layer during shutdown + // ARROW-17785 and GH-863: suppress exceptions caused by flaky gRPC layer during + // shutdown LOGGER.debug("Suppressed error {}", operationDescription, fre); } @@ -388,25 +394,40 @@ public interface PreparedStatement extends AutoCloseable { /** A connection is created with catalog set as a session option. */ private void setSetCatalogInSessionIfPresent() { if (catalog.isPresent()) { - final SetSessionOptionsRequest setSessionOptionRequest = - new SetSessionOptionsRequest( - ImmutableMap.builder() - .put(CATALOG, SessionOptionValueFactory.makeSessionOptionValue(catalog.get())) - .build()); - final SetSessionOptionsResult result = - sqlClient.setSessionOptions(setSessionOptionRequest, getOptions()); + try { + setCatalog(catalog.get()); + } catch (SQLException e) { + throw CallStatus.INVALID_ARGUMENT + .withDescription(e.getMessage()) + .withCause(e) + .toRuntimeException(); + } + } + } + /** + * Sets the catalog for the current session. + * + * @param catalog the catalog to set. + * @throws SQLException if an error occurs while setting the catalog. + */ + public void setCatalog(final String catalog) throws SQLException { + final SetSessionOptionsRequest request = + new SetSessionOptionsRequest( + ImmutableMap.of(CATALOG, SessionOptionValueFactory.makeSessionOptionValue(catalog))); + try { + final SetSessionOptionsResult result = sqlClient.setSessionOptions(request, getOptions()); if (result.hasErrors()) { - Map errors = result.getErrors(); - for (Map.Entry error : errors.entrySet()) { + final Map errors = result.getErrors(); + for (final Map.Entry error : errors.entrySet()) { LOGGER.warn(error.toString()); } - throw CallStatus.INVALID_ARGUMENT - .withDescription( - String.format( - "Cannot set session option for catalog = %s. Check log for details.", catalog)) - .toRuntimeException(); + throw new SQLException( + String.format( + "Cannot set session option for catalog = %s. Check log for details.", catalog)); } + } catch (final FlightRuntimeException e) { + throw new SQLException(e); } } @@ -654,7 +675,8 @@ public static final class Builder { @VisibleForTesting @Nullable Duration connectTimeout; - // These two middleware are for internal use within build() and should not be exposed by builder + // These two middleware are for internal use within build() and should not be + // exposed by builder // APIs. // Note that these middleware may not necessarily be registered. @VisibleForTesting @@ -980,7 +1002,8 @@ public Location getLocation() { * @throws SQLException on error. */ public ArrowFlightSqlClientHandler build() throws SQLException { - // Copy middleware so that the build method doesn't change the state of the builder fields + // Copy middleware so that the build method doesn't change the state of the + // builder fields // itself. Set buildTimeMiddlewareFactories = new HashSet<>(this.middlewareFactories); @@ -988,7 +1011,8 @@ public ArrowFlightSqlClientHandler build() throws SQLException { boolean isUsingUserPasswordAuth = username != null && token == null; try { - // Token should take priority since some apps pass in a username/password even when a token + // Token should take priority since some apps pass in a username/password even + // when a token // is provided if (isUsingUserPasswordAuth) { buildTimeMiddlewareFactories.add(authFactory); @@ -1047,8 +1071,10 @@ public ArrowFlightSqlClientHandler build() throws SQLException { allocator, channelBuilder.build(), clientBuilder.middleware()); final ArrayList credentialOptions = new ArrayList<>(); if (isUsingUserPasswordAuth) { - // If the authFactory has already been used for a handshake, use the existing token. - // This can occur if the authFactory is being re-used for a new connection spawned for + // If the authFactory has already been used for a handshake, use the existing + // token. + // This can occur if the authFactory is being re-used for a new connection + // spawned for // getStream(). if (authFactory.getCredentialCallOption() != null) { credentialOptions.add(authFactory.getCredentialCallOption()); diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java index 72e4b222a3..46762f3319 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.driver.jdbc; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -26,12 +27,15 @@ import java.sql.Driver; import java.sql.DriverManager; import java.sql.SQLException; +import java.util.Map; import java.util.Properties; import org.apache.arrow.driver.jdbc.authentication.UserPasswordAuthentication; import org.apache.arrow.driver.jdbc.client.ArrowFlightSqlClientHandler; import org.apache.arrow.driver.jdbc.utils.ArrowFlightConnectionConfigImpl.ArrowFlightConnectionProperty; import org.apache.arrow.driver.jdbc.utils.MockFlightSqlProducer; import org.apache.arrow.flight.FlightMethod; +import org.apache.arrow.flight.NoOpSessionOptionValueVisitor; +import org.apache.arrow.flight.SessionOptionValue; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.util.AutoCloseables; @@ -614,7 +618,8 @@ public void testJdbcDriverVersionIntegration() throws Exception { var expectedUserAgent = "JDBC Flight SQL Driver " + driverVersion.getDriverVersion().versionString; - // Driver appends version to grpc user-agent header. Assert the header starts with the + // Driver appends version to grpc user-agent header. Assert the header starts + // with the // expected // value and ignored grpc version. assertTrue( @@ -622,4 +627,37 @@ public void testJdbcDriverVersionIntegration() throws Exception { "Expected: " + expectedUserAgent + " but found: " + actualUserAgent); } } + + @Test + public void testSetCatalogShouldUpdateSessionOptions() throws Exception { + final Properties properties = new Properties(); + properties.put(ArrowFlightConnectionProperty.USER.camelName(), userTest); + properties.put(ArrowFlightConnectionProperty.PASSWORD.camelName(), passTest); + properties.put("useEncryption", false); + + try (Connection connection = + DriverManager.getConnection( + "jdbc:arrow-flight-sql://" + + FLIGHT_SERVER_TEST_EXTENSION.getHost() + + ":" + + FLIGHT_SERVER_TEST_EXTENSION.getPort(), + properties)) { + final String catalog = "new_catalog"; + connection.setCatalog(catalog); + + final Map options = PRODUCER.getSessionOptions(); + assertTrue(options.containsKey("catalog")); + String actualCatalog = + options + .get("catalog") + .acceptVisitor( + new NoOpSessionOptionValueVisitor() { + @Override + public String visit(String value) { + return value; + } + }); + assertEquals(catalog, actualCatalog); + } + } } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/MockFlightSqlProducer.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/MockFlightSqlProducer.java index a8874c4869..45c2a96404 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/MockFlightSqlProducer.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/MockFlightSqlProducer.java @@ -52,6 +52,9 @@ import org.apache.arrow.flight.PutResult; import org.apache.arrow.flight.Result; import org.apache.arrow.flight.SchemaResult; +import org.apache.arrow.flight.SessionOptionValue; +import org.apache.arrow.flight.SetSessionOptionsRequest; +import org.apache.arrow.flight.SetSessionOptionsResult; import org.apache.arrow.flight.Ticket; import org.apache.arrow.flight.sql.FlightSqlProducer; import org.apache.arrow.flight.sql.SqlInfoBuilder; @@ -664,6 +667,22 @@ public SqlInfoBuilder getSqlInfoBuilder() { return sqlInfoBuilder; } + private final Map sessionOptions = new HashMap<>(); + + @Override + public void setSessionOptions( + final SetSessionOptionsRequest request, + final CallContext context, + final StreamListener listener) { + sessionOptions.putAll(request.getSessionOptions()); + listener.onNext(new SetSessionOptionsResult(Collections.emptyMap())); + listener.onCompleted(); + } + + public Map getSessionOptions() { + return sessionOptions; + } + private static final class TicketConversionUtils { private TicketConversionUtils() { // Prevent instantiation. From 94dfea8cb20a8e92efe5a188cc7fe5fd28702231 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Thu, 8 Jan 2026 14:11:06 +0100 Subject: [PATCH 104/271] GH-951: Fix CI completely, especially JNI on Windows 2022 and MacOS platforms (#925) This fixes #951 --- ci/scripts/jni_macos_build.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/scripts/jni_macos_build.sh b/ci/scripts/jni_macos_build.sh index 13c0675d38..65ab450666 100755 --- a/ci/scripts/jni_macos_build.sh +++ b/ci/scripts/jni_macos_build.sh @@ -77,7 +77,9 @@ cmake \ cmake --build "${build_dir}/cpp" --target install github_actions_group_end -export JAVA_JNI_CMAKE_ARGS="-DProtobuf_ROOT=${build_dir}/cpp/protobuf_ep-install" +JAVA_JNI_CMAKE_ARGS="-DProtobuf_ROOT=${build_dir}/cpp/_deps/protobuf-build" +JAVA_JNI_CMAKE_ARGS+=" -DProtobuf_SRC_ROOT_FOLDER=${build_dir}/cpp/_deps/protobuf-src" +export JAVA_JNI_CMAKE_ARGS "${source_dir}/ci/scripts/jni_build.sh" \ "${source_dir}" \ "${install_dir}" \ From 007744191764d93628bdbf29084cc77c3c6aac5a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Jan 2026 15:46:59 +0100 Subject: [PATCH 105/271] MINOR: Bump checker.framework.version from 3.52.0 to 3.52.1 (#927) Bumps `checker.framework.version` from 3.52.0 to 3.52.1. Updates `org.checkerframework:checker-qual` from 3.52.0 to 3.52.1
Release notes

Sourced from org.checkerframework:checker-qual's releases.

Checker Framework 3.52.1

Version 3.52.1 (2025-12-02)

User-visible changes:

Added Opt.ifPresentOrElse() method.

Closed issues: #7243, #7398.

Changelog

Sourced from org.checkerframework:checker-qual's changelog.

Version 3.52.1 (2025-12-02)

User-visible changes:

Added Opt.ifPresentOrElse() method.

Closed issues: #7243, #7398.

Commits
  • 72c0de5 new release 3.52.1
  • 0549f00 Remove link.
  • 85842ab Prep for release.
  • 294c7ca Fix the dataflow shaded jars that are published. (#7404)
  • cd7c953 Update cimg/base Docker tag to v2025.12 (#7403)
  • 026bd52 Link from the developer manual to "building from source" in the manual (#7385)
  • e086cba More signature annotations
  • 31ca5d3 Correct shaded dataflow jars. (#7402)
  • 08cc5d1 Update dependency com.amazonaws:aws-java-sdk-bom to v1.12.794 (#7401)
  • 4a22556 Nullness annotations for java.lang.classfile
  • Additional commits viewable in compare view

Updates `org.checkerframework:checker` from 3.52.0 to 3.52.1
Release notes

Sourced from org.checkerframework:checker's releases.

Checker Framework 3.52.1

Version 3.52.1 (2025-12-02)

User-visible changes:

Added Opt.ifPresentOrElse() method.

Closed issues: #7243, #7398.

Changelog

Sourced from org.checkerframework:checker's changelog.

Version 3.52.1 (2025-12-02)

User-visible changes:

Added Opt.ifPresentOrElse() method.

Closed issues: #7243, #7398.

Commits
  • 72c0de5 new release 3.52.1
  • 0549f00 Remove link.
  • 85842ab Prep for release.
  • 294c7ca Fix the dataflow shaded jars that are published. (#7404)
  • cd7c953 Update cimg/base Docker tag to v2025.12 (#7403)
  • 026bd52 Link from the developer manual to "building from source" in the manual (#7385)
  • e086cba More signature annotations
  • 31ca5d3 Correct shaded dataflow jars. (#7402)
  • 08cc5d1 Update dependency com.amazonaws:aws-java-sdk-bom to v1.12.794 (#7401)
  • 4a22556 Nullness annotations for java.lang.classfile
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 777f7d5cef..8cfd4eceff 100644 --- a/pom.xml +++ b/pom.xml @@ -110,7 +110,7 @@ under the License. 10.23.0 true 2.42.0 - 3.52.0 + 3.53.0 1.5.21 none -Xdoclint:none From 32ea946a99d1a0276fc61390eefd2b9b12a8fcf7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Jan 2026 17:25:06 +0100 Subject: [PATCH 106/271] MINOR: Bump org.jacoco:jacoco-maven-plugin from 0.8.13 to 0.8.14 (#924) Bumps [org.jacoco:jacoco-maven-plugin](https://github.com/jacoco/jacoco) from 0.8.13 to 0.8.14.
Release notes

Sourced from org.jacoco:jacoco-maven-plugin's releases.

0.8.14

New Features

  • JaCoCo now officially supports Java 25 (GitHub #1950).
  • Experimental support for Java 26 class files (GitHub #1870).
  • Branches added by the Kotlin compiler for default argument number 33 or higher are filtered out during generation of report (GitHub #1655).
  • Part of bytecode generated by the Kotlin compiler for elvis operator that follows safe call operator is filtered out during generation of report (GitHub #1814, #1954).
  • Part of bytecode generated by the Kotlin compiler for more cases of chained safe call operators is filtered out during generation of report (GitHub #1956).
  • Part of bytecode generated by the Kotlin compiler for invocations of suspendCoroutineUninterceptedOrReturn intrinsic is filtered out during generation of report (GitHub #1929).
  • Part of bytecode generated by the Kotlin compiler for suspending lambdas with parameters is filtered out during generation of report (GitHub #1945).
  • Part of bytecode generated by the Kotlin compiler for suspending functions and lambdas with suspension points that return inline value class is filtered out during generation of report (GitHub #1871).
  • Part of bytecode generated by the Kotlin Compose compiler plugin for pausable composition is filtered out during generation of report (GitHub #1911).
  • Methods generated by the Kotlin serialization compiler plugin are filtered out (GitHub #1885, #1970, #1971).

Fixed bugs

  • Fixed handling of implicit else clause of when with String subject in Kotlin (GitHub #1813, #1940).
  • Fixed handling of implicit default clause of switch by String in Java when compiled by ECJ (GitHub #1813, #1940). Fixed handling of exceptions in chains of safe call operators in Kotlin (GitHub #1819).

Non-functional Changes

  • JaCoCo now depends on ASM 9.9 (GitHub #1965).
Commits
  • 2eb2483 Prepare release v0.8.14
  • de76181 KotlinSerializableFilter should filter more methods (#1971)
  • 89c4bd5 Fix NPE in KotlinSerializableFilter (#1970)
  • 0981128 Migrate release staging to the Central Publisher Portal (#1968)
  • d07bc6b Add filter for bytecode generated by Kotlin serialization compiler plugin (#1...
  • 5e35fd5 Upgrade maven-dependency-plugin to 3.9.0 (#1966)
  • c2fe5cc Upgrade ASM to 9.9 (#1965)
  • b0f8e23 KotlinSafeCallOperatorFilter should filter "unoptimized" safe call followed b...
  • c7bd3f4 Upgrade spotless-maven-plugin to 3.0.0 (#1961)
  • faa289d KotlinSafeCallOperatorFilter should not be affected by presence of pseudo ins...
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.jacoco:jacoco-maven-plugin&package-manager=maven&previous-version=0.8.13&new-version=0.8.14)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8cfd4eceff..a695f91b1d 100644 --- a/pom.xml +++ b/pom.xml @@ -350,7 +350,7 @@ under the License. org.jacoco jacoco-maven-plugin - 0.8.13 + 0.8.14

... (truncated)

Commits
  • 2148f28 v2.11.7
  • e65fa80 #1614 investigating "duplicate" nullable annotations
  • d2ac4f1 Merge pull request #1616 from werli/fix-build-with-optional
  • 1470f17 Conditionally remove unnecessary cast for optional record wither methods
  • c56f082 #1611 #1579 advancing hacks and workarounds for type_use / nullable annotations
  • 95df0cd Custom nullable in nullableAnnotation should not use qualified notation
  • f7a662e #1610 derived arrays, nullable array cloning
  • 9001c82 #1612 false negative in test
  • eecbc5b #1612 fixing and refining no-arg constructors
  • b659a65 whatever to build
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.immutables:value&package-manager=maven&previous-version=2.10.1&new-version=2.11.7)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a695f91b1d..bd6e004424 100644 --- a/pom.xml +++ b/pom.xml @@ -312,7 +312,7 @@ under the License. org.immutables value - 2.10.1 + 2.12.1 From 30eb4cf8cae6b18c9eb934956230888fa2a804e6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 9 Jan 2026 11:40:31 +0100 Subject: [PATCH 108/271] MINOR: Bump com.google.api.grpc:proto-google-common-protos from 2.56.0 to 2.63.1 (#920) Bumps [com.google.api.grpc:proto-google-common-protos](https://github.com/googleapis/sdk-platform-java) from 2.56.0 to 2.63.1.
Release notes

Sourced from com.google.api.grpc:proto-google-common-protos's releases.

v2.63.0

2.63.0 (2025-10-16)

Features

Dependencies

  • Bump errorprone-annotations to v2.42.0 (8d6c1f9)
  • Bump guava to v33.5.0 (8d6c1f9)
  • Bump j2objc-annotations to v3.1 (8d6c1f9)
  • update google auth library dependencies to v1.40.0 (#3945) (1d74663)
  • Upgrade Google Http Java Client to v2.0.2 (#3946) (7fb4f15)

v2.62.3

2.62.3 (2025-10-02)

Bug Fixes

  • mtls: Fix EndpointContext's determineEndpoint logic to respect env var (#3912) (e5948d0)

v2.62.2

2.62.2 (2025-09-18)

Dependencies

v2.62.1

2.62.1 (2025-09-05)

Dependencies

v2.62.0

2.62.0 (2025-08-19)

... (truncated)

Changelog

Sourced from com.google.api.grpc:proto-google-common-protos's changelog.

Changelog

2.64.1 (2025-11-07)

Dependencies

2.64.0 (2025-10-31)

Features

  • [common-protos] Add Carousel widget (1e4a7e5)
  • librariangen: add generate package (#3952) (2f6c75d)
  • librariangen: generate grpc stubs and resource helpers (#3967) (452d703)

Dependencies

2.63.0 (2025-10-16)

Features

Dependencies

  • Bump errorprone-annotations to v2.42.0 (8d6c1f9)
  • Bump guava to v33.5.0 (8d6c1f9)
  • Bump j2objc-annotations to v3.1 (8d6c1f9)
  • update google auth library dependencies to v1.40.0 (#3945) (1d74663)
  • Upgrade Google Http Java Client to v2.0.2 (#3946) (7fb4f15)

2.62.3 (2025-10-02)

Bug Fixes

  • mtls: Fix EndpointContext's determineEndpoint logic to respect env var (#3912) (e5948d0)

... (truncated)

Commits
  • 4aaea1e chore(main): release 2.55.1 (#3695)
  • 2725744 deps: revert "deps: update arrow.version to v18.2.0" (#3694)
  • 3d06ab7 chore(main): release 2.55.1-SNAPSHOT (#3692)
  • a38020a chore(main): release 2.55.0 (#3669)
  • 8fd7b62 build(deps): update dependency com.google.cloud:google-cloud-shared-config to...
  • 2562a7d chore: update googleapis commit at Thu Feb 27 02:27:38 UTC 2025 (#3666)
  • 542d98d chore: add aliases to generate command options. (#3689)
  • 5192426 chore: add java 8 compatibility check (#3688)
  • 25d3101 chore: fix logback-classic version for testing (#3686)
  • 0932605 test: Reduce the LRO timeout value in Showcase tests (#3684)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.api.grpc:proto-google-common-protos&package-manager=maven&previous-version=2.56.0&new-version=2.63.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml index 24beac391e..b1f755844e 100644 --- a/flight/flight-core/pom.xml +++ b/flight/flight-core/pom.xml @@ -134,7 +134,7 @@ under the License. com.google.api.grpc proto-google-common-protos - 2.56.0 + 2.63.2 test From bd3a6ee4efbd637e814867637b6e7b7f8f09bcc8 Mon Sep 17 00:00:00 2001 From: Tamas Mate <50709850+tmater@users.noreply.github.com> Date: Fri, 9 Jan 2026 13:43:33 +0100 Subject: [PATCH 109/271] MINOR: Add private constructor to UuidType singleton (#945) Add private constructor to UuidType singleton. --- .../org/apache/arrow/c/RoundtripTest.java | 81 ++----------------- .../org/apache/arrow/vector/UuidVector.java | 4 +- .../arrow/vector/extension/UuidType.java | 2 + .../apache/arrow/vector/TestListVector.java | 10 +-- .../apache/arrow/vector/TestMapVector.java | 8 +- .../apache/arrow/vector/TestStructVector.java | 4 +- .../org/apache/arrow/vector/TestUuidType.java | 26 +++--- .../complex/impl/TestComplexCopier.java | 10 +-- .../complex/impl/TestPromotableWriter.java | 5 +- .../complex/writer/TestComplexWriter.java | 2 +- 10 files changed, 42 insertions(+), 110 deletions(-) diff --git a/c/src/test/java/org/apache/arrow/c/RoundtripTest.java b/c/src/test/java/org/apache/arrow/c/RoundtripTest.java index 010a305495..f6ff88571e 100644 --- a/c/src/test/java/org/apache/arrow/c/RoundtripTest.java +++ b/c/src/test/java/org/apache/arrow/c/RoundtripTest.java @@ -35,7 +35,6 @@ import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; -import org.apache.arrow.memory.util.hash.ArrowBufHasher; import org.apache.arrow.vector.BaseLargeVariableWidthVector; import org.apache.arrow.vector.BaseVariableWidthVector; import org.apache.arrow.vector.BigIntVector; @@ -44,7 +43,6 @@ import org.apache.arrow.vector.DateMilliVector; import org.apache.arrow.vector.DecimalVector; import org.apache.arrow.vector.DurationVector; -import org.apache.arrow.vector.ExtensionTypeVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.FixedSizeBinaryVector; import org.apache.arrow.vector.Float2Vector; @@ -74,6 +72,7 @@ import org.apache.arrow.vector.UInt2Vector; import org.apache.arrow.vector.UInt4Vector; import org.apache.arrow.vector.UInt8Vector; +import org.apache.arrow.vector.UuidVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; @@ -92,6 +91,7 @@ import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.complex.UnionVector; import org.apache.arrow.vector.complex.impl.UnionMapWriter; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.holders.IntervalDayHolder; import org.apache.arrow.vector.holders.NullableLargeVarBinaryHolder; import org.apache.arrow.vector.holders.NullableUInt4Holder; @@ -100,7 +100,6 @@ import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.ArrowType.ExtensionType; -import org.apache.arrow.vector.types.pojo.ExtensionTypeRegistry; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; @@ -810,9 +809,8 @@ public void testEmptyRunEndEncodedVector() { @Test public void testExtensionTypeVector() { - ExtensionTypeRegistry.register(new UuidType()); final Schema schema = - new Schema(Collections.singletonList(Field.nullable("a", new UuidType()))); + new Schema(Collections.singletonList(Field.nullable("a", UuidType.INSTANCE))); try (final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { // Fill with data UUID u1 = UUID.randomUUID(); @@ -830,13 +828,12 @@ public void testExtensionTypeVector() { assertEquals(root.getSchema(), importedRoot.getSchema()); final Field field = importedRoot.getSchema().getFields().get(0); - final UuidType expectedType = new UuidType(); assertEquals( field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_NAME), - expectedType.extensionName()); + UuidType.INSTANCE.extensionName()); assertEquals( field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_METADATA), - expectedType.serialize()); + UuidType.INSTANCE.serialize()); final UuidVector deserialized = (UuidVector) importedRoot.getFieldVectors().get(0); assertEquals(vector.getValueCount(), deserialized.getValueCount()); @@ -1115,72 +1112,4 @@ private VectorSchemaRoot createTestVSR() { return new VectorSchemaRoot(fields, vectors); } - - static class UuidType extends ExtensionType { - - @Override - public ArrowType storageType() { - return new ArrowType.FixedSizeBinary(16); - } - - @Override - public String extensionName() { - return "uuid"; - } - - @Override - public boolean extensionEquals(ExtensionType other) { - return other instanceof UuidType; - } - - @Override - public ArrowType deserialize(ArrowType storageType, String serializedData) { - if (!storageType.equals(storageType())) { - throw new UnsupportedOperationException( - "Cannot construct UuidType from underlying type " + storageType); - } - return new UuidType(); - } - - @Override - public String serialize() { - return ""; - } - - @Override - public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator) { - return new UuidVector(name, allocator, new FixedSizeBinaryVector(name, allocator, 16)); - } - } - - static class UuidVector extends ExtensionTypeVector { - - public UuidVector( - String name, BufferAllocator allocator, FixedSizeBinaryVector underlyingVector) { - super(name, allocator, underlyingVector); - } - - @Override - public UUID getObject(int index) { - final ByteBuffer bb = ByteBuffer.wrap(getUnderlyingVector().getObject(index)); - return new UUID(bb.getLong(), bb.getLong()); - } - - @Override - public int hashCode(int index) { - return hashCode(index, null); - } - - @Override - public int hashCode(int index, ArrowBufHasher hasher) { - return getUnderlyingVector().hashCode(index, hasher); - } - - public void set(int index, UUID uuid) { - ByteBuffer bb = ByteBuffer.allocate(16); - bb.putLong(uuid.getMostSignificantBits()); - bb.putLong(uuid.getLeastSignificantBits()); - getUnderlyingVector().set(index, bb.array()); - } - } } diff --git a/vector/src/main/java/org/apache/arrow/vector/UuidVector.java b/vector/src/main/java/org/apache/arrow/vector/UuidVector.java index c662a6e064..e0dadd1c67 100644 --- a/vector/src/main/java/org/apache/arrow/vector/UuidVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/UuidVector.java @@ -69,7 +69,7 @@ public class UuidVector extends ExtensionTypeVector public UuidVector( String name, BufferAllocator allocator, FixedSizeBinaryVector underlyingVector) { super(name, allocator, underlyingVector); - this.field = new Field(name, FieldType.nullable(new UuidType()), null); + this.field = new Field(name, FieldType.nullable(UuidType.INSTANCE), null); } /** @@ -99,7 +99,7 @@ public UuidVector( */ public UuidVector(String name, BufferAllocator allocator) { super(name, allocator, new FixedSizeBinaryVector(name, allocator, UUID_BYTE_WIDTH)); - this.field = new Field(name, FieldType.nullable(new UuidType()), null); + this.field = new Field(name, FieldType.nullable(UuidType.INSTANCE), null); } /** diff --git a/vector/src/main/java/org/apache/arrow/vector/extension/UuidType.java b/vector/src/main/java/org/apache/arrow/vector/extension/UuidType.java index f0f2636c82..cd29f930e1 100644 --- a/vector/src/main/java/org/apache/arrow/vector/extension/UuidType.java +++ b/vector/src/main/java/org/apache/arrow/vector/extension/UuidType.java @@ -63,6 +63,8 @@ public class UuidType extends ExtensionType { /** Storage type for UUID: FixedSizeBinary(16). */ public static final ArrowType STORAGE_TYPE = new ArrowType.FixedSizeBinary(UUID_BYTE_WIDTH); + private UuidType() {} + static { ExtensionTypeRegistry.register(INSTANCE); } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index 41a95a8d11..df3a609f53 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -1208,7 +1208,7 @@ public void testGetTransferPairWithField() { @Test public void testListVectorWithExtensionType() throws Exception { - final FieldType type = FieldType.nullable(new UuidType()); + final FieldType type = FieldType.nullable(UuidType.INSTANCE); try (final ListVector inVector = new ListVector("list", allocator, type, null)) { UnionListWriter writer = inVector.getWriter(); writer.allocate(); @@ -1216,7 +1216,7 @@ public void testListVectorWithExtensionType() throws Exception { UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); writer.startList(); - ExtensionWriter extensionWriter = writer.extension(new UuidType()); + ExtensionWriter extensionWriter = writer.extension(UuidType.INSTANCE); extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter.writeExtension(u1); extensionWriter.writeExtension(u2); @@ -1236,7 +1236,7 @@ public void testListVectorWithExtensionType() throws Exception { @Test public void testListVectorReaderForExtensionType() throws Exception { - final FieldType type = FieldType.nullable(new UuidType()); + final FieldType type = FieldType.nullable(UuidType.INSTANCE); try (final ListVector inVector = new ListVector("list", allocator, type, null)) { UnionListWriter writer = inVector.getWriter(); writer.allocate(); @@ -1244,7 +1244,7 @@ public void testListVectorReaderForExtensionType() throws Exception { UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); writer.startList(); - ExtensionWriter extensionWriter = writer.extension(new UuidType()); + ExtensionWriter extensionWriter = writer.extension(UuidType.INSTANCE); extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter.writeExtension(u1); extensionWriter.writeExtension(u2); @@ -1279,7 +1279,7 @@ public void testCopyFromForExtensionType() throws Exception { UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); writer.startList(); - ExtensionWriter extensionWriter = writer.extension(new UuidType()); + ExtensionWriter extensionWriter = writer.extension(UuidType.INSTANCE); extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter.writeExtension(u1); extensionWriter.writeExtension(u2); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java index df8f338f45..d9d2ca50dc 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java @@ -1284,13 +1284,13 @@ public void testMapVectorWithExtensionType() throws Exception { writer.startMap(); writer.startEntry(); writer.key().bigInt().writeBigInt(0); - ExtensionWriter extensionWriter = writer.value().extension(new UuidType()); + ExtensionWriter extensionWriter = writer.value().extension(UuidType.INSTANCE); extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter.writeExtension(u1); writer.endEntry(); writer.startEntry(); writer.key().bigInt().writeBigInt(1); - extensionWriter = writer.value().extension(new UuidType()); + extensionWriter = writer.value().extension(UuidType.INSTANCE); extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter.writeExtension(u2); writer.endEntry(); @@ -1326,13 +1326,13 @@ public void testCopyFromForExtensionType() throws Exception { writer.startMap(); writer.startEntry(); writer.key().bigInt().writeBigInt(0); - ExtensionWriter extensionWriter = writer.value().extension(new UuidType()); + ExtensionWriter extensionWriter = writer.value().extension(UuidType.INSTANCE); extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter.writeExtension(u1); writer.endEntry(); writer.startEntry(); writer.key().bigInt().writeBigInt(1); - extensionWriter = writer.value().extension(new UuidType()); + extensionWriter = writer.value().extension(UuidType.INSTANCE); extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter.writeExtension(u2); writer.endEntry(); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java b/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java index b8abfe1ef6..21ebeebc86 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java @@ -341,7 +341,7 @@ public void testGetTransferPairWithFieldAndCallBack() { @Test public void testStructVectorWithExtensionTypes() { - UuidType uuidType = new UuidType(); + UuidType uuidType = UuidType.INSTANCE; Field uuidField = new Field("struct_child", FieldType.nullable(uuidType), null); Field structField = new Field("struct", FieldType.nullable(new ArrowType.Struct()), List.of(uuidField)); @@ -353,7 +353,7 @@ public void testStructVectorWithExtensionTypes() { @Test public void testStructVectorTransferPairWithExtensionType() { - UuidType uuidType = new UuidType(); + UuidType uuidType = UuidType.INSTANCE; Field uuidField = new Field("uuid_child", FieldType.nullable(uuidType), null); Field structField = new Field("struct", FieldType.nullable(new ArrowType.Struct()), List.of(uuidField)); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestUuidType.java b/vector/src/test/java/org/apache/arrow/vector/TestUuidType.java index 9f7c65b82b..acf9dd6868 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestUuidType.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestUuidType.java @@ -75,21 +75,21 @@ void testConstants() { @Test void testStorageType() { - UuidType type = new UuidType(); + UuidType type = UuidType.INSTANCE; assertEquals(UuidType.STORAGE_TYPE, type.storageType()); assertInstanceOf(ArrowType.FixedSizeBinary.class, type.storageType()); } @Test void testExtensionName() { - UuidType type = new UuidType(); + UuidType type = UuidType.INSTANCE; assertEquals("arrow.uuid", type.extensionName()); } @Test void testExtensionEquals() { - UuidType type1 = new UuidType(); - UuidType type2 = new UuidType(); + UuidType type1 = UuidType.INSTANCE; + UuidType type2 = UuidType.INSTANCE; UuidType type3 = UuidType.INSTANCE; assertTrue(type1.extensionEquals(type2)); @@ -99,20 +99,20 @@ void testExtensionEquals() { @Test void testIsComplex() { - UuidType type = new UuidType(); + UuidType type = UuidType.INSTANCE; assertFalse(type.isComplex()); } @Test void testSerialize() { - UuidType type = new UuidType(); + UuidType type = UuidType.INSTANCE; String serialized = type.serialize(); assertEquals("", serialized); } @Test void testDeserializeValid() { - UuidType type = new UuidType(); + UuidType type = UuidType.INSTANCE; ArrowType storageType = new ArrowType.FixedSizeBinary(UuidType.UUID_BYTE_WIDTH); ArrowType deserialized = assertDoesNotThrow(() -> type.deserialize(storageType, "")); @@ -122,7 +122,7 @@ void testDeserializeValid() { @Test void testDeserializeInvalidStorageType() { - UuidType type = new UuidType(); + UuidType type = UuidType.INSTANCE; ArrowType wrongStorageType = new ArrowType.FixedSizeBinary(32); assertThrows(UnsupportedOperationException.class, () -> type.deserialize(wrongStorageType, "")); @@ -130,7 +130,7 @@ void testDeserializeInvalidStorageType() { @Test void testGetNewVector() { - UuidType type = new UuidType(); + UuidType type = UuidType.INSTANCE; try (FieldVector vector = type.getNewVector("uuid_field", FieldType.nullable(type), allocator)) { assertInstanceOf(UuidVector.class, vector); @@ -141,7 +141,7 @@ void testGetNewVector() { @Test void testVectorOperations() { - UuidType type = new UuidType(); + UuidType type = UuidType.INSTANCE; try (FieldVector vector = type.getNewVector("uuid_field", FieldType.nullable(type), allocator)) { UuidVector uuidVector = (UuidVector) vector; @@ -218,7 +218,7 @@ void testVectorIpcRoundTrip() throws IOException { @Test void testVectorByteArrayOperations() { - UuidType type = new UuidType(); + UuidType type = UuidType.INSTANCE; try (FieldVector vector = type.getNewVector("uuid_field", FieldType.nullable(type), allocator)) { UuidVector uuidVector = (UuidVector) vector; @@ -240,7 +240,7 @@ void testVectorByteArrayOperations() { @Test void testGetNewVectorWithCustomFieldType() { - UuidType type = new UuidType(); + UuidType type = UuidType.INSTANCE; FieldType fieldType = new FieldType(false, type, null); try (FieldVector vector = type.getNewVector("non_nullable_uuid", fieldType, allocator)) { @@ -262,7 +262,7 @@ void testSingleton() { @Test void testUnderlyingVector() { - UuidType type = new UuidType(); + UuidType type = UuidType.INSTANCE; try (FieldVector vector = type.getNewVector("uuid_field", FieldType.nullable(type), allocator)) { UuidVector uuidVector = (UuidVector) vector; diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java index 493a4b26ab..73c1cd3b74 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java @@ -860,7 +860,7 @@ public void testCopyListVectorWithExtensionType() { for (int i = 0; i < COUNT; i++) { listWriter.setPosition(i); listWriter.startList(); - ExtensionWriter extensionWriter = listWriter.extension(new UuidType()); + ExtensionWriter extensionWriter = listWriter.extension(UuidType.INSTANCE); extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter.writeExtension(UUID.randomUUID()); extensionWriter.writeExtension(UUID.randomUUID()); @@ -896,10 +896,10 @@ public void testCopyMapVectorWithExtensionType() { mapWriter.setPosition(i); mapWriter.startMap(); mapWriter.startEntry(); - ExtensionWriter extensionKeyWriter = mapWriter.key().extension(new UuidType()); + ExtensionWriter extensionKeyWriter = mapWriter.key().extension(UuidType.INSTANCE); extensionKeyWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionKeyWriter.writeExtension(UUID.randomUUID()); - ExtensionWriter extensionValueWriter = mapWriter.value().extension(new UuidType()); + ExtensionWriter extensionValueWriter = mapWriter.value().extension(UuidType.INSTANCE); extensionValueWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionValueWriter.writeExtension(UUID.randomUUID()); mapWriter.endEntry(); @@ -934,10 +934,10 @@ public void testCopyStructVectorWithExtensionType() { for (int i = 0; i < COUNT; i++) { structWriter.setPosition(i); structWriter.start(); - ExtensionWriter extensionWriter1 = structWriter.extension("timestamp1", new UuidType()); + ExtensionWriter extensionWriter1 = structWriter.extension("timestamp1", UuidType.INSTANCE); extensionWriter1.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter1.writeExtension(UUID.randomUUID()); - ExtensionWriter extensionWriter2 = structWriter.extension("timestamp2", new UuidType()); + ExtensionWriter extensionWriter2 = structWriter.extension("timestamp2", UuidType.INSTANCE); extensionWriter2.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter2.writeExtension(UUID.randomUUID()); structWriter.end(); diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java index a4594024fa..c71717a027 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java @@ -785,7 +785,7 @@ public void testExtensionType() throws Exception { try (final NonNullableStructVector container = NonNullableStructVector.empty(EMPTY_SCHEMA_PATH, allocator); final UuidVector v = - container.addOrGet("uuid", FieldType.nullable(new UuidType()), UuidVector.class); + container.addOrGet("uuid", FieldType.nullable(UuidType.INSTANCE), UuidVector.class); final PromotableWriter writer = new PromotableWriter(v, container)) { UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); @@ -810,7 +810,8 @@ public void testExtensionType() throws Exception { public void testExtensionTypeForList() throws Exception { try (final ListVector container = ListVector.empty(EMPTY_SCHEMA_PATH, allocator); final UuidVector v = - (UuidVector) container.addOrGetVector(FieldType.nullable(new UuidType())).getVector(); + (UuidVector) + container.addOrGetVector(FieldType.nullable(UuidType.INSTANCE)).getVector(); final PromotableWriter writer = new PromotableWriter(v, container)) { UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index 871a3cc461..3a8f3f8e6a 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -2510,7 +2510,7 @@ public void extensionWriterReader() throws Exception { StructWriter rootWriter = writer.rootAsStruct(); { - ExtensionWriter extensionWriter = rootWriter.extension("uuid1", new UuidType()); + ExtensionWriter extensionWriter = rootWriter.extension("uuid1", UuidType.INSTANCE); extensionWriter.setPosition(0); extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter.writeExtension(u1); From 7d4cf21bd6502fc650dfad4a6e9fbe0ae5cf4360 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 9 Jan 2026 15:22:30 +0100 Subject: [PATCH 110/271] MINOR: Bump io.netty:netty-bom from 4.1.119.Final to 4.2.7.Final (#887) Bumps [io.netty:netty-bom](https://github.com/netty/netty) from 4.1.119.Final to 4.2.7.Final.
Commits
  • 511cbac [maven-release-plugin] prepare release netty-4.2.7.Final
  • bf1cad6 Adjust plugin config to not publish testsuite artifacts
  • 690f56f [maven-release-plugin] rollback the release of netty-4.2.7.Final
  • 63b5232 [maven-release-plugin] prepare for next development iteration
  • 9f99dfd [maven-release-plugin] prepare release netty-4.2.7.Final
  • 551c32a Upgrade publishing plugin
  • a6660fe [maven-release-plugin] rollback the release of netty-4.2.7.Final
  • 297b7c1 [maven-release-plugin] prepare for next development iteration
  • 2c89a17 [maven-release-plugin] prepare release netty-4.2.7.Final
  • 1782e8c Merge commit from fork
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=io.netty:netty-bom&package-manager=maven&previous-version=4.1.119.Final&new-version=4.2.7.Final)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index bd6e004424..f3025e16ee 100644 --- a/pom.xml +++ b/pom.xml @@ -97,7 +97,7 @@ under the License. 5.12.2 2.0.17 33.4.8-jre - 4.1.127.Final + 4.2.9.Final 1.73.0 4.33.1 2.18.3 From 794277963b4c42cd019230ec096f354c2cb685f1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 10 Jan 2026 07:25:50 +0100 Subject: [PATCH 111/271] MINOR: Bump parquet.version from 1.15.2 to 1.16.0 (#913) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps `parquet.version` from 1.15.2 to 1.16.0. Updates `org.apache.parquet:parquet-avro` from 1.15.2 to 1.16.0
Release notes

Sourced from org.apache.parquet:parquet-avro's releases.

Apache Parquet Java 1.16.0

What's Changed

... (truncated)

Commits
  • 402c381 [maven-release-plugin] prepare release apache-parquet-1.16.0-rc2
  • 0e279ef Add comparator for UnknownLogicalType (#3292) (#3295)
  • f85f083 [maven-release-plugin] prepare for next development iteration
  • 36d1880 [maven-release-plugin] prepare release apache-parquet-1.16.0-rc1
  • 2d463ee [maven-release-plugin] prepare for next development iteration
  • 1e3d701 [maven-release-plugin] prepare release apache-parquet-1.16.0-rc0
  • 7ef2f91 bump parquet-plugins to 1.16.0 for release
  • 0d25e13 MINOR: Bump parquet-format to 2.12.0 (#3285)
  • 299b0ae MINOR: Bump thrift to 0.22.0 (#3229)
  • 36a5f9c Bump jackson.version from 2.19.0 to 2.19.2 (#3266)
  • Additional commits viewable in compare view

Updates `org.apache.parquet:parquet-hadoop` from 1.15.2 to 1.16.0
Release notes

Sourced from org.apache.parquet:parquet-hadoop's releases.

Apache Parquet Java 1.16.0

What's Changed

... (truncated)

Commits
  • 402c381 [maven-release-plugin] prepare release apache-parquet-1.16.0-rc2
  • 0e279ef Add comparator for UnknownLogicalType (#3292) (#3295)
  • f85f083 [maven-release-plugin] prepare for next development iteration
  • 36d1880 [maven-release-plugin] prepare release apache-parquet-1.16.0-rc1
  • 2d463ee [maven-release-plugin] prepare for next development iteration
  • 1e3d701 [maven-release-plugin] prepare release apache-parquet-1.16.0-rc0
  • 7ef2f91 bump parquet-plugins to 1.16.0 for release
  • 0d25e13 MINOR: Bump parquet-format to 2.12.0 (#3285)
  • 299b0ae MINOR: Bump thrift to 0.22.0 (#3229)
  • 36a5f9c Bump jackson.version from 2.19.0 to 2.19.2 (#3266)
  • Additional commits viewable in compare view

You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dataset/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataset/pom.xml b/dataset/pom.xml index 66233c3970..6bca75bdd0 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -32,7 +32,7 @@ under the License. ../../../cpp/release-build/ - 1.15.2 + 1.16.0 1.12.0 From a602e6a21e6a783dd1a23403ee7614ef6031516d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 10 Jan 2026 10:05:07 +0100 Subject: [PATCH 112/271] MINOR: Bump org.immutables:value-annotations from 2.10.1 to 2.11.7 (#917) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.immutables:value-annotations](https://github.com/immutables/immutables) from 2.10.1 to 2.11.7.
Release notes

Sourced from org.immutables:value-annotations's releases.

2.11.7

Maintenance & refinements release

Thank you for the bug reports and suggestions!

Issues

  • #1621 Version 2.11.7 tag not present on GitHub
  • #1611 Jspecify Nullable doesn't work properly with generics
  • #1612 Conflicting constructor on empty interfaces when allParameters = true, and privateNoArgConstructor = true/ protectedNoArgConstructor = true (edge case regression after #1604)
  • #1579 TYPE_USE Nullable annotation not respected in the builder for arrays (arrays/elements annotation mirrors are missing) (addressed with some source code parsing, which requires -sourcepath to be provided during compilation)

PRs

New Contributors

Full Changelog: https://github.com/immutables/immutables/compare/2.11.6...2.11.7

2.11.6

Maintenance & refinements release

Thank you for the bug reports and suggestions!

Issues

  • #1602 Avoid calling check/validation method twice when using plain public constructors (@Style(of = "new")
  • #1603 Fixed compilation error with staged builders and complex generics
  • #1604 parameterless constructor when there's no attributes, but allParameters=true or allMandatoryParameters=true

Full Changelog: https://github.com/immutables/immutables/compare/2.11.5...2.11.6

2.11.5

Maintenance & refinements release

Thank you for the bug reports and PRs!

Issues

  • #1602 @Check methods (returning void i.e. non-normalizing) now works from plain public constructors (@Style(of = "new")
  • #1583 Staged builder now works for "outside"/top-level class builders, including record builders (with *BuildStages class generated to hold stage interfaces)
  • #1598 fixed: @Data from org.immutables:datatype can be used as meta-annotation
  • #1433 additionalStrictContainerConstructor=false can be used to suppress redundant strict factory method (constructor) overload

PRs

New Contributors

... (truncated)

Commits
  • 2148f28 v2.11.7
  • e65fa80 #1614 investigating "duplicate" nullable annotations
  • d2ac4f1 Merge pull request #1616 from werli/fix-build-with-optional
  • 1470f17 Conditionally remove unnecessary cast for optional record wither methods
  • c56f082 #1611 #1579 advancing hacks and workarounds for type_use / nullable annotations
  • 95df0cd Custom nullable in nullableAnnotation should not use qualified notation
  • f7a662e #1610 derived arrays, nullable array cloning
  • 9001c82 #1612 false negative in test
  • eecbc5b #1612 fixing and refining no-arg constructors
  • b659a65 whatever to build
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.immutables:value-annotations&package-manager=maven&previous-version=2.10.1&new-version=2.11.7)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f3025e16ee..6d1181d1d3 100644 --- a/pom.xml +++ b/pom.xml @@ -181,7 +181,7 @@ under the License. org.immutables value-annotations - 2.10.1 + 2.12.1 provided From e620c4481b51b47be952918b1d7e1441b22f0b44 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jan 2026 14:13:57 +0100 Subject: [PATCH 113/271] MINOR: Bump logback.version from 1.5.21 to 1.5.24 (#962) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps `logback.version` from 1.5.21 to 1.5.24. Updates `ch.qos.logback:logback-classic` from 1.5.21 to 1.5.24
Release notes

Sourced from ch.qos.logback:logback-classic's releases.

Logback 1.5.24

2026-01-06 Release of logback version 1.5.24

• Added ExpressionPropertyCondition a PropertyCondition that can evaluate boolean expressions similar to Java. See the relevant documentation for further details.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit 62bc5fc245dd3a52f3dd45e232733f4cefb4806d associated with the tag v_1.5.24. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Logback 1.5.23

2025-12-21 Release of logback version 1.5.23

• In response to issues/959 file name collisions are detected at configuration time by analyzing the configuration file and no longer at run time. This avoids the ConcurrentModificationException reported in the issue.

• ZIP and XZ compression now use a BufferedOutputStream when writing to the compressed file. This issue was reported in issues/988.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit 0bcc3feb54a6d99caac70969ee5f8334aad1fbaf associated with the tag v_1.5.23. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Logback 1.5.22

2025-12-11 Release of logback version 1.5.22

• In order to prevent involuntary information leakage, Logback will no longer output the value of a substituted variable, if the variable name contains any of the case-insensitive strings "password", "secret" or "confidential". This problem was reported by Chintan Rohila in issues/986.

• Logback now takes the overridden toString() method of Throwable subclasses into account when printing stack traces. This issue was reported in LOGBACK-543 by Alvin Chee, with a fix provided in PR 404 by Brett Kail.

• Instead of limit-counting guard, Logback now uses a tumbling-window guard to rate limit internal error messages.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit 572379aabd2f672b49593e4020696c624541e5b0 associated with the tag v_1.5.22. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits
  • 62bc5fc prepare release 1.5.24
  • aac604d typo fix of local variable name
  • 8a6df9e ExpressionPropertyCondition constructor should be public
  • 95e588c minor changes in ExpressionPropertyCondition
  • 859f5a1 added ExpressionPropertyCondition capable of parsing logical expressions on p...
  • 348075a start work on 1.5.24-SNAPSHOT
  • 0bcc3fe prepare release 1.5.23
  • 4627dbd better to use BufferedOutputStream during ZIP and XZ compression, especially ...
  • 299f091 add collision test in presence of conditional processing
  • b446f3f In Context, remove collision map
  • Additional commits viewable in compare view

Updates `ch.qos.logback:logback-core` from 1.5.21 to 1.5.24
Release notes

Sourced from ch.qos.logback:logback-core's releases.

Logback 1.5.24

2026-01-06 Release of logback version 1.5.24

• Added ExpressionPropertyCondition a PropertyCondition that can evaluate boolean expressions similar to Java. See the relevant documentation for further details.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit 62bc5fc245dd3a52f3dd45e232733f4cefb4806d associated with the tag v_1.5.24. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Logback 1.5.23

2025-12-21 Release of logback version 1.5.23

• In response to issues/959 file name collisions are detected at configuration time by analyzing the configuration file and no longer at run time. This avoids the ConcurrentModificationException reported in the issue.

• ZIP and XZ compression now use a BufferedOutputStream when writing to the compressed file. This issue was reported in issues/988.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit 0bcc3feb54a6d99caac70969ee5f8334aad1fbaf associated with the tag v_1.5.23. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Logback 1.5.22

2025-12-11 Release of logback version 1.5.22

• In order to prevent involuntary information leakage, Logback will no longer output the value of a substituted variable, if the variable name contains any of the case-insensitive strings "password", "secret" or "confidential". This problem was reported by Chintan Rohila in issues/986.

• Logback now takes the overridden toString() method of Throwable subclasses into account when printing stack traces. This issue was reported in LOGBACK-543 by Alvin Chee, with a fix provided in PR 404 by Brett Kail.

• Instead of limit-counting guard, Logback now uses a tumbling-window guard to rate limit internal error messages.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit 572379aabd2f672b49593e4020696c624541e5b0 associated with the tag v_1.5.22. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits
  • 62bc5fc prepare release 1.5.24
  • aac604d typo fix of local variable name
  • 8a6df9e ExpressionPropertyCondition constructor should be public
  • 95e588c minor changes in ExpressionPropertyCondition
  • 859f5a1 added ExpressionPropertyCondition capable of parsing logical expressions on p...
  • 348075a start work on 1.5.24-SNAPSHOT
  • 0bcc3fe prepare release 1.5.23
  • 4627dbd better to use BufferedOutputStream during ZIP and XZ compression, especially ...
  • 299f091 add collision test in presence of conditional processing
  • b446f3f In Context, remove collision map
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6d1181d1d3..0659c43008 100644 --- a/pom.xml +++ b/pom.xml @@ -111,7 +111,7 @@ under the License. true 2.42.0 3.53.0 - 1.5.21 + 1.5.24 none -Xdoclint:none From 05292ac0d16a4735189683b43c2084dd2ee92e20 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jan 2026 14:14:29 +0100 Subject: [PATCH 114/271] MINOR: Bump org.codehaus.mojo:exec-maven-plugin from 3.5.0 to 3.6.3 (#959) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.codehaus.mojo:exec-maven-plugin](https://github.com/mojohaus/exec-maven-plugin) from 3.5.0 to 3.6.3.
Release notes

Sourced from org.codehaus.mojo:exec-maven-plugin's releases.

3.6.3

📝 Documentation updates

👻 Maintenance

📦 Dependency updates

3.6.2

🚀 New features and improvements

📦 Dependency updates

3.6.1

🐛 Bug Fixes

📦 Dependency updates

3.6.0

🚀 New features and improvements

🐛 Bug Fixes

... (truncated)

Commits
  • fe1fa8c [maven-release-plugin] prepare release 3.6.3
  • 5b3feca Bump asm.version from 9.9 to 9.9.1
  • efc7faa Bump org.apache.commons:commons-exec from 1.5.0 to 1.6.0
  • cdaf267 JUnit 5 best practices (#505)
  • f3f5997 Move ExecJavaMojoTest, ExecMojoTest to JUnit 5
  • 03b87b5 Document thread group isolation limitation in java goal (#503)
  • 7a66c3e Add support for JEP 512 for for package-private static main methods with and ...
  • a6d01ef Move to JUnit 5
  • 88d5961 [maven-release-plugin] prepare for next development iteration
  • 416fdf1 [maven-release-plugin] prepare release 3.6.2
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.codehaus.mojo:exec-maven-plugin&package-manager=maven&previous-version=3.5.0&new-version=3.6.3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0659c43008..2e734396ee 100644 --- a/pom.xml +++ b/pom.xml @@ -505,7 +505,7 @@ under the License. org.codehaus.mojo exec-maven-plugin - 3.5.0 + 3.6.3 org.codehaus.mojo From 3206fe558b21ac75e6754149a4ca5961d4d29cb7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jan 2026 14:14:51 +0100 Subject: [PATCH 115/271] MINOR: Bump org.apache.commons:commons-text from 1.13.1 to 1.15.0 (#956) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.apache.commons:commons-text](https://github.com/apache/commons-text) from 1.13.1 to 1.15.0.
Changelog

Sourced from org.apache.commons:commons-text's changelog.

Apache Commons Text 1.15.0 Release Notes

The Apache Commons Text team is pleased to announce the release of Apache Commons Text 1.15.0.

Apache Commons Text is a set of utility functions and reusable components for processing and manipulating text in a Java environment.

Release 1.15.0. This is a feature and maintenance release. Java 8 or later is required.

New features

  •  Add experimental CycloneDX VEX file
[#683](https://github.com/apache/commons-text/issues/683). Thanks to
Piotr P. Karwasz, Gary Gregory.
  • TEXT-235: Add Damerau-Levenshtein distance #687. Thanks to LorgeN, Gary Gregory.
  •  Add unit tests to increase coverage
[#719](https://github.com/apache/commons-text/issues/719). Thanks to
Michael Hausegger, Gary Gregory.

  •  Add new test for CharSequenceTranslator#with()
[#725](https://github.com/apache/commons-text/issues/725). Thanks to
Michael Hausegger, Gary Gregory.

  •  Add tests and assertions to
org.apache.commons.text.similarity to get to 100% code coverage
[#727](https://github.com/apache/commons-text/issues/727),
[#728](https://github.com/apache/commons-text/issues/728). Thanks to
Michael Hausegger.

Fixed Bugs

  •  Fix exception message typo in
XmlStringLookup.XmlStringLookup(Map, Path...). Thanks to Gary Gregory.
  • TEXT-236: Inserting at the end of a TextStringBuilder throws a StringIndexOutOfBoundsException. Thanks to Pierre Post, Sumit Bera, Alex Herbert, Gary Gregory.
  •  Fix TextStringBuilderTest.testAppendToCharBuffer() to use
proper argument type
[#724](https://github.com/apache/commons-text/issues/724). Thanks to
Michael Hausegger.

  •  Fix Apache RAT plugin console warnings. Thanks to Gary
Gregory.

  •  Fix site XML to use version 2.0.0 XML schema. Thanks to Gary
Gregory.

  •  Removed unreachable threshold verification code in
src/main/java/org/apache/commons/text/similarity
[#730](https://github.com/apache/commons-text/issues/730). Thanks to
Michael Hausegger.

  •  Enable secure processing for the XML parser in
XmlStringLookup in case the underlying JAXP implementation doesn't
[#729](https://github.com/apache/commons-text/issues/729). Thanks to 김민재
(minjas0507), Gary Gregory, Piotr Karwasz.

Changes

  •  Bump org.apache.commons:commons-parent from 85 to 93
[#704](https://github.com/apache/commons-text/issues/704),
[#723](https://github.com/apache/commons-text/issues/723),
[#726](https://github.com/apache/commons-text/issues/726). Thanks to
Gary Gregory.

  •  Bump commons.bytebuddy.version from 1.17.6 to 1.18.2
[#696](https://github.com/apache/commons-text/issues/696),
[#722](https://github.com/apache/commons-text/issues/722). Thanks to
Gary Gregory.

  •  Bump graalvm.version from 24.2.2 to 25.0.1
[#703](https://github.com/apache/commons-text/issues/703),
[#716](https://github.com/apache/commons-text/issues/716). Thanks to
Gary Gregory, Dependabot.

  •  Bump org.apache.commons:commons-lang3 from 3.18.0 to 3.20.0.
Thanks to Gary Gregory.

  •  Bump commons-io:commons-io from 2.20.0 to 2.21.0. Thanks to
Gary Gregory.

Historical list of changes: https://commons.apache.org/proper/commons-text/changes.html

For complete information on Apache Commons Text, including instructions on how to submit bug reports, patches, or suggestions for improvement, see the Apache Commons Text website:

https://commons.apache.org/proper/commons-text

Download page: https://commons.apache.org/proper/commons-text/download_text.cgi

... (truncated)

Commits
  • 04e9374 Prepare for the release candidate 1.15.0 RC1
  • 502c4c4 Prepare for the next release candidate
  • c6e17ec Use direct access
  • 58e1e12 Simplify XML FSP (#731)
  • b5052c9 Bump actions/setup-java from 5.0.0 to 5.1.0
  • 2e2d4bc Revert "Bump actions/setup-java from 5.0.0 to 5.1.0"
  • b0ddbd1 Bump actions/setup-java from 5.0.0 to 5.1.0
  • 1c2d382 Add tests with external DTD
  • ed3df4b Internal clean up
  • bb508f3 Bump actions/checkout from 6.0.0 to 6.0.1
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.commons:commons-text&package-manager=maven&previous-version=1.13.1&new-version=1.15.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql/pom.xml b/flight/flight-sql/pom.xml index 15d00e3e18..4175ff70d3 100644 --- a/flight/flight-sql/pom.xml +++ b/flight/flight-sql/pom.xml @@ -113,7 +113,7 @@ under the License. org.apache.commons commons-text - 1.13.1 + 1.15.0 test From 936a31a4e59f099fa422c0a4d6a4316941dcd841 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jan 2026 14:16:04 +0100 Subject: [PATCH 116/271] MINOR: Bump io.grpc:grpc-bom from 1.73.0 to 1.78.0 (#958) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [io.grpc:grpc-bom](https://github.com/grpc/grpc-java) from 1.73.0 to 1.78.0.
Release notes

Sourced from io.grpc:grpc-bom's releases.

V1.78.0

Bug Fixes

  • core: Fix shutdown failing accepted RPCs during channel startup (02e98a806). This fixes a race where RPCs could fail with "UNAVAILABLE: Channel shutdown invoked" even though they were created before channel.shutdown()
  • okhttp: Fix race condition overwriting MAX_CONCURRENT_STREAMS (#12548) (8d49dc1c9)
  • binder: Stop leaking this from BinderServerTransport's ctor (#12453) (89d77e062)
  • rls: Avoid missed config update from reentrancy (55ae1d054). This fixes a regression since 1.75.0 triggered by CdsLb being converted to XdsDepManager. Without this fix, a second channel to the same target may hang when starting, causing DEADLINE_EXCEEDED, and unhang when the control plane delivers an update (e.g., endpoint address update)

Improvements

  • xds: gRFC A88 - Changes to XdsClient Watcher APIs (#12446) (f385add31). We now have improved xDS error handling and this provides a clearer mechanism for the xDS server to report per-resource errors to the client, resulting in better error messages for debugging and faster detection of non-existent resources. This also improves the handling of all xDS-related data errors and the behavior of the xDS resource timer.
  • rls: Control plane channel monitor state and back off handling (#12460) (26c1c1341). Resets RLS request backoff timers when the Control plane channel state transitions to READY. Also when the backoff timer expires, instead of making a RLS request immediately, it just causes a picker update to allow making rpc again to the RLS target.
  • core: simplify DnsNameResolver.resolveAddresses() (4843256af)
  • netty: Run handshakeCompleteRunnable in success cases (283f1031f)
  • api,netty: Add custom header support for HTTP CONNECT proxy (bbc0aa369)
  • binder: Pre-factor out the guts of the BinderClientTransport handshake. (9313e87df)
  • compiler: Add RISC-V 64-bit architecture support to compiler build configuration (725ab22f3)
  • core: Release lock before closing shared resource (cb73f217e). Shared resources are internal to gRPC for sharing expensive objects across channels and servers, like threads. This reduces the chances of forming a deadlock, like seen with s2a in d50098f
  • Upgrade gson to 2.12.1 (6dab2ceab)
  • Upgrade dependencies (f36defa2d). proto-google-common-protos to 2.63.1, google-auth-library to 1.40.0, error-prone annotations to 2.44.0, guava to 33.5.0-android, opentelemetry to 1.56.0
  • compiler: Update maximum supported protobuf edition to EDITION_2024 (2f64092b8)
  • binder: Introduce server authorization strategy v2 (d9710725d). Adds support for android:isolatedProcess Services and moves all security checks to the handshake, making subsequent transactions more efficient.

New Features

  • compiler: Upgrade to C++ protobuf 33.1 (#12534) (58ae5f808).
  • util: Add gRFC A68 random subsetting LB (48a42889d). The policy uses the name random_subsetting_experimental. If it is working for you, tell us so we can gauge marking it stable. While the xDS portions haven’t yet landed, it is possible to use with xDS with JSON-style Structs as supported by gRFC A52
  • xds: Support for System Root Certs (#12499) (51611bad1). Most service mesh workloads use mTLS, as described in gRFC A29. However, there are cases where it is useful for applications to use normal TLS rather than using certificates for workload identity, such as when a mesh wants to move some workloads behind a reverse proxy. The xDS CertificateValidationContext message (see envoyproxy/envoy#34235) has a system_root_certs field. In the gRPC client, if this field is present and the ca_certificate_provider_instance field is unset, system root certificates will be used for validation. This implements gRFC A82.
  • xds: Support for GCP Authentication Filter (#12499) (51611bad1). In service mesh environments, there are cases where intermediate proxies make it impossible to rely on mTLS for end-to-end authentication. These cases can be addressed instead by the use of service account identity JWT tokens. The xDS GCP Authentication filter provides a mechanism for attaching such JWT tokens as gRPC call credentials on GCP. gRPC already supports a framework for xDS HTTP filters, as described in gRFC A39. This release supports the GCP Authentication filter under this framework as described in gRFC A83.
  • xds: Support for xDS-based authority rewriting (#12499) (51611bad1). gRPC supports getting routing configuration from an xDS server, as described in gRFCs A27 and A28. The xDS configuration can configure the client to rewrite the authority header on requests. This functionality can be useful in cases where the server is using the authority header to make decisions about how to process the request, such as when multiple hosts are handled via a reverse proxy. Note that this feature is solely about rewriting the authority header on data plane RPCs; it does not affect the authority used in the TLS handshake.
    As mentioned in gRFC A29, there are use-cases for gRPC that prohibit trusting the xDS server to control security-centric configuration. The authority rewriting feature falls under the same umbrella as mTLS configuration. As a result, the authority rewriting feature will only be enabled when the bootstrap config for the xDS server has trusted_xds_server in the server_features field.
  • xds: xDS based SNI setting and SAN validation (#12378) (0567531). When using xDS credentials make SNI for the Tls handshake to be configured via xDS, rather than use the channel authority as the SNI, and make SAN validation to be able to use the SNI sent when so instructed via xDS. Implements gRFC A101.

Documentation

  • api: Document gRFC A18 TCP_USER_TIMEOUT handling for keepalive (da7038782)
  • core: Fix AbstractClientStream Javadoc (28a6130e8)
  • examples: Document how to preserve META-INF/services in uber jars (97695d523)

Thanks to

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=io.grpc:grpc-bom&package-manager=maven&previous-version=1.73.0&new-version=1.78.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2e734396ee..a9eb5ebbdd 100644 --- a/pom.xml +++ b/pom.xml @@ -98,7 +98,7 @@ under the License. 2.0.17 33.4.8-jre 4.2.9.Final - 1.73.0 + 1.78.0 4.33.1 2.18.3 3.4.2 From 109063f7d970d717e2210fd294cef09f99d16706 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 Jan 2026 11:27:18 +0100 Subject: [PATCH 117/271] MINOR: Bump com.github.ben-manes.caffeine:caffeine from 3.2.0 to 3.2.3 (#960) Bumps [com.github.ben-manes.caffeine:caffeine](https://github.com/ben-manes/caffeine) from 3.2.0 to 3.2.3.
Release notes

Sourced from com.github.ben-manes.caffeine:caffeine's releases.

3.2.3

  • Fixed frequency tracking of weak keys to use the object's identity hash code (#1902)
  • Added support for underscores in CaffeineSpec when using numeric literals (#1890)
  • Improved the external api to no longer lock when querying for the maximum size or weighted size (#1897)
  • Added detection and recovery when a custom CompletableFuture is in an inconsistent state (quarkus#50513)

3.2.2

  • Fixed characteristics returned by Spliterators (#1883)

3.2.1

  • Fixed computeIfAbsent for an async cache's synchronous view to retry if incomplete
  • Improved CaffeineSpec when being reflectively constructed (#1839)
  • Improved the handling of negative durations with variable expiration
  • Fixed intermittent null after replacing a weak/soft value (#1820)
Commits
  • 5227a98 minor build touchups
  • cc3f37d reorganize into separate gradle test suites
  • 2299add Allow users to read the maximum size without locking (fixes #1897)
  • 6250b38 clarify policy javadoc and add corresponding test cases (fixes #1927)
  • c975fc0 upgrade error-prone static analyzer
  • d8e0a92 allow the project.version to be overridden by external builders
  • 0e46d22 detect if the user's future is inconsistent with the results
  • 1971428 use the assemble task for a full build without running the test suites
  • 782ac79 use the key reference with the frequency sketch (fixes #1902)
  • e0dd94b minor build clean up
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.github.ben-manes.caffeine:caffeine&package-manager=maven&previous-version=3.2.0&new-version=3.2.3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 965e071e72..8801ad8178 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -151,7 +151,7 @@ under the License. com.github.ben-manes.caffeine caffeine - 3.2.0 + 3.2.3 From 68451bfd55f6fff268a0e11dd4f6e4b3e2a025d2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Jan 2026 16:38:46 +0100 Subject: [PATCH 118/271] MINOR: Bump org.apache.avro:avro from 1.12.0 to 1.12.1 (#955) Bumps org.apache.avro:avro from 1.12.0 to 1.12.1. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.avro:avro&package-manager=maven&previous-version=1.12.0&new-version=1.12.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dataset/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dataset/pom.xml b/dataset/pom.xml index 6bca75bdd0..2d582268a6 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -33,7 +33,7 @@ under the License. ../../../cpp/release-build/ 1.16.0 - 1.12.0 + 1.12.1 diff --git a/pom.xml b/pom.xml index a9eb5ebbdd..d6fdfdd477 100644 --- a/pom.xml +++ b/pom.xml @@ -103,7 +103,7 @@ under the License. 2.18.3 3.4.2 25.2.10 - 1.12.0 + 1.12.1 5.17.0 2 From 9cdda52550e5d95b9868e5fda26d51465c8c258d Mon Sep 17 00:00:00 2001 From: Joana Hrotko Date: Thu, 15 Jan 2026 13:49:14 +0000 Subject: [PATCH 119/271] GH-891: Add ExtensionTypeWriterFactory to TransferPair (#892) ## What's Changed This PR simplifies extension type writer creation by moving from a factory-based pattern to a type-based pattern. Instead of passing `ExtensionTypeWriterFactory` instances through multiple API layers, extension types now provide their own writers via a new `getNewFieldWriter()` method on `ArrowType.ExtensionType`. - Added `getNewFieldWriter(ValueVector)` abstract method to `ArrowType.ExtensionType` - Removed `ExtensionTypeWriterFactory` interface and all implementations - Removed factory parameters from `ComplexCopier`, `PromotableWriter`, and `TransferPair` APIs - Updated `UnionWriter` to support extension types (previously threw `UnsupportedOperationException`) - Simplified extension type implementations (`UuidType`, `OpaqueType`) The factory pattern didn't scale well. Each new extension type required creating a separate factory class and passing it through multiple API layers. This was especially painful for external developers who had to maintain two classes per extension type and manage factory parameters everywhere. The new approach follows the same pattern as `MinorType`, where each type knows how to create its own writer. This reduces boilerplate, simplifies the API, and makes it easier to implement custom extension types outside arrow-java. ## Breaking Changes - `ExtensionTypeWriterFactory` has been removed - Extension types must now implement `getNewFieldWriter(ValueVector vector)` method - ExtensionHolders must implement `type()` which returns the `ExtensionType` for that Holder - (Writers are obtained directly from the extension type, not from a factory) ### Migration Guide - _Extension types must now implement `getNewFieldWriter(ValueVector vector)` method_ ```java public class UuidType extends ExtensionType { ... @Override public FieldWriter getNewFieldWriter(ValueVector vector) { return new UuidWriterImpl((UuidVector) vector); } ... } ``` - _ExtensionHolders must implement `type()` which returns the `ExtensionType` for that Holder_ ```java public class UuidHolder extends ExtensionHolder { ... @Override public ArrowType type() { return UuidType.INSTANCE; } ``` - How to use Extension Writers? **Before:** ```java writer.extension(UuidType.INSTANCE); writer.addExtensionTypeWriterFactory(extensionTypeWriterFactory); writer.writeExtension(value); ``` **After:** ```java writer.extension(UuidType.INSTANCE); writer.writeExtension(value); ``` - Also `copyAsValue` does not need to provide the factory anymore. Closes #891 . --- .../templates/AbstractFieldReader.java | 5 +- .../templates/AbstractFieldWriter.java | 11 +- .../src/main/codegen/templates/ArrowType.java | 6 + .../main/codegen/templates/BaseReader.java | 3 - .../main/codegen/templates/BaseWriter.java | 7 +- .../main/codegen/templates/ComplexCopier.java | 23 +--- .../main/codegen/templates/NullReader.java | 1 - .../codegen/templates/PromotableWriter.java | 14 +-- .../codegen/templates/UnionListWriter.java | 12 +- .../main/codegen/templates/UnionReader.java | 23 ++++ .../main/codegen/templates/UnionVector.java | 18 +++ .../main/codegen/templates/UnionWriter.java | 27 +++- .../apache/arrow/vector/BaseValueVector.java | 13 -- .../org/apache/arrow/vector/NullVector.java | 13 -- .../org/apache/arrow/vector/ValueVector.java | 25 ---- .../complex/AbstractContainerVector.java | 13 -- .../arrow/vector/complex/LargeListVector.java | 33 +---- .../vector/complex/LargeListViewVector.java | 15 --- .../arrow/vector/complex/ListVector.java | 33 +---- .../arrow/vector/complex/ListViewVector.java | 15 +-- .../complex/impl/AbstractBaseReader.java | 10 -- .../impl/ExtensionTypeWriterFactory.java | 38 ------ .../complex/impl/UnionExtensionWriter.java | 8 +- .../complex/impl/UnionLargeListReader.java | 4 - .../complex/impl/UuidWriterFactory.java | 45 ------- .../vector/complex/impl/UuidWriterImpl.java | 6 + .../arrow/vector/extension/OpaqueType.java | 7 ++ .../arrow/vector/extension/UuidType.java | 8 ++ .../arrow/vector/holders/ExtensionHolder.java | 4 + .../vector/holders/NullableUuidHolder.java | 7 ++ .../arrow/vector/holders/UuidHolder.java | 7 ++ .../arrow/vector/TestLargeListVector.java | 79 ++++++++++++ .../apache/arrow/vector/TestListVector.java | 89 ++++++++++++-- .../apache/arrow/vector/TestMapVector.java | 115 ++++++++++++++++-- .../apache/arrow/vector/TestStructVector.java | 10 +- .../apache/arrow/vector/TestUuidVector.java | 17 ++- .../complex/impl/TestComplexCopier.java | 23 ++-- .../complex/impl/TestPromotableWriter.java | 36 ++++-- .../complex/writer/TestComplexWriter.java | 22 +++- .../vector/types/pojo/TestExtensionType.java | 7 ++ 40 files changed, 493 insertions(+), 359 deletions(-) delete mode 100644 vector/src/main/java/org/apache/arrow/vector/complex/impl/ExtensionTypeWriterFactory.java delete mode 100644 vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java diff --git a/vector/src/main/codegen/templates/AbstractFieldReader.java b/vector/src/main/codegen/templates/AbstractFieldReader.java index c7c5b4d78d..556fb576ce 100644 --- a/vector/src/main/codegen/templates/AbstractFieldReader.java +++ b/vector/src/main/codegen/templates/AbstractFieldReader.java @@ -109,10 +109,6 @@ public void copyAsField(String name, ${name}Writer writer) { - public void copyAsValue(StructWriter writer, ExtensionTypeWriterFactory writerFactory) { - fail("CopyAsValue StructWriter"); - } - public void read(ExtensionHolder holder) { fail("Extension"); } @@ -147,4 +143,5 @@ public int size() { private void fail(String name) { throw new IllegalArgumentException(String.format("You tried to read a [%s] type when you are using a field reader of type [%s].", name, this.getClass().getSimpleName())); } + } diff --git a/vector/src/main/codegen/templates/AbstractFieldWriter.java b/vector/src/main/codegen/templates/AbstractFieldWriter.java index ae5b97faef..4b4a17d932 100644 --- a/vector/src/main/codegen/templates/AbstractFieldWriter.java +++ b/vector/src/main/codegen/templates/AbstractFieldWriter.java @@ -107,14 +107,17 @@ public void endEntry() { throw new IllegalStateException(String.format("You tried to end a map entry when you are using a ValueWriter of type %s.", this.getClass().getSimpleName())); } + @Override public void write(ExtensionHolder var1) { - this.fail("ExtensionType"); + this.fail("Cannot write ExtensionHolder"); } + @Override public void writeExtension(Object var1) { - this.fail("ExtensionType"); + this.fail("Cannot write extension object"); } - public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory var1) { - this.fail("ExtensionType"); + @Override + public void writeExtension(Object var1, ArrowType type) { + this.fail("Cannot write extension with type " + type); } <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> diff --git a/vector/src/main/codegen/templates/ArrowType.java b/vector/src/main/codegen/templates/ArrowType.java index fd35c1cd2b..b428f09155 100644 --- a/vector/src/main/codegen/templates/ArrowType.java +++ b/vector/src/main/codegen/templates/ArrowType.java @@ -27,8 +27,10 @@ import org.apache.arrow.flatbuf.Type; import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.writer.FieldWriter; import org.apache.arrow.vector.types.*; import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.ValueVector; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -331,6 +333,10 @@ public boolean equals(Object obj) { public T accept(ArrowTypeVisitor visitor) { return visitor.visit(this); } + + public FieldWriter getNewFieldWriter(ValueVector vector) { + throw new UnsupportedOperationException("WriterImpl not yet implemented."); + } } private static final int defaultDecimalBitWidth = 128; diff --git a/vector/src/main/codegen/templates/BaseReader.java b/vector/src/main/codegen/templates/BaseReader.java index 4c6f49ab9b..c52345af21 100644 --- a/vector/src/main/codegen/templates/BaseReader.java +++ b/vector/src/main/codegen/templates/BaseReader.java @@ -49,7 +49,6 @@ public interface RepeatedStructReader extends StructReader{ boolean next(); int size(); void copyAsValue(StructWriter writer); - void copyAsValue(StructWriter writer, ExtensionTypeWriterFactory writerFactory); } public interface ListReader extends BaseReader{ @@ -60,7 +59,6 @@ public interface RepeatedListReader extends ListReader{ boolean next(); int size(); void copyAsValue(ListWriter writer); - void copyAsValue(ListWriter writer, ExtensionTypeWriterFactory writerFactory); } public interface MapReader extends BaseReader{ @@ -71,7 +69,6 @@ public interface RepeatedMapReader extends MapReader{ boolean next(); int size(); void copyAsValue(MapWriter writer); - void copyAsValue(MapWriter writer, ExtensionTypeWriterFactory writerFactory); } public interface ScalarReader extends diff --git a/vector/src/main/codegen/templates/BaseWriter.java b/vector/src/main/codegen/templates/BaseWriter.java index 78da7fddc3..a4c98d7089 100644 --- a/vector/src/main/codegen/templates/BaseWriter.java +++ b/vector/src/main/codegen/templates/BaseWriter.java @@ -125,11 +125,12 @@ public interface ExtensionWriter extends BaseWriter { void writeExtension(Object value); /** - * Adds the given extension type factory. This factory allows configuring writer implementations for specific ExtensionTypeVector. + * Writes the given extension type value. * - * @param factory the extension type factory to add + * @param value the extension type value to write + * @param type of the extension */ - void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory factory); + void writeExtension(Object value, ArrowType type); } public interface ScalarWriter extends diff --git a/vector/src/main/codegen/templates/ComplexCopier.java b/vector/src/main/codegen/templates/ComplexCopier.java index 4df5478f48..6655f6c2a7 100644 --- a/vector/src/main/codegen/templates/ComplexCopier.java +++ b/vector/src/main/codegen/templates/ComplexCopier.java @@ -41,15 +41,8 @@ public class ComplexCopier { * @param input field to read from * @param output field to write to */ - public static void copy(FieldReader input, FieldWriter output) { - writeValue(input, output, null); - } - - public static void copy(FieldReader input, FieldWriter output, ExtensionTypeWriterFactory extensionTypeWriterFactory) { - writeValue(input, output, extensionTypeWriterFactory); - } + public static void copy(FieldReader reader, FieldWriter writer) { - private static void writeValue(FieldReader reader, FieldWriter writer, ExtensionTypeWriterFactory extensionTypeWriterFactory) { final MinorType mt = reader.getMinorType(); switch (mt) { @@ -65,7 +58,7 @@ private static void writeValue(FieldReader reader, FieldWriter writer, Extension FieldReader childReader = reader.reader(); FieldWriter childWriter = getListWriterForReader(childReader, writer); if (childReader.isSet()) { - writeValue(childReader, childWriter, extensionTypeWriterFactory); + copy(childReader, childWriter); } else { childWriter.writeNull(); } @@ -83,8 +76,8 @@ private static void writeValue(FieldReader reader, FieldWriter writer, Extension FieldReader structReader = reader.reader(); if (structReader.isSet()) { writer.startEntry(); - writeValue(mapReader.key(), getMapWriterForReader(mapReader.key(), writer.key()), extensionTypeWriterFactory); - writeValue(mapReader.value(), getMapWriterForReader(mapReader.value(), writer.value()), extensionTypeWriterFactory); + copy(mapReader.key(), getMapWriterForReader(mapReader.key(), writer.key())); + copy(mapReader.value(), getMapWriterForReader(mapReader.value(), writer.value())); writer.endEntry(); } else { writer.writeNull(); @@ -103,7 +96,7 @@ private static void writeValue(FieldReader reader, FieldWriter writer, Extension if (childReader.getMinorType() != Types.MinorType.NULL) { FieldWriter childWriter = getStructWriterForReader(childReader, writer, name); if (childReader.isSet()) { - writeValue(childReader, childWriter, extensionTypeWriterFactory); + copy(childReader, childWriter); } else { childWriter.writeNull(); } @@ -115,14 +108,10 @@ private static void writeValue(FieldReader reader, FieldWriter writer, Extension } break; case EXTENSIONTYPE: - if (extensionTypeWriterFactory == null) { - throw new IllegalArgumentException("Must provide ExtensionTypeWriterFactory"); - } if (reader.isSet()) { Object value = reader.readObject(); if (value != null) { - writer.addExtensionTypeWriterFactory(extensionTypeWriterFactory); - writer.writeExtension(value); + writer.writeExtension(value, reader.getField().getType()); } } else { writer.writeNull(); diff --git a/vector/src/main/codegen/templates/NullReader.java b/vector/src/main/codegen/templates/NullReader.java index 0529633478..88e6ea98ea 100644 --- a/vector/src/main/codegen/templates/NullReader.java +++ b/vector/src/main/codegen/templates/NullReader.java @@ -86,7 +86,6 @@ public void read(int arrayIndex, Nullable${name}Holder holder){ } - public void copyAsValue(StructWriter writer, ExtensionTypeWriterFactory writerFactory){} public void read(ExtensionHolder holder) { holder.isSet = 0; } diff --git a/vector/src/main/codegen/templates/PromotableWriter.java b/vector/src/main/codegen/templates/PromotableWriter.java index d22eb00b2c..11d34f72c9 100644 --- a/vector/src/main/codegen/templates/PromotableWriter.java +++ b/vector/src/main/codegen/templates/PromotableWriter.java @@ -286,7 +286,7 @@ protected void setWriter(ValueVector v) { writer = new UnionWriter((UnionVector) vector, nullableStructWriterFactory); break; case EXTENSIONTYPE: - writer = new UnionExtensionWriter((ExtensionTypeVector) vector); + writer = ((ExtensionType) vector.getField().getType()).getNewFieldWriter(vector); break; default: writer = type.getNewFieldWriter(vector); @@ -541,17 +541,13 @@ public void writeLargeVarChar(String value) { } @Override - public void writeExtension(Object value) { - getWriter(MinorType.EXTENSIONTYPE).writeExtension(value); + public void writeExtension(Object value, ArrowType arrowType) { + getWriter(MinorType.EXTENSIONTYPE, arrowType).writeExtension(value, arrowType); } @Override - public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory factory) { - getWriter(MinorType.EXTENSIONTYPE).addExtensionTypeWriterFactory(factory); - } - - public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory factory, ArrowType arrowType) { - getWriter(MinorType.EXTENSIONTYPE, arrowType).addExtensionTypeWriterFactory(factory); + public void write(ExtensionHolder holder) { + getWriter(MinorType.EXTENSIONTYPE, holder.type()).write(holder); } @Override diff --git a/vector/src/main/codegen/templates/UnionListWriter.java b/vector/src/main/codegen/templates/UnionListWriter.java index 3c41ac72b6..4b54739230 100644 --- a/vector/src/main/codegen/templates/UnionListWriter.java +++ b/vector/src/main/codegen/templates/UnionListWriter.java @@ -204,13 +204,13 @@ public MapWriter map(String name, boolean keysSorted) { @Override public ExtensionWriter extension(ArrowType arrowType) { - this.extensionType = arrowType; + extensionType = arrowType; return this; } + @Override public ExtensionWriter extension(String name, ArrowType arrowType) { - ExtensionWriter extensionWriter = writer.extension(name, arrowType); - return extensionWriter; + return writer.extension(name, arrowType); } <#if listName == "LargeList"> @@ -337,13 +337,13 @@ public void writeNull() { @Override public void writeExtension(Object value) { - writer.writeExtension(value); + writer.writeExtension(value, extensionType); writer.setPosition(writer.idx() + 1); } @Override - public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory var1) { - writer.addExtensionTypeWriterFactory(var1, extensionType); + public void writeExtension(Object value, ArrowType type) { + writeExtension(value); } public void write(ExtensionHolder var1) { diff --git a/vector/src/main/codegen/templates/UnionReader.java b/vector/src/main/codegen/templates/UnionReader.java index 96ad3e1b9b..0edae7ade0 100644 --- a/vector/src/main/codegen/templates/UnionReader.java +++ b/vector/src/main/codegen/templates/UnionReader.java @@ -79,6 +79,10 @@ public void read(int index, UnionHolder holder) { } private FieldReader getReaderForIndex(int index) { + return getReaderForIndex(index, null); + } + + private FieldReader getReaderForIndex(int index, ArrowType type) { int typeValue = data.getTypeValue(index); FieldReader reader = (FieldReader) readers[typeValue]; if (reader != null) { @@ -105,11 +109,26 @@ private FieldReader getReaderForIndex(int index) { + case EXTENSIONTYPE: + if(type == null) { + throw new RuntimeException("Cannot get Extension reader without an ArrowType"); + } + return (FieldReader) getExtension(type); default: throw new UnsupportedOperationException("Unsupported type: " + MinorType.values()[typeValue]); } } + private ExtensionReader extensionReader; + + private ExtensionReader getExtension(ArrowType type) { + if (extensionReader == null) { + extensionReader = data.getExtension(type).getReader(); + extensionReader.setPosition(idx()); + } + return extensionReader; + } + private SingleStructReaderImpl structReader; private StructReader getStruct() { @@ -240,4 +259,8 @@ public FieldReader reader() { public boolean next() { return getReaderForIndex(idx()).next(); } + + public void read(ExtensionHolder holder){ + getReaderForIndex(idx(), holder.type()).read(holder); + } } diff --git a/vector/src/main/codegen/templates/UnionVector.java b/vector/src/main/codegen/templates/UnionVector.java index 67efdf60f7..c706591966 100644 --- a/vector/src/main/codegen/templates/UnionVector.java +++ b/vector/src/main/codegen/templates/UnionVector.java @@ -379,6 +379,22 @@ public MapVector getMap(String name, ArrowType arrowType) { return mapVector; } + private ExtensionTypeVector extensionVector; + + public ExtensionTypeVector getExtension(ArrowType arrowType) { + if (extensionVector == null) { + int vectorCount = internalStruct.size(); + extensionVector = addOrGet(null, MinorType.EXTENSIONTYPE, arrowType, ExtensionTypeVector.class); + if (internalStruct.size() > vectorCount) { + extensionVector.allocateNew(); + if (callBack != null) { + callBack.doWork(); + } + } + } + return extensionVector; + } + public int getTypeValue(int index) { return typeBuffer.getByte(index * TYPE_WIDTH); } @@ -725,6 +741,8 @@ public ValueVector getVectorByType(int typeId, ArrowType arrowType) { return getListView(); case MAP: return getMap(name, arrowType); + case EXTENSIONTYPE: + return getExtension(arrowType); default: throw new UnsupportedOperationException("Cannot support type: " + MinorType.values()[typeId]); } diff --git a/vector/src/main/codegen/templates/UnionWriter.java b/vector/src/main/codegen/templates/UnionWriter.java index 272edab17c..0db699fd8c 100644 --- a/vector/src/main/codegen/templates/UnionWriter.java +++ b/vector/src/main/codegen/templates/UnionWriter.java @@ -28,6 +28,8 @@ package org.apache.arrow.vector.complex.impl; <#include "/@includes/vv_imports.ftl" /> +import java.util.HashMap; + import org.apache.arrow.vector.complex.writer.BaseWriter; import org.apache.arrow.vector.types.Types.MinorType; @@ -213,8 +215,31 @@ public MapWriter asMap(ArrowType arrowType) { return getMapWriter(arrowType); } + private java.util.Map extensionWriters = new HashMap<>(); + private ExtensionWriter getExtensionWriter(ArrowType arrowType) { - throw new UnsupportedOperationException("ExtensionTypes are not supported yet."); + ExtensionWriter w = extensionWriters.get(arrowType); + if (w == null) { + w = ((ExtensionType) arrowType).getNewFieldWriter(data.getExtension(arrowType)); + w.setPosition(idx()); + extensionWriters.put(arrowType, w); + } + return w; + } + + public void writeExtension(Object value, ArrowType type) { + data.setType(idx(), MinorType.EXTENSIONTYPE); + ExtensionWriter w = getExtensionWriter(type); + w.setPosition(idx()); + w.writeExtension(value); + } + + @Override + public void write(ExtensionHolder holder) { + data.setType(idx(), MinorType.EXTENSIONTYPE); + ExtensionWriter w = getExtensionWriter(holder.type()); + w.setPosition(idx()); + w.write(holder); } BaseWriter getWriter(MinorType minorType) { diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java index cc57cde29e..37dfa20616 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java @@ -22,7 +22,6 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.ReferenceManager; import org.apache.arrow.util.Preconditions; -import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.util.DataSizeRoundingUtil; import org.apache.arrow.vector.util.TransferPair; @@ -261,18 +260,6 @@ public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { throw new UnsupportedOperationException(); } - @Override - public void copyFrom( - int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { - throw new UnsupportedOperationException(); - } - - @Override - public void copyFromSafe( - int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { - throw new UnsupportedOperationException(); - } - /** * Transfer the validity buffer from `validityBuffer` to the target vector's `validityBuffer`. * Start at `startIndex` and copy `length` number of elements. If the starting index is 8 byte diff --git a/vector/src/main/java/org/apache/arrow/vector/NullVector.java b/vector/src/main/java/org/apache/arrow/vector/NullVector.java index 0d6dab2837..6bfe540d23 100644 --- a/vector/src/main/java/org/apache/arrow/vector/NullVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/NullVector.java @@ -27,7 +27,6 @@ import org.apache.arrow.memory.util.hash.ArrowBufHasher; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.compare.VectorVisitor; -import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.impl.NullReader; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.ipc.message.ArrowFieldNode; @@ -330,18 +329,6 @@ public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { throw new UnsupportedOperationException(); } - @Override - public void copyFrom( - int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { - throw new UnsupportedOperationException(); - } - - @Override - public void copyFromSafe( - int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { - throw new UnsupportedOperationException(); - } - @Override public String getName() { return this.getField().getName(); diff --git a/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/vector/src/main/java/org/apache/arrow/vector/ValueVector.java index e0628c2ee1..3a5058256c 100644 --- a/vector/src/main/java/org/apache/arrow/vector/ValueVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/ValueVector.java @@ -22,7 +22,6 @@ import org.apache.arrow.memory.OutOfMemoryException; import org.apache.arrow.memory.util.hash.ArrowBufHasher; import org.apache.arrow.vector.compare.VectorVisitor; -import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.Field; @@ -310,30 +309,6 @@ public interface ValueVector extends Closeable, Iterable { */ void copyFromSafe(int fromIndex, int thisIndex, ValueVector from); - /** - * Copy a cell value from a particular index in source vector to a particular position in this - * vector. - * - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - * @param writerFactory the extension type writer factory to use for copying extension type values - */ - void copyFrom( - int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory); - - /** - * Same as {@link #copyFrom(int, int, ValueVector)} except that it handles the case when the - * capacity of the vector needs to be expanded before copy. - * - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - * @param writerFactory the extension type writer factory to use for copying extension type values - */ - void copyFromSafe( - int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory); - /** * Accept a generic {@link VectorVisitor} and return the result. * diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java index 429f9884bb..a6a71cf1a4 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java @@ -21,7 +21,6 @@ import org.apache.arrow.vector.DensityAwareVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.ValueVector; -import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeList; @@ -152,18 +151,6 @@ public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { throw new UnsupportedOperationException(); } - @Override - public void copyFrom( - int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { - throw new UnsupportedOperationException(); - } - - @Override - public void copyFromSafe( - int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { - throw new UnsupportedOperationException(); - } - @Override public String getName() { return name; diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java index 48c8127e23..997b5a8b78 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java @@ -49,7 +49,6 @@ import org.apache.arrow.vector.ZeroVector; import org.apache.arrow.vector.compare.VectorVisitor; import org.apache.arrow.vector.complex.impl.ComplexCopier; -import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.impl.UnionLargeListReader; import org.apache.arrow.vector.complex.impl.UnionLargeListWriter; import org.apache.arrow.vector.complex.reader.FieldReader; @@ -483,42 +482,12 @@ public void copyFromSafe(int inIndex, int outIndex, ValueVector from) { */ @Override public void copyFrom(int inIndex, int outIndex, ValueVector from) { - copyFrom(inIndex, outIndex, from, null); - } - - /** - * Copy a cell value from a particular index in source vector to a particular position in this - * vector. - * - * @param inIndex position to copy from in source vector - * @param outIndex position to copy to in this vector - * @param from source vector - * @param writerFactory the extension type writer factory to use for copying extension type values - */ - @Override - public void copyFrom( - int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { Preconditions.checkArgument(this.getMinorType() == from.getMinorType()); FieldReader in = from.getReader(); in.setPosition(inIndex); UnionLargeListWriter out = getWriter(); out.setPosition(outIndex); - ComplexCopier.copy(in, out, writerFactory); - } - - /** - * Same as {@link #copyFrom(int, int, ValueVector)} except that it handles the case when the - * capacity of the vector needs to be expanded before copy. - * - * @param inIndex position to copy from in source vector - * @param outIndex position to copy to in this vector - * @param from source vector - * @param writerFactory the extension type writer factory to use for copying extension type values - */ - @Override - public void copyFromSafe( - int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { - copyFrom(inIndex, outIndex, from, writerFactory); + ComplexCopier.copy(in, out); } /** diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java index 992a664449..2da7eb057e 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java @@ -41,7 +41,6 @@ import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.ZeroVector; import org.apache.arrow.vector.compare.VectorVisitor; -import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.impl.UnionLargeListViewReader; import org.apache.arrow.vector.complex.impl.UnionLargeListViewWriter; import org.apache.arrow.vector.complex.impl.UnionListReader; @@ -347,20 +346,6 @@ public void copyFrom(int inIndex, int outIndex, ValueVector from) { "LargeListViewVector does not support copyFrom operation yet."); } - @Override - public void copyFromSafe( - int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { - throw new UnsupportedOperationException( - "LargeListViewVector does not support copyFromSafe operation yet."); - } - - @Override - public void copyFrom( - int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { - throw new UnsupportedOperationException( - "LargeListViewVector does not support copyFrom operation yet."); - } - @Override public FieldVector getDataVector() { return vector; diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 89549257c4..93a313ef4f 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -42,7 +42,6 @@ import org.apache.arrow.vector.ZeroVector; import org.apache.arrow.vector.compare.VectorVisitor; import org.apache.arrow.vector.complex.impl.ComplexCopier; -import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.impl.UnionListReader; import org.apache.arrow.vector.complex.impl.UnionListWriter; import org.apache.arrow.vector.complex.reader.FieldReader; @@ -401,42 +400,12 @@ public void copyFromSafe(int inIndex, int outIndex, ValueVector from) { */ @Override public void copyFrom(int inIndex, int outIndex, ValueVector from) { - copyFrom(inIndex, outIndex, from, null); - } - - /** - * Same as {@link #copyFrom(int, int, ValueVector)} except that it handles the case when the - * capacity of the vector needs to be expanded before copy. - * - * @param inIndex position to copy from in source vector - * @param outIndex position to copy to in this vector - * @param from source vector - * @param writerFactory the extension type writer factory to use for copying extension type values - */ - @Override - public void copyFromSafe( - int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { - copyFrom(inIndex, outIndex, from, writerFactory); - } - - /** - * Copy a cell value from a particular index in source vector to a particular position in this - * vector. - * - * @param inIndex position to copy from in source vector - * @param outIndex position to copy to in this vector - * @param from source vector - * @param writerFactory the extension type writer factory to use for copying extension type values - */ - @Override - public void copyFrom( - int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { Preconditions.checkArgument(this.getMinorType() == from.getMinorType()); FieldReader in = from.getReader(); in.setPosition(inIndex); FieldWriter out = getWriter(); out.setPosition(outIndex); - ComplexCopier.copy(in, out, writerFactory); + ComplexCopier.copy(in, out); } /** diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java index 2784240429..8711db5e0f 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java @@ -42,7 +42,6 @@ import org.apache.arrow.vector.ZeroVector; import org.apache.arrow.vector.compare.VectorVisitor; import org.apache.arrow.vector.complex.impl.ComplexCopier; -import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory; import org.apache.arrow.vector.complex.impl.UnionListViewReader; import org.apache.arrow.vector.complex.impl.UnionListViewWriter; import org.apache.arrow.vector.complex.reader.FieldReader; @@ -339,12 +338,6 @@ public void copyFromSafe(int inIndex, int outIndex, ValueVector from) { copyFrom(inIndex, outIndex, from); } - @Override - public void copyFromSafe( - int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { - copyFrom(inIndex, outIndex, from, writerFactory); - } - @Override public OUT accept(VectorVisitor visitor, IN value) { return visitor.visit(this, value); @@ -352,18 +345,12 @@ public OUT accept(VectorVisitor visitor, IN value) { @Override public void copyFrom(int inIndex, int outIndex, ValueVector from) { - copyFrom(inIndex, outIndex, from, null); - } - - @Override - public void copyFrom( - int inIndex, int outIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) { Preconditions.checkArgument(this.getMinorType() == from.getMinorType()); FieldReader in = from.getReader(); in.setPosition(inIndex); FieldWriter out = getWriter(); out.setPosition(outIndex); - ComplexCopier.copy(in, out, writerFactory); + ComplexCopier.copy(in, out); } @Override diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java index bf074ecb90..b2e95663f7 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java @@ -115,14 +115,4 @@ public void copyAsValue(ListWriter writer) { public void copyAsValue(MapWriter writer) { ComplexCopier.copy(this, (FieldWriter) writer); } - - @Override - public void copyAsValue(ListWriter writer, ExtensionTypeWriterFactory writerFactory) { - ComplexCopier.copy(this, (FieldWriter) writer, writerFactory); - } - - @Override - public void copyAsValue(MapWriter writer, ExtensionTypeWriterFactory writerFactory) { - ComplexCopier.copy(this, (FieldWriter) writer, writerFactory); - } } diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/ExtensionTypeWriterFactory.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/ExtensionTypeWriterFactory.java deleted file mode 100644 index a01d591555..0000000000 --- a/vector/src/main/java/org/apache/arrow/vector/complex/impl/ExtensionTypeWriterFactory.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.arrow.vector.complex.impl; - -import org.apache.arrow.vector.ExtensionTypeVector; -import org.apache.arrow.vector.complex.writer.FieldWriter; - -/** - * A factory interface for creating instances of {@link AbstractExtensionTypeWriter}. This factory - * allows configuring writer implementations for specific {@link ExtensionTypeVector}. - * - * @param the type of writer implementation for a specific {@link ExtensionTypeVector}. - */ -public interface ExtensionTypeWriterFactory { - - /** - * Returns an instance of the writer implementation for the given {@link ExtensionTypeVector}. - * - * @param vector the {@link ExtensionTypeVector} for which the writer implementation is to be - * returned. - * @return an instance of the writer implementation for the given {@link ExtensionTypeVector}. - */ - T getWriterImpl(ExtensionTypeVector vector); -} diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionExtensionWriter.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionExtensionWriter.java index 4219069cba..93796aa77e 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionExtensionWriter.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionExtensionWriter.java @@ -60,11 +60,6 @@ public void writeExtension(Object var1) { } @Override - public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory factory) { - this.writer = factory.getWriterImpl(vector); - this.writer.setPosition(idx()); - } - public void write(ExtensionHolder holder) { this.writer.write(holder); } @@ -79,6 +74,7 @@ public void setPosition(int index) { @Override public void writeNull() { - this.writer.writeNull(); + this.vector.setNull(getPosition()); + this.vector.setValueCount(getPosition() + 1); } } diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionLargeListReader.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionLargeListReader.java index a9104cb0d2..be236c3166 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionLargeListReader.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionLargeListReader.java @@ -105,8 +105,4 @@ public boolean next() { public void copyAsValue(UnionLargeListWriter writer) { ComplexCopier.copy(this, (FieldWriter) writer); } - - public void copyAsValue(UnionLargeListWriter writer, ExtensionTypeWriterFactory writerFactory) { - ComplexCopier.copy(this, (FieldWriter) writer, writerFactory); - } } diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java deleted file mode 100644 index 35988129cb..0000000000 --- a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterFactory.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.arrow.vector.complex.impl; - -import org.apache.arrow.vector.ExtensionTypeVector; -import org.apache.arrow.vector.UuidVector; - -/** - * Factory for creating {@link UuidWriterImpl} instances. - * - *

This factory is used to create writers for UUID extension type vectors. - * - * @see UuidWriterImpl - * @see org.apache.arrow.vector.extension.UuidType - */ -public class UuidWriterFactory implements ExtensionTypeWriterFactory { - - /** - * Creates a writer implementation for the given extension type vector. - * - * @param extensionTypeVector the vector to create a writer for - * @return a {@link UuidWriterImpl} if the vector is a {@link UuidVector}, null otherwise - */ - @Override - public AbstractFieldWriter getWriterImpl(ExtensionTypeVector extensionTypeVector) { - if (extensionTypeVector instanceof UuidVector) { - return new UuidWriterImpl((UuidVector) extensionTypeVector); - } - return null; - } -} diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java index 8a78add11c..ee3c79d5e3 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java @@ -21,6 +21,7 @@ import org.apache.arrow.vector.holders.ExtensionHolder; import org.apache.arrow.vector.holders.NullableUuidHolder; import org.apache.arrow.vector.holders.UuidHolder; +import org.apache.arrow.vector.types.pojo.ArrowType; /** * Writer implementation for {@link UuidVector}. @@ -56,6 +57,11 @@ public void writeExtension(Object value) { vector.setValueCount(getPosition() + 1); } + @Override + public void writeExtension(Object value, ArrowType type) { + writeExtension(value); + } + @Override public void write(ExtensionHolder holder) { if (holder instanceof UuidHolder) { diff --git a/vector/src/main/java/org/apache/arrow/vector/extension/OpaqueType.java b/vector/src/main/java/org/apache/arrow/vector/extension/OpaqueType.java index ca56214fda..780a4ee659 100644 --- a/vector/src/main/java/org/apache/arrow/vector/extension/OpaqueType.java +++ b/vector/src/main/java/org/apache/arrow/vector/extension/OpaqueType.java @@ -54,10 +54,12 @@ import org.apache.arrow.vector.TimeStampNanoVector; import org.apache.arrow.vector.TimeStampSecTZVector; import org.apache.arrow.vector.TimeStampSecVector; +import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.ViewVarBinaryVector; import org.apache.arrow.vector.ViewVarCharVector; +import org.apache.arrow.vector.complex.writer.FieldWriter; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.ExtensionTypeRegistry; @@ -177,6 +179,11 @@ public int hashCode() { return Objects.hash(super.hashCode(), storageType, typeName, vendorName); } + @Override + public FieldWriter getNewFieldWriter(ValueVector vector) { + throw new UnsupportedOperationException("WriterImpl not yet implemented."); + } + @Override public String toString() { return "OpaqueType(" diff --git a/vector/src/main/java/org/apache/arrow/vector/extension/UuidType.java b/vector/src/main/java/org/apache/arrow/vector/extension/UuidType.java index cd29f930e1..c249c6eda9 100644 --- a/vector/src/main/java/org/apache/arrow/vector/extension/UuidType.java +++ b/vector/src/main/java/org/apache/arrow/vector/extension/UuidType.java @@ -20,6 +20,9 @@ import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.FixedSizeBinaryVector; import org.apache.arrow.vector.UuidVector; +import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.complex.impl.UuidWriterImpl; +import org.apache.arrow.vector.complex.writer.FieldWriter; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.ArrowType.ExtensionType; import org.apache.arrow.vector.types.pojo.ExtensionTypeRegistry; @@ -108,4 +111,9 @@ public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocato return new UuidVector( name, fieldType, allocator, new FixedSizeBinaryVector(name, allocator, UUID_BYTE_WIDTH)); } + + @Override + public FieldWriter getNewFieldWriter(ValueVector vector) { + return new UuidWriterImpl((UuidVector) vector); + } } diff --git a/vector/src/main/java/org/apache/arrow/vector/holders/ExtensionHolder.java b/vector/src/main/java/org/apache/arrow/vector/holders/ExtensionHolder.java index fc7ed85878..4d3f767aef 100644 --- a/vector/src/main/java/org/apache/arrow/vector/holders/ExtensionHolder.java +++ b/vector/src/main/java/org/apache/arrow/vector/holders/ExtensionHolder.java @@ -16,7 +16,11 @@ */ package org.apache.arrow.vector.holders; +import org.apache.arrow.vector.types.pojo.ArrowType; + /** Base {@link ValueHolder} class for a {@link org.apache.arrow.vector.ExtensionTypeVector}. */ public abstract class ExtensionHolder implements ValueHolder { public int isSet; + + public abstract ArrowType type(); } diff --git a/vector/src/main/java/org/apache/arrow/vector/holders/NullableUuidHolder.java b/vector/src/main/java/org/apache/arrow/vector/holders/NullableUuidHolder.java index e5398d82cf..7fa50ca761 100644 --- a/vector/src/main/java/org/apache/arrow/vector/holders/NullableUuidHolder.java +++ b/vector/src/main/java/org/apache/arrow/vector/holders/NullableUuidHolder.java @@ -17,6 +17,8 @@ package org.apache.arrow.vector.holders; import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.vector.extension.UuidType; +import org.apache.arrow.vector.types.pojo.ArrowType; /** * Value holder for nullable UUID values. @@ -32,4 +34,9 @@ public class NullableUuidHolder extends ExtensionHolder { /** Buffer containing 16-byte UUID data. */ public ArrowBuf buffer; + + @Override + public ArrowType type() { + return UuidType.INSTANCE; + } } diff --git a/vector/src/main/java/org/apache/arrow/vector/holders/UuidHolder.java b/vector/src/main/java/org/apache/arrow/vector/holders/UuidHolder.java index 484e05c24b..8a0a66e435 100644 --- a/vector/src/main/java/org/apache/arrow/vector/holders/UuidHolder.java +++ b/vector/src/main/java/org/apache/arrow/vector/holders/UuidHolder.java @@ -17,6 +17,8 @@ package org.apache.arrow.vector.holders; import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.vector.extension.UuidType; +import org.apache.arrow.vector.types.pojo.ArrowType; /** * Value holder for non-nullable UUID values. @@ -35,4 +37,9 @@ public class UuidHolder extends ExtensionHolder { public UuidHolder() { this.isSet = 1; } + + @Override + public ArrowType type() { + return UuidType.INSTANCE; + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java index d5cbf925b2..759c84651d 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java @@ -26,18 +26,24 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.UUID; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.complex.BaseRepeatedValueVector; import org.apache.arrow.vector.complex.LargeListVector; import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.impl.UnionLargeListReader; import org.apache.arrow.vector.complex.impl.UnionLargeListWriter; import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.complex.writer.BaseWriter.ExtensionWriter; +import org.apache.arrow.vector.extension.UuidType; +import org.apache.arrow.vector.holders.UuidHolder; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.TransferPair; +import org.apache.arrow.vector.util.UuidUtility; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -1021,6 +1027,79 @@ public void testGetTransferPairWithField() throws Exception { } } + @Test + public void testCopyValueSafeForExtensionType() throws Exception { + try (LargeListVector inVector = LargeListVector.empty("input", allocator); + LargeListVector outVector = LargeListVector.empty("output", allocator)) { + UnionLargeListWriter writer = inVector.getWriter(); + writer.allocate(); + + // Create first list with UUIDs + writer.setPosition(0); + UUID u1 = UUID.randomUUID(); + UUID u2 = UUID.randomUUID(); + writer.startList(); + ExtensionWriter extensionWriter = writer.extension(UuidType.INSTANCE); + extensionWriter.writeExtension(u1); + extensionWriter.writeExtension(u2); + writer.endList(); + + // Create second list with UUIDs + writer.setPosition(1); + UUID u3 = UUID.randomUUID(); + UUID u4 = UUID.randomUUID(); + writer.startList(); + extensionWriter = writer.extension(UuidType.INSTANCE); + extensionWriter.writeExtension(u3); + extensionWriter.writeExtension(u4); + extensionWriter.writeNull(); + + writer.endList(); + writer.setValueCount(2); + + // Use copyFromSafe with ExtensionTypeWriterFactory + // This internally calls TransferImpl.copyValueSafe with ExtensionTypeWriterFactory + outVector.allocateNew(); + TransferPair tp = inVector.makeTransferPair(outVector); + tp.copyValueSafe(0, 0); + tp.copyValueSafe(1, 1); + outVector.setValueCount(2); + + // Verify first list + UnionLargeListReader reader = outVector.getReader(); + reader.setPosition(0); + assertTrue(reader.isSet(), "first list shouldn't be null"); + reader.next(); + FieldReader uuidReader = reader.reader(); + UuidHolder holder = new UuidHolder(); + uuidReader.read(holder); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + assertEquals(u1, actualUuid); + reader.next(); + uuidReader = reader.reader(); + uuidReader.read(holder); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + assertEquals(u2, actualUuid); + + // Verify second list + reader.setPosition(1); + assertTrue(reader.isSet(), "second list shouldn't be null"); + reader.next(); + uuidReader = reader.reader(); + uuidReader.read(holder); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + assertEquals(u3, actualUuid); + reader.next(); + uuidReader = reader.reader(); + uuidReader.read(holder); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + assertEquals(u4, actualUuid); + reader.next(); + uuidReader = reader.reader(); + assertFalse(uuidReader.isSet(), "third element should be null"); + } + } + private void writeIntValues(UnionLargeListWriter writer, int[] values) { writer.startList(); for (int v : values) { diff --git a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index df3a609f53..e96ac3027c 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -35,7 +35,6 @@ import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.impl.UnionListReader; import org.apache.arrow.vector.complex.impl.UnionListWriter; -import org.apache.arrow.vector.complex.impl.UuidWriterFactory; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.complex.writer.BaseWriter.ExtensionWriter; import org.apache.arrow.vector.extension.UuidType; @@ -1217,7 +1216,6 @@ public void testListVectorWithExtensionType() throws Exception { UUID u2 = UUID.randomUUID(); writer.startList(); ExtensionWriter extensionWriter = writer.extension(UuidType.INSTANCE); - extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter.writeExtension(u1); extensionWriter.writeExtension(u2); writer.endList(); @@ -1245,7 +1243,6 @@ public void testListVectorReaderForExtensionType() throws Exception { UUID u2 = UUID.randomUUID(); writer.startList(); ExtensionWriter extensionWriter = writer.extension(UuidType.INSTANCE); - extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter.writeExtension(u1); extensionWriter.writeExtension(u2); writer.endList(); @@ -1279,23 +1276,78 @@ public void testCopyFromForExtensionType() throws Exception { UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); writer.startList(); + + writer.extension(UuidType.INSTANCE).writeExtension(u1); + writer.writeExtension(u2); + writer.writeNull(); + writer.endList(); + + writer.setValueCount(3); + + // copy values from input to output + outVector.allocateNew(); + outVector.copyFrom(0, 0, inVector); + outVector.setValueCount(3); + + UnionListReader reader = outVector.getReader(); + assertTrue(reader.isSet(), "shouldn't be null"); + reader.setPosition(0); + reader.next(); + FieldReader uuidReader = reader.reader(); + UuidHolder holder = new UuidHolder(); + uuidReader.read(holder); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + assertEquals(u1, actualUuid); + reader.next(); + uuidReader = reader.reader(); + uuidReader.read(holder); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + assertEquals(u2, actualUuid); + } + } + + @Test + public void testCopyValueSafeForExtensionType() throws Exception { + try (ListVector inVector = ListVector.empty("input", allocator); + ListVector outVector = ListVector.empty("output", allocator)) { + UnionListWriter writer = inVector.getWriter(); + writer.allocate(); + + // Create first list with UUIDs + writer.setPosition(0); + UUID u1 = UUID.randomUUID(); + UUID u2 = UUID.randomUUID(); + writer.startList(); ExtensionWriter extensionWriter = writer.extension(UuidType.INSTANCE); - extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter.writeExtension(u1); extensionWriter.writeExtension(u2); - extensionWriter.writeNull(); writer.endList(); - writer.setValueCount(1); + // Create second list with UUIDs + writer.setPosition(1); + UUID u3 = UUID.randomUUID(); + UUID u4 = UUID.randomUUID(); + writer.startList(); + extensionWriter = writer.extension(UuidType.INSTANCE); + extensionWriter.writeExtension(u3); + extensionWriter.writeExtension(u4); + extensionWriter.writeNull(); - // copy values from input to output + writer.endList(); + writer.setValueCount(2); + + // Use TransferPair with ExtensionTypeWriterFactory + // This tests the new makeTransferPair API with writerFactory parameter outVector.allocateNew(); - outVector.copyFrom(0, 0, inVector, new UuidWriterFactory()); - outVector.setValueCount(1); + TransferPair transferPair = inVector.makeTransferPair(outVector); + transferPair.copyValueSafe(0, 0); + transferPair.copyValueSafe(1, 1); + outVector.setValueCount(2); + // Verify first list UnionListReader reader = outVector.getReader(); - assertTrue(reader.isSet(), "shouldn't be null"); reader.setPosition(0); + assertTrue(reader.isSet(), "first list shouldn't be null"); reader.next(); FieldReader uuidReader = reader.reader(); UuidHolder holder = new UuidHolder(); @@ -1307,6 +1359,23 @@ public void testCopyFromForExtensionType() throws Exception { uuidReader.read(holder); actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); assertEquals(u2, actualUuid); + + // Verify second list + reader.setPosition(1); + assertTrue(reader.isSet(), "second list shouldn't be null"); + reader.next(); + uuidReader = reader.reader(); + uuidReader.read(holder); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + assertEquals(u3, actualUuid); + reader.next(); + uuidReader = reader.reader(); + uuidReader.read(holder); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + assertEquals(u4, actualUuid); + reader.next(); + uuidReader = reader.reader(); + assertFalse(uuidReader.isSet(), "third element should be null"); } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java index d9d2ca50dc..bfac1237a4 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java @@ -35,7 +35,6 @@ import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.complex.impl.UnionMapReader; import org.apache.arrow.vector.complex.impl.UnionMapWriter; -import org.apache.arrow.vector.complex.impl.UuidWriterFactory; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.complex.writer.BaseWriter.ExtensionWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter; @@ -1285,14 +1284,12 @@ public void testMapVectorWithExtensionType() throws Exception { writer.startEntry(); writer.key().bigInt().writeBigInt(0); ExtensionWriter extensionWriter = writer.value().extension(UuidType.INSTANCE); - extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); - extensionWriter.writeExtension(u1); + extensionWriter.writeExtension(u1, UuidType.INSTANCE); writer.endEntry(); writer.startEntry(); writer.key().bigInt().writeBigInt(1); extensionWriter = writer.value().extension(UuidType.INSTANCE); - extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); - extensionWriter.writeExtension(u2); + extensionWriter.writeExtension(u2, UuidType.INSTANCE); writer.endEntry(); writer.endMap(); @@ -1327,20 +1324,17 @@ public void testCopyFromForExtensionType() throws Exception { writer.startEntry(); writer.key().bigInt().writeBigInt(0); ExtensionWriter extensionWriter = writer.value().extension(UuidType.INSTANCE); - extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); - extensionWriter.writeExtension(u1); + extensionWriter.writeExtension(u1, UuidType.INSTANCE); writer.endEntry(); writer.startEntry(); writer.key().bigInt().writeBigInt(1); - extensionWriter = writer.value().extension(UuidType.INSTANCE); - extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); - extensionWriter.writeExtension(u2); + extensionWriter.writeExtension(u2, UuidType.INSTANCE); writer.endEntry(); writer.endMap(); writer.setValueCount(1); outVector.allocateNew(); - outVector.copyFrom(0, 0, inVector, new UuidWriterFactory()); + outVector.copyFrom(0, 0, inVector); outVector.setValueCount(1); UnionMapReader mapReader = outVector.getReader(); @@ -1576,4 +1570,103 @@ public void testFixedSizeBinaryFirstInitialization() { assertArrayEquals(new byte[] {32, 21}, (byte[]) resultStruct.get(MapVector.VALUE_NAME)); } } + + @Test + public void testMapWithUuidKeyAndListUuidValue() throws Exception { + try (final MapVector mapVector = MapVector.empty("map", allocator, false)) { + mapVector.allocateNew(); + UnionMapWriter writer = mapVector.getWriter(); + + // Create test UUIDs + UUID key1 = UUID.randomUUID(); + UUID key2 = UUID.randomUUID(); + UUID value1a = UUID.randomUUID(); + UUID value1b = UUID.randomUUID(); + UUID value2a = UUID.randomUUID(); + UUID value2b = UUID.randomUUID(); + UUID value2c = UUID.randomUUID(); + + // Write first map entry: {key1 -> [value1a, value1b]} + writer.setPosition(0); + writer.startMap(); + + writer.startEntry(); + ExtensionWriter keyWriter = writer.key().extension(UuidType.INSTANCE); + keyWriter.writeExtension(key1, UuidType.INSTANCE); + ListWriter valueWriter = writer.value().list(); + valueWriter.startList(); + ExtensionWriter listItemWriter = valueWriter.extension(UuidType.INSTANCE); + listItemWriter.writeExtension(value1a, UuidType.INSTANCE); + listItemWriter = valueWriter.extension(UuidType.INSTANCE); + listItemWriter.writeExtension(value1b, UuidType.INSTANCE); + valueWriter.endList(); + writer.endEntry(); + + writer.startEntry(); + keyWriter = writer.key().extension(UuidType.INSTANCE); + keyWriter.writeExtension(key2, UuidType.INSTANCE); + valueWriter = writer.value().list(); + valueWriter.startList(); + listItemWriter = valueWriter.extension(UuidType.INSTANCE); + listItemWriter.writeExtension(value2a, UuidType.INSTANCE); + listItemWriter = valueWriter.extension(UuidType.INSTANCE); + listItemWriter.writeExtension(value2b, UuidType.INSTANCE); + listItemWriter = valueWriter.extension(UuidType.INSTANCE); + listItemWriter.writeExtension(value2c, UuidType.INSTANCE); + valueWriter.endList(); + writer.endEntry(); + + writer.endMap(); + writer.setValueCount(1); + + // Read and verify the data + UnionMapReader mapReader = mapVector.getReader(); + mapReader.setPosition(0); + + // Read first entry + mapReader.next(); + FieldReader keyReader = mapReader.key(); + UuidHolder keyHolder = new UuidHolder(); + keyReader.read(keyHolder); + UUID actualKey = UuidUtility.uuidFromArrowBuf(keyHolder.buffer, 0); + assertEquals(key1, actualKey); + + FieldReader valueReader = mapReader.value(); + assertTrue(valueReader.isSet()); + List listValue = (List) valueReader.readObject(); + assertEquals(2, listValue.size()); + + // Verify first list item - readObject() returns UUID objects for extension types + UUID actualValue1a = (UUID) listValue.get(0); + assertEquals(value1a, actualValue1a); + + // Verify second list item + UUID actualValue1b = (UUID) listValue.get(1); + assertEquals(value1b, actualValue1b); + + // Read second entry + mapReader.next(); + keyReader = mapReader.key(); + keyReader.read(keyHolder); + actualKey = UuidUtility.uuidFromArrowBuf(keyHolder.buffer, 0); + assertEquals(key2, actualKey); + + valueReader = mapReader.value(); + assertTrue(valueReader.isSet()); + listValue = (List) valueReader.readObject(); + assertEquals(3, listValue.size()); + + // Verify first list item - readObject() returns UUID objects for extension types + UUID actualValue2a = (UUID) listValue.get(0); + assertEquals(value2a, actualValue2a); + + // Verify second list item + UUID actualValue2b = (UUID) listValue.get(1); + assertEquals(value2b, actualValue2b); + + // Verify third list item + UUID actualValue2c = (UUID) listValue.get(2); + assertEquals(value2c, actualValue2c); + } + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java b/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java index 21ebeebc86..8c8a45f588 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java @@ -160,17 +160,23 @@ public void testGetPrimitiveVectors() { UnionVector unionVector = vector.addOrGetUnion("union"); unionVector.addVector(new BigIntVector("bigInt", allocator)); unionVector.addVector(new SmallIntVector("smallInt", allocator)); + unionVector.addVector(new UuidVector("uuid", allocator)); // add varchar vector vector.addOrGet( "varchar", FieldType.nullable(MinorType.VARCHAR.getType()), VarCharVector.class); + // add extension vector + vector.addOrGet("extension", FieldType.nullable(UuidType.INSTANCE), UuidVector.class); + List primitiveVectors = vector.getPrimitiveVectors(); - assertEquals(4, primitiveVectors.size()); + assertEquals(6, primitiveVectors.size()); assertEquals(MinorType.INT, primitiveVectors.get(0).getMinorType()); assertEquals(MinorType.BIGINT, primitiveVectors.get(1).getMinorType()); assertEquals(MinorType.SMALLINT, primitiveVectors.get(2).getMinorType()); - assertEquals(MinorType.VARCHAR, primitiveVectors.get(3).getMinorType()); + assertEquals(MinorType.EXTENSIONTYPE, primitiveVectors.get(3).getMinorType()); + assertEquals(MinorType.VARCHAR, primitiveVectors.get(4).getMinorType()); + assertEquals(MinorType.EXTENSIONTYPE, primitiveVectors.get(5).getMinorType()); } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestUuidVector.java b/vector/src/test/java/org/apache/arrow/vector/TestUuidVector.java index 3d70238ece..a3690461cf 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestUuidVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestUuidVector.java @@ -33,6 +33,7 @@ import org.apache.arrow.vector.holders.ExtensionHolder; import org.apache.arrow.vector.holders.NullableUuidHolder; import org.apache.arrow.vector.holders.UuidHolder; +import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.util.UuidUtility; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -358,7 +359,13 @@ void testReaderReadWithUnsupportedHolder() throws Exception { reader.setPosition(0); // Create a mock unsupported holder - ExtensionHolder unsupportedHolder = new ExtensionHolder() {}; + ExtensionHolder unsupportedHolder = + new ExtensionHolder() { + @Override + public ArrowType type() { + return null; + } + }; IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> reader.read(unsupportedHolder)); @@ -377,7 +384,13 @@ void testReaderReadWithArrayIndexUnsupportedHolder() throws Exception { UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); // Create a mock unsupported holder - ExtensionHolder unsupportedHolder = new ExtensionHolder() {}; + ExtensionHolder unsupportedHolder = + new ExtensionHolder() { + @Override + public ArrowType type() { + return null; + } + }; IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> reader.read(0, unsupportedHolder)); diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java index 73c1cd3b74..b2a8cf9ba4 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java @@ -861,7 +861,6 @@ public void testCopyListVectorWithExtensionType() { listWriter.setPosition(i); listWriter.startList(); ExtensionWriter extensionWriter = listWriter.extension(UuidType.INSTANCE); - extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); extensionWriter.writeExtension(UUID.randomUUID()); extensionWriter.writeExtension(UUID.randomUUID()); listWriter.endList(); @@ -874,7 +873,7 @@ public void testCopyListVectorWithExtensionType() { for (int i = 0; i < COUNT; i++) { in.setPosition(i); out.setPosition(i); - ComplexCopier.copy(in, out, new UuidWriterFactory()); + ComplexCopier.copy(in, out); } to.setValueCount(COUNT); @@ -897,11 +896,9 @@ public void testCopyMapVectorWithExtensionType() { mapWriter.startMap(); mapWriter.startEntry(); ExtensionWriter extensionKeyWriter = mapWriter.key().extension(UuidType.INSTANCE); - extensionKeyWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); - extensionKeyWriter.writeExtension(UUID.randomUUID()); + extensionKeyWriter.writeExtension(UUID.randomUUID(), UuidType.INSTANCE); ExtensionWriter extensionValueWriter = mapWriter.value().extension(UuidType.INSTANCE); - extensionValueWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); - extensionValueWriter.writeExtension(UUID.randomUUID()); + extensionValueWriter.writeExtension(UUID.randomUUID(), UuidType.INSTANCE); mapWriter.endEntry(); mapWriter.endMap(); } @@ -914,7 +911,7 @@ public void testCopyMapVectorWithExtensionType() { for (int i = 0; i < COUNT; i++) { in.setPosition(i); out.setPosition(i); - ComplexCopier.copy(in, out, new UuidWriterFactory()); + ComplexCopier.copy(in, out); } to.setValueCount(COUNT); @@ -934,12 +931,10 @@ public void testCopyStructVectorWithExtensionType() { for (int i = 0; i < COUNT; i++) { structWriter.setPosition(i); structWriter.start(); - ExtensionWriter extensionWriter1 = structWriter.extension("timestamp1", UuidType.INSTANCE); - extensionWriter1.addExtensionTypeWriterFactory(new UuidWriterFactory()); - extensionWriter1.writeExtension(UUID.randomUUID()); - ExtensionWriter extensionWriter2 = structWriter.extension("timestamp2", UuidType.INSTANCE); - extensionWriter2.addExtensionTypeWriterFactory(new UuidWriterFactory()); - extensionWriter2.writeExtension(UUID.randomUUID()); + ExtensionWriter extensionWriter1 = structWriter.extension("uuid1", UuidType.INSTANCE); + extensionWriter1.writeExtension(UUID.randomUUID(), UuidType.INSTANCE); + ExtensionWriter extensionWriter2 = structWriter.extension("uuid2", UuidType.INSTANCE); + extensionWriter2.writeExtension(UUID.randomUUID(), UuidType.INSTANCE); structWriter.end(); } @@ -951,7 +946,7 @@ public void testCopyStructVectorWithExtensionType() { for (int i = 0; i < COUNT; i++) { in.setPosition(i); out.setPosition(i); - ComplexCopier.copy(in, out, new UuidWriterFactory()); + ComplexCopier.copy(in, out); } to.setValueCount(COUNT); diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java index c71717a027..5b6d65d6ba 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java @@ -31,6 +31,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.DecimalVector; import org.apache.arrow.vector.DirtyRootAllocator; +import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.LargeVarBinaryVector; import org.apache.arrow.vector.LargeVarCharVector; import org.apache.arrow.vector.UuidVector; @@ -49,6 +50,7 @@ import org.apache.arrow.vector.holders.NullableTimeStampMilliTZHolder; import org.apache.arrow.vector.holders.TimeStampMilliTZHolder; import org.apache.arrow.vector.holders.UnionHolder; +import org.apache.arrow.vector.holders.UuidHolder; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -57,6 +59,7 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.DecimalUtility; import org.apache.arrow.vector.util.Text; +import org.apache.arrow.vector.util.UuidUtility; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -100,7 +103,6 @@ public void testPromoteToUnion() throws Exception { writer.integer("A").writeInt(10); // we don't write anything in 3 - writer.setPosition(4); writer.integer("A").writeInt(100); @@ -130,9 +132,21 @@ public void testPromoteToUnion() throws Exception { binHolder.buffer = buf; writer.fixedSizeBinary("A", 4).write(binHolder); + writer.setPosition(9); + UUID uuid = UUID.randomUUID(); + writer.extension("A", UuidType.INSTANCE).writeExtension(uuid, UuidType.INSTANCE); + writer.end(); + + writer.setPosition(10); + UUID uuid2 = UUID.randomUUID(); + UuidHolder uuidHolder = new UuidHolder(); + uuidHolder.buffer = allocator.buffer(UuidType.UUID_BYTE_WIDTH); + uuidHolder.buffer.setBytes(0, UuidUtility.getBytesFromUUID(uuid2)); + writer.extension("A", UuidType.INSTANCE).write(uuidHolder); writer.end(); + allocator.releaseBytes(UuidType.UUID_BYTE_WIDTH); - container.setValueCount(9); + container.setValueCount(11); final UnionVector uv = v.getChild("A", UnionVector.class); @@ -169,6 +183,12 @@ public void testPromoteToUnion() throws Exception { .order(ByteOrder.nativeOrder()) .getInt()); + assertFalse(uv.isNull(9), "9 shouldn't be null"); + assertEquals(uuid, uv.getObject(9)); + + assertFalse(uv.isNull(10), "10 shouldn't be null"); + assertEquals(uuid2, uv.getObject(10)); + container.clear(); container.allocateNew(); @@ -791,12 +811,11 @@ public void testExtensionType() throws Exception { UUID u2 = UUID.randomUUID(); container.allocateNew(); container.setValueCount(1); - writer.addExtensionTypeWriterFactory(new UuidWriterFactory()); writer.setPosition(0); - writer.writeExtension(u1); + writer.writeExtension(u1, UuidType.INSTANCE); writer.setPosition(1); - writer.writeExtension(u2); + writer.writeExtension(u2, UuidType.INSTANCE); container.setValueCount(2); @@ -817,16 +836,15 @@ public void testExtensionTypeForList() throws Exception { UUID u2 = UUID.randomUUID(); container.allocateNew(); container.setValueCount(1); - writer.addExtensionTypeWriterFactory(new UuidWriterFactory()); writer.setPosition(0); - writer.writeExtension(u1); + writer.writeExtension(u1, UuidType.INSTANCE); writer.setPosition(1); - writer.writeExtension(u2); + writer.writeExtension(u2, UuidType.INSTANCE); container.setValueCount(2); - UuidVector uuidVector = (UuidVector) container.getDataVector(); + FieldVector uuidVector = container.getDataVector(); assertEquals(u1, uuidVector.getObject(0)); assertEquals(u2, uuidVector.getObject(1)); } diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index 3a8f3f8e6a..b131bf07e2 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -66,7 +66,6 @@ import org.apache.arrow.vector.complex.impl.UnionMapReader; import org.apache.arrow.vector.complex.impl.UnionReader; import org.apache.arrow.vector.complex.impl.UnionWriter; -import org.apache.arrow.vector.complex.impl.UuidWriterFactory; import org.apache.arrow.vector.complex.reader.BaseReader.StructReader; import org.apache.arrow.vector.complex.reader.BigIntReader; import org.apache.arrow.vector.complex.reader.FieldReader; @@ -87,6 +86,7 @@ import org.apache.arrow.vector.holders.NullableFixedSizeBinaryHolder; import org.apache.arrow.vector.holders.NullableTimeStampMilliTZHolder; import org.apache.arrow.vector.holders.NullableTimeStampNanoTZHolder; +import org.apache.arrow.vector.holders.NullableUuidHolder; import org.apache.arrow.vector.holders.TimeStampMilliTZHolder; import org.apache.arrow.vector.holders.UuidHolder; import org.apache.arrow.vector.types.TimeUnit; @@ -1106,6 +1106,13 @@ public void simpleUnion() throws Exception { new UnionVector("union", allocator, /* field type */ null, /* call-back */ null); UnionWriter unionWriter = new UnionWriter(vector); unionWriter.allocate(); + + UUID uuid = UUID.randomUUID(); + ByteBuffer bb = ByteBuffer.allocate(16); + bb.putLong(uuid.getMostSignificantBits()); + bb.putLong(uuid.getLeastSignificantBits()); + byte[] uuidByte = bb.array(); + for (int i = 0; i < COUNT; i++) { unionWriter.setPosition(i); if (i % 5 == 0) { @@ -1128,6 +1135,12 @@ public void simpleUnion() throws Exception { holder.buffer = buf; unionWriter.write(holder); bufs.add(buf); + } else if (i % 5 == 4) { + UuidHolder holder = new UuidHolder(); + holder.buffer = allocator.buffer(UuidType.UUID_BYTE_WIDTH); + holder.buffer.setBytes(0, uuidByte); + unionWriter.write(holder); + allocator.releaseBytes(UuidType.UUID_BYTE_WIDTH); } else { unionWriter.writeFloat4((float) i); } @@ -1153,6 +1166,10 @@ public void simpleUnion() throws Exception { unionReader.read(holder); assertEquals(i, holder.buffer.getInt(0)); assertEquals(4, holder.byteWidth); + } else if (i % 5 == 4) { + NullableUuidHolder holder = new NullableUuidHolder(); + unionReader.read(holder); + assertEquals(UuidUtility.uuidFromArrowBuf(holder.buffer, 0), uuid); } else { assertEquals((float) i, unionReader.readFloat(), 1e-12); } @@ -2512,8 +2529,7 @@ public void extensionWriterReader() throws Exception { { ExtensionWriter extensionWriter = rootWriter.extension("uuid1", UuidType.INSTANCE); extensionWriter.setPosition(0); - extensionWriter.addExtensionTypeWriterFactory(new UuidWriterFactory()); - extensionWriter.writeExtension(u1); + extensionWriter.writeExtension(u1, UuidType.INSTANCE); } // read StructReader rootReader = new SingleStructReaderImpl(parent).reader("root"); diff --git a/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java b/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java index 2ac4045aa2..ae5ac0726c 100644 --- a/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java +++ b/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java @@ -44,10 +44,12 @@ import org.apache.arrow.vector.Float4Vector; import org.apache.arrow.vector.UuidVector; import org.apache.arrow.vector.ValueIterableVector; +import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.compare.Range; import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.complex.StructVector; +import org.apache.arrow.vector.complex.writer.FieldWriter; import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.ipc.ArrowFileReader; import org.apache.arrow.vector.ipc.ArrowFileWriter; @@ -333,6 +335,11 @@ public String serialize() { public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator) { return new LocationVector(name, allocator); } + + @Override + public FieldWriter getNewFieldWriter(ValueVector vector) { + throw new UnsupportedOperationException("Not yet implemented."); + } } public static class LocationVector extends ExtensionTypeVector From 349d402a61733084399cc791710e251097b87ea3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Sat, 17 Jan 2026 05:00:44 +0000 Subject: [PATCH 120/271] GH-964: Fix IndexOutOfBoundsException in Array.getResultSet() for JDBC clients (#965) ## What's Changed - Fixed JDBC specification in ArrowFlightJdbcArray.getResultSet() that caused IndexOutOfBoundsException in JDBC clients like DBeaver when reading array columns - The method returned a single-column ResultSet containing only array values, but JDBC spec requires a 2-column format - Not it returns two columns: - Column 1 (INDEX): 1-based element indices per JDBC specification - Column 2: The actual array element values Closes #964. --- .../driver/jdbc/ArrowFlightJdbcArray.java | 19 +++++++++++++++---- .../driver/jdbc/ArrowFlightJdbcArrayTest.java | 4 ++-- ...stractArrowFlightJdbcListAccessorTest.java | 7 ++++++- .../ArrowFlightJdbcMapVectorAccessorTest.java | 16 ++++++++-------- 4 files changed, 31 insertions(+), 15 deletions(-) diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcArray.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcArray.java index 9b9eba51e5..f3d76ace92 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcArray.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcArray.java @@ -26,6 +26,7 @@ import org.apache.arrow.driver.jdbc.utils.SqlTypes; import org.apache.arrow.memory.util.LargeMemoryUtil; import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -135,12 +136,22 @@ public ResultSet getResultSet(long index, int count) throws SQLException { private static ResultSet getResultSetNoBoundariesCheck( ValueVector dataVector, long start, long count) throws SQLException { + int intStart = LargeMemoryUtil.checkedCastToInt(start); + int intCount = LargeMemoryUtil.checkedCastToInt(count); + + // Create an index vector with 1-based indices (per JDBC spec) to return with value vector + IntVector indexVector = new IntVector("INDEX", dataVector.getAllocator()); + indexVector.allocateNew(intCount); + for (int i = 0; i < intCount; i++) { + indexVector.set(i, i + 1); + } + indexVector.setValueCount(intCount); + TransferPair transferPair = dataVector.getTransferPair(dataVector.getAllocator()); - transferPair.splitAndTransfer( - LargeMemoryUtil.checkedCastToInt(start), LargeMemoryUtil.checkedCastToInt(count)); - FieldVector vectorSlice = (FieldVector) transferPair.getTo(); + transferPair.splitAndTransfer(intStart, intCount); + FieldVector valueVector = (FieldVector) transferPair.getTo(); - VectorSchemaRoot vectorSchemaRoot = VectorSchemaRoot.of(vectorSlice); + VectorSchemaRoot vectorSchemaRoot = VectorSchemaRoot.of(indexVector, valueVector); return ArrowFlightJdbcVectorSchemaRootResultSet.fromVectorSchemaRoot(vectorSchemaRoot); } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcArrayTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcArrayTest.java index 06d101724c..cb6abacb2f 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcArrayTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcArrayTest.java @@ -129,7 +129,7 @@ public void testShouldGetResultSetReturnValidResultSet() throws SQLException { try (ResultSet resultSet = arrowFlightJdbcArray.getResultSet()) { int count = 0; while (resultSet.next()) { - assertEquals((Object) resultSet.getInt(1), dataVector.getObject(count)); + assertEquals((Object) resultSet.getInt(2), dataVector.getObject(count)); count++; } } @@ -142,7 +142,7 @@ public void testShouldGetResultSetReturnValidResultSetWithOffsets() throws SQLEx try (ResultSet resultSet = arrowFlightJdbcArray.getResultSet(3, 5)) { int count = 0; while (resultSet.next()) { - assertEquals((Object) resultSet.getInt(1), dataVector.getObject(count + 3)); + assertEquals((Object) resultSet.getInt(2), dataVector.getObject(count + 3)); count++; } assertEquals(5, count); diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/AbstractArrowFlightJdbcListAccessorTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/AbstractArrowFlightJdbcListAccessorTest.java index ad689837e2..c5eb6e34ef 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/AbstractArrowFlightJdbcListAccessorTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/AbstractArrowFlightJdbcListAccessorTest.java @@ -191,7 +191,12 @@ public void testShouldGetArrayGetResultSetReturnValidResultSet( try (ResultSet rs = array.getResultSet()) { int count = 0; while (rs.next()) { - final int value = rs.getInt(1); + // Column 1: 1-based index (per JDBC spec) + final int index = rs.getInt(1); + assertThat(index, equalTo(count + 1)); + + // Column 2: actual value (per JDBC spec) + final int value = rs.getInt(2); assertThat(value, equalTo(currentRow * count)); count++; } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/ArrowFlightJdbcMapVectorAccessorTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/ArrowFlightJdbcMapVectorAccessorTest.java index 696e5afb71..f2d1725fd8 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/ArrowFlightJdbcMapVectorAccessorTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/ArrowFlightJdbcMapVectorAccessorTest.java @@ -153,15 +153,15 @@ public void testShouldGetArrayReturnValidArray() throws SQLException { try (ResultSet resultSet = array.getResultSet()) { assertTrue(resultSet.next()); - Map entry = resultSet.getObject(1, Map.class); + Map entry = resultSet.getObject(2, Map.class); assertEquals(1, entry.get("key")); assertEquals(11, entry.get("value")); assertTrue(resultSet.next()); - entry = resultSet.getObject(1, Map.class); + entry = resultSet.getObject(2, Map.class); assertEquals(2, entry.get("key")); assertEquals(22, entry.get("value")); assertTrue(resultSet.next()); - entry = resultSet.getObject(1, Map.class); + entry = resultSet.getObject(2, Map.class); assertEquals(3, entry.get("key")); assertEquals(33, entry.get("value")); assertFalse(resultSet.next()); @@ -173,7 +173,7 @@ public void testShouldGetArrayReturnValidArray() throws SQLException { assertFalse(accessor.wasNull()); try (ResultSet resultSet = array.getResultSet()) { assertTrue(resultSet.next()); - Map entry = resultSet.getObject(1, Map.class); + Map entry = resultSet.getObject(2, Map.class); assertEquals(2, entry.get("key")); assertNull(entry.get("value")); assertFalse(resultSet.next()); @@ -185,19 +185,19 @@ public void testShouldGetArrayReturnValidArray() throws SQLException { assertFalse(accessor.wasNull()); try (ResultSet resultSet = array.getResultSet()) { assertTrue(resultSet.next()); - Map entry = resultSet.getObject(1, Map.class); + Map entry = resultSet.getObject(2, Map.class); assertEquals(0, entry.get("key")); assertEquals(2000, entry.get("value")); assertTrue(resultSet.next()); - entry = resultSet.getObject(1, Map.class); + entry = resultSet.getObject(2, Map.class); assertEquals(1, entry.get("key")); assertEquals(2001, entry.get("value")); assertTrue(resultSet.next()); - entry = resultSet.getObject(1, Map.class); + entry = resultSet.getObject(2, Map.class); assertEquals(2, entry.get("key")); assertEquals(2002, entry.get("value")); assertTrue(resultSet.next()); - entry = resultSet.getObject(1, Map.class); + entry = resultSet.getObject(2, Map.class); assertEquals(3, entry.get("key")); assertEquals(2003, entry.get("value")); assertFalse(resultSet.next()); From 6eab884d7d868863a283435c29e7bbd912138379 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jan 2026 09:33:31 +0100 Subject: [PATCH 121/271] MINOR: Bump org.bouncycastle:bcpkix-jdk18on from 1.82 to 1.83 (#969) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.bouncycastle:bcpkix-jdk18on](https://github.com/bcgit/bc-java) from 1.82 to 1.83.

Changelog

Sourced from org.bouncycastle:bcpkix-jdk18on's changelog.

2.1.1 Version Release: 1.84 Date:      TBD

2.2.1 Version Release: 1.83 Date:      2025, November 27th.

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.bouncycastle:bcpkix-jdk18on&package-manager=maven&previous-version=1.82&new-version=1.83)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 8801ad8178..d84352e2da 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -140,7 +140,7 @@ under the License. org.bouncycastle bcpkix-jdk18on - 1.82 + 1.83 From 1e8608a5b3205eff9d41dac11cf0d2a9f334be83 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jan 2026 09:34:59 +0100 Subject: [PATCH 122/271] MINOR: Bump logback.version from 1.5.24 to 1.5.25 (#975) Bumps `logback.version` from 1.5.24 to 1.5.25. Updates `ch.qos.logback:logback-classic` from 1.5.24 to 1.5.25
Commits
  • f426e00 prepare release of 1.5.25
  • d28931f restrict object creation to expected supertype
  • aa264f7 test default variable values in appender-ref ref attribute
  • 8fb403a adjust copyright year
  • b294a12 check optionList in start()
  • b65040a Add EpochConverter for milliseconds/seconds since epoch (related to issue #96...
  • 0690174 cla for Duncan Jauncey
  • 71dc2af Removed email address for Tony.
  • 1f97ae1 check for undeclared by referenced appenders
  • b07355e Move the artifact version checking code to VersionUtil in logback-core.
  • Additional commits viewable in compare view

Updates `ch.qos.logback:logback-core` from 1.5.24 to 1.5.25
Commits
  • f426e00 prepare release of 1.5.25
  • d28931f restrict object creation to expected supertype
  • aa264f7 test default variable values in appender-ref ref attribute
  • 8fb403a adjust copyright year
  • b294a12 check optionList in start()
  • b65040a Add EpochConverter for milliseconds/seconds since epoch (related to issue #96...
  • 0690174 cla for Duncan Jauncey
  • 71dc2af Removed email address for Tony.
  • 1f97ae1 check for undeclared by referenced appenders
  • b07355e Move the artifact version checking code to VersionUtil in logback-core.
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d6fdfdd477..64e4a612ec 100644 --- a/pom.xml +++ b/pom.xml @@ -111,7 +111,7 @@ under the License. true 2.42.0 3.53.0 - 1.5.24 + 1.5.25 none -Xdoclint:none From 3d44a1c2e71d5f38980cc1030e5652f5f62b942d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jan 2026 09:46:51 +0100 Subject: [PATCH 123/271] MINOR: Bump com.fasterxml.jackson:jackson-bom from 2.18.3 to 2.21.0 (#973) Bumps [com.fasterxml.jackson:jackson-bom](https://github.com/FasterXML/jackson-bom) from 2.18.3 to 2.21.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.fasterxml.jackson:jackson-bom&package-manager=maven&previous-version=2.18.3&new-version=2.21.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 64e4a612ec..b3141d5cd8 100644 --- a/pom.xml +++ b/pom.xml @@ -100,7 +100,7 @@ under the License. 4.2.9.Final 1.78.0 4.33.1 - 2.18.3 + 2.21.0 3.4.2 25.2.10 1.12.1 From f36777c09246532090c01ca2b5127bb80fd703fb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jan 2026 09:48:12 +0100 Subject: [PATCH 124/271] MINOR: Bump parquet.version from 1.16.0 to 1.17.0 (#968) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps `parquet.version` from 1.16.0 to 1.17.0. Updates `org.apache.parquet:parquet-avro` from 1.16.0 to 1.17.0
Release notes

Sourced from org.apache.parquet:parquet-avro's releases.

Apache Parquet 1.17.0

What's Changed

New Contributors

... (truncated)

Commits
  • fac0c74 [maven-release-plugin] prepare release apache-parquet-1.17.0-rc0
  • a8ead9d Bump protobuf.version from 4.33.1 to 4.33.2 (#3373)
  • 0ecd799 Allow reading dictionary encoded boolean (#3370)
  • 46218f2 Bump commons-io:commons-io from 2.18.0 to 2.21.0 (#3369)
  • 7ec3284 Exclude package-info.class from shaded fastutil (#3322)
  • 7453be4 Bump com.google.guava:guava from 33.4.0-jre to 33.5.0-jre (#3366)
  • 893ef11 Bump easymock 5.6.0 to support Java 25 (#3363)
  • 6b2940c Remove unused parquet-thrift dependencies (#3323)
  • 5040a63 Bump protobuf.version from 3.25.6 to 4.30.2 (#3182)
  • 2ccc243 MINOR: parquet-avro tests should not debug to stderr (#3329)
  • Additional commits viewable in compare view

Updates `org.apache.parquet:parquet-hadoop` from 1.16.0 to 1.17.0
Release notes

Sourced from org.apache.parquet:parquet-hadoop's releases.

Apache Parquet 1.17.0

What's Changed

New Contributors

... (truncated)

Commits
  • fac0c74 [maven-release-plugin] prepare release apache-parquet-1.17.0-rc0
  • a8ead9d Bump protobuf.version from 4.33.1 to 4.33.2 (#3373)
  • 0ecd799 Allow reading dictionary encoded boolean (#3370)
  • 46218f2 Bump commons-io:commons-io from 2.18.0 to 2.21.0 (#3369)
  • 7ec3284 Exclude package-info.class from shaded fastutil (#3322)
  • 7453be4 Bump com.google.guava:guava from 33.4.0-jre to 33.5.0-jre (#3366)
  • 893ef11 Bump easymock 5.6.0 to support Java 25 (#3363)
  • 6b2940c Remove unused parquet-thrift dependencies (#3323)
  • 5040a63 Bump protobuf.version from 3.25.6 to 4.30.2 (#3182)
  • 2ccc243 MINOR: parquet-avro tests should not debug to stderr (#3329)
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dataset/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataset/pom.xml b/dataset/pom.xml index 2d582268a6..fcf54e785f 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -32,7 +32,7 @@ under the License. ../../../cpp/release-build/ - 1.16.0 + 1.17.0 1.12.1 From f9ab35c5628ab19af8df6c1146ac495edf917219 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jan 2026 13:40:23 +0100 Subject: [PATCH 125/271] MINOR: Bump commons-io:commons-io from 2.19.0 to 2.21.0 (#974) Bumps [commons-io:commons-io](https://github.com/apache/commons-io) from 2.19.0 to 2.21.0.
Changelog

Sourced from commons-io:commons-io's changelog.

Apache Commons IO 2.21.0 Release Notes

The Apache Commons IO team is pleased to announce the release of Apache Commons IO 2.21.0.

Introduction

The Apache Commons IO library contains utility classes, stream implementations, file filters, file comparators, endian transformation classes, and much more.

Version 2.21.0: Java 8 or later is required.

New features

o FileUtils#byteCountToDisplaySize() supports Zettabyte, Yottabyte, Ronnabyte and Quettabyte #763. Thanks to strangelookingnerd, Gary Gregory. o Add org.apache.commons.io.FileUtils.ONE_RB #763. Thanks to strangelookingnerd, Gary Gregory. o Add org.apache.commons.io.FileUtils.ONE_QB #763. Thanks to strangelookingnerd, Gary Gregory. o Add org.apache.commons.io.output.ProxyOutputStream.writeRepeat(byte[], int, int, long). Thanks to Gary Gregory. o Add org.apache.commons.io.output.ProxyOutputStream.writeRepeat(byte[], long). Thanks to Gary Gregory. o Add org.apache.commons.io.output.ProxyOutputStream.writeRepeat(int, long). Thanks to Gary Gregory. o Add length unit support in FileSystem limits. Thanks to Piotr P. Karwasz. o Add IOUtils.toByteArray(InputStream, int, int) for safer chunked reading with size validation. Thanks to Piotr P. Karwasz. o Add org.apache.commons.io.file.PathUtils.getPath(String, String). Thanks to Gary Gregory. o Add org.apache.commons.io.channels.ByteArraySeekableByteChannel. Thanks to Gary Gregory. o Add IOIterable.asIterable(). Thanks to Gary Gregory. o Add NIO channel support to AbstractStreamBuilder. Thanks to Piotr P. Karwasz. o Add CloseShieldChannel to close-shielded NIO Channels #786. Thanks to Piotr P. Karwasz. o Added IOUtils.checkFromIndexSize as a Java 8 backport of Objects.checkFromIndexSize #790. Thanks to Piotr P. Karwasz.

Fixed Bugs

o When testing on Java 21 and up, enable -XX:+EnableDynamicAgentLoading. Thanks to Gary Gregory. o When testing on Java 24 and up, don't fail FileUtilsListFilesTest for a different behavior in the JRE. Thanks to Gary Gregory. o ValidatingObjectInputStream does not validate dynamic proxy interfaces. Thanks to Stanislav Fort, Gary Gregory. o BoundedInputStream.getRemaining() now reports Long.MAX_VALUE instead of 0 when no limit is set. Thanks to Piotr P. Karwasz. o BoundedInputStream.available() correctly accounts for the maximum read limit. Thanks to Piotr P. Karwasz. o Deprecate IOUtils.readFully(InputStream, int) in favor of toByteArray(InputStream, int). Thanks to Gary Gregory, Piotr P. Karwasz. o IOUtils.toByteArray(InputStream) now throws IOException on byte array overflow. Thanks to Piotr P. Karwasz. o Javadoc general improvements. Thanks to Gary Gregory, Piotr P. Karwasz. o IOUtils.toByteArray() now throws EOFException when not enough data is available #796. Thanks to Piotr P. Karwasz. o Fix IOUtils.skip() usage in concurrent scenarios. Thanks to Piotr P. Karwasz. o [javadoc] Fix XmlStreamReader Javadoc to indicate the correct class that is built #806. Thanks to J Hawkins.

Changes

o Bump org.apache.commons:commons-parent from 85 to 91 #774, #783, #808. Thanks to Gary Gregory, Dependabot.

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=commons-io:commons-io&package-manager=maven&previous-version=2.19.0&new-version=2.21.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dataset/pom.xml | 2 +- flight/flight-sql-jdbc-core/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dataset/pom.xml b/dataset/pom.xml index fcf54e785f..3a8f048628 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -156,7 +156,7 @@ under the License. commons-io commons-io - 2.19.0 + 2.21.0 test diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index d84352e2da..bb191ca9ed 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -105,7 +105,7 @@ under the License. commons-io commons-io - 2.19.0 + 2.21.0 test From a74728d490a8307b926f0539eeb582220496ce58 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jan 2026 14:06:30 +0100 Subject: [PATCH 126/271] MINOR: Bump com.gradle:develocity-maven-extension from 2.0 to 2.3.1 (#976) Bumps com.gradle:develocity-maven-extension from 2.0 to 2.3.1. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.gradle:develocity-maven-extension&package-manager=maven&previous-version=2.0&new-version=2.3.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .mvn/extensions.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index 943140738d..b136e95f43 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -23,7 +23,7 @@ com.gradle develocity-maven-extension - 2.0 + 2.3.1 com.gradle From db9fff8638e907012b4fb4585723ebbc6514ab20 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jan 2026 14:26:37 +0100 Subject: [PATCH 127/271] MINOR: Bump org.apache.orc:orc-core from 2.2.1 to 2.2.2 (#971) Bumps org.apache.orc:orc-core from 2.2.1 to 2.2.2. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.orc:orc-core&package-manager=maven&previous-version=2.2.1&new-version=2.2.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- adapter/orc/pom.xml | 2 +- dataset/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/adapter/orc/pom.xml b/adapter/orc/pom.xml index ef72b2de65..89d45e155c 100644 --- a/adapter/orc/pom.xml +++ b/adapter/orc/pom.xml @@ -61,7 +61,7 @@ under the License. org.apache.orc orc-core - 2.2.1 + 2.2.2 test diff --git a/dataset/pom.xml b/dataset/pom.xml index 3a8f048628..686a234358 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -130,7 +130,7 @@ under the License. org.apache.orc orc-core - 2.2.1 + 2.2.2 test From 71c418c2fde3c49d4cfa4ec564514d04f66f71bf Mon Sep 17 00:00:00 2001 From: Joana Hrotko Date: Mon, 19 Jan 2026 13:41:06 +0000 Subject: [PATCH 128/271] GH-948: Use buffer indexing for UUID vector (#949) ## What's Changed The current UUID vector implementation creates new buffer slices when reading values through holders, which has several drawbacks: - Memory overhead: Each slice creates a new ArrowBuf object - Performance impact: Buffer slicing is slower than direct buffer indexing - Inconsistency: Other fixed-width types (like Decimal) use buffer indexing with a `start` offset field ### Proposed Changes 1. Add `start` field to UUID holders to track buffer offsets: - `UuidHolder`: Add `public int start = 0;` - `NullableUuidHolder`: Add `public int start = 0;` 2. Update `UuidVector` to use buffer indexing 3. Update readers and writers ### Related Work - Original UUID extension type implementation: GH-825 (#903) Closes #948 --- .../arrow/vector/UuidVectorBenchmarks.java | 134 +++++++ .../org/apache/arrow/vector/UuidVector.java | 115 +++--- .../impl/NullableUuidHolderReaderImpl.java | 123 +++++++ .../vector/complex/impl/UuidReaderImpl.java | 8 +- .../vector/complex/impl/UuidWriterImpl.java | 9 +- .../vector/holders/NullableUuidHolder.java | 3 + .../arrow/vector/holders/UuidHolder.java | 3 + .../arrow/vector/TestLargeListVector.java | 12 +- .../apache/arrow/vector/TestListVector.java | 24 +- .../apache/arrow/vector/TestMapVector.java | 20 +- .../org/apache/arrow/vector/TestUuidType.java | 3 +- .../apache/arrow/vector/TestUuidVector.java | 334 ++++++++++++++++-- .../complex/writer/TestComplexWriter.java | 4 +- 13 files changed, 649 insertions(+), 143 deletions(-) create mode 100644 performance/src/main/java/org/apache/arrow/vector/UuidVectorBenchmarks.java create mode 100644 vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableUuidHolderReaderImpl.java diff --git a/performance/src/main/java/org/apache/arrow/vector/UuidVectorBenchmarks.java b/performance/src/main/java/org/apache/arrow/vector/UuidVectorBenchmarks.java new file mode 100644 index 0000000000..b5f87e7a75 --- /dev/null +++ b/performance/src/main/java/org/apache/arrow/vector/UuidVectorBenchmarks.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector; + +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.complex.impl.UuidWriterImpl; +import org.apache.arrow.vector.holders.NullableUuidHolder; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.profile.GCProfiler; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +/** Benchmarks for {@link UuidVector}. */ +@State(Scope.Benchmark) +public class UuidVectorBenchmarks { + // checkstyle:off: MissingJavadocMethod + + private static final int VECTOR_LENGTH = 10_000; + + private static final int ALLOCATOR_CAPACITY = 1024 * 1024; + + private BufferAllocator allocator; + + private UuidVector vector; + + private UUID[] testUuids; + + @Setup + public void prepare() { + allocator = new RootAllocator(ALLOCATOR_CAPACITY); + vector = new UuidVector("vector", allocator); + vector.allocateNew(VECTOR_LENGTH); + vector.setValueCount(VECTOR_LENGTH); + + // Pre-generate UUIDs for consistent benchmarking + testUuids = new UUID[VECTOR_LENGTH]; + for (int i = 0; i < VECTOR_LENGTH; i++) { + testUuids[i] = new UUID(i, i * 2L); + } + } + + @TearDown + public void tearDown() { + vector.close(); + allocator.close(); + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + public void setWithHolder() { + NullableUuidHolder holder = new NullableUuidHolder(); + for (int i = 0; i < VECTOR_LENGTH; i++) { + vector.get(i, holder); + vector.setSafe(i, holder); + } + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + public void setUuidDirectly() { + for (int i = 0; i < VECTOR_LENGTH; i++) { + vector.setSafe(i, testUuids[i]); + } + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + public void setWithWriter() { + UuidWriterImpl writer = new UuidWriterImpl(vector); + for (int i = 0; i < VECTOR_LENGTH; i++) { + writer.writeExtension(testUuids[i]); + } + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + public void getWithUuidHolder() { + NullableUuidHolder holder = new NullableUuidHolder(); + for (int i = 0; i < VECTOR_LENGTH; i++) { + vector.get(i, holder); + } + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + public void getUuidDirectly() { + for (int i = 0; i < VECTOR_LENGTH; i++) { + UUID uuid = vector.getObject(i); + } + } + + public static void main(String[] args) throws RunnerException { + Options opt = + new OptionsBuilder() + .include(UuidVectorBenchmarks.class.getSimpleName()) + .forks(1) + .addProfiler(GCProfiler.class) + .build(); + + new Runner(opt).run(); + } + // checkstyle:on: MissingJavadocMethod +} diff --git a/vector/src/main/java/org/apache/arrow/vector/UuidVector.java b/vector/src/main/java/org/apache/arrow/vector/UuidVector.java index e0dadd1c67..e1e61a5a2e 100644 --- a/vector/src/main/java/org/apache/arrow/vector/UuidVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/UuidVector.java @@ -23,7 +23,9 @@ import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.util.ArrowBufPointer; +import org.apache.arrow.memory.util.ByteFunctionHelpers; import org.apache.arrow.memory.util.hash.ArrowBufHasher; +import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.complex.impl.UuidReaderImpl; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.extension.UuidType; @@ -132,7 +134,8 @@ public int hashCode(int index) { @Override public int hashCode(int index, ArrowBufHasher hasher) { - return getUnderlyingVector().hashCode(index, hasher); + int start = this.getStartOffset(index); + return ByteFunctionHelpers.hash(hasher, this.getDataBuffer(), start, start + UUID_BYTE_WIDTH); } /** @@ -145,21 +148,6 @@ public int isSet(int index) { return getUnderlyingVector().isSet(index); } - /** - * Gets the UUID value at the given index as an ArrowBuf. - * - * @param index the index to retrieve - * @return a buffer slice containing the 16-byte UUID - * @throws IllegalStateException if the value at the index is null and null checking is enabled - */ - public ArrowBuf get(int index) throws IllegalStateException { - if (NullCheckingForGet.NULL_CHECKING_ENABLED && this.isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } else { - return getBufferSlicePostNullCheck(index); - } - } - /** * Reads the UUID value at the given index into a NullableUuidHolder. * @@ -167,23 +155,24 @@ public ArrowBuf get(int index) throws IllegalStateException { * @param holder the holder to populate with the UUID data */ public void get(int index, NullableUuidHolder holder) { - if (NullCheckingForGet.NULL_CHECKING_ENABLED && this.isSet(index) == 0) { + Preconditions.checkArgument(index >= 0, "Cannot get negative index in UUID vector."); + if (isSet(index) == 0) { holder.isSet = 0; - } else { - holder.isSet = 1; - holder.buffer = getBufferSlicePostNullCheck(index); + return; } + holder.isSet = 1; + holder.buffer = getDataBuffer(); + holder.start = getStartOffset(index); } /** - * Reads the UUID value at the given index into a UuidHolder. + * Calculates the byte offset for a given index in the data buffer. * - * @param index the index to read from - * @param holder the holder to populate with the UUID data + * @param index the index of the UUID value + * @return the byte offset in the data buffer */ - public void get(int index, UuidHolder holder) { - holder.isSet = 1; - holder.buffer = getBufferSlicePostNullCheck(index); + public final int getStartOffset(int index) { + return index * UUID_BYTE_WIDTH; } /** @@ -207,7 +196,7 @@ public void set(int index, UUID value) { * @param holder the holder containing the UUID data */ public void set(int index, UuidHolder holder) { - this.set(index, holder.isSet, holder.buffer); + this.set(index, holder.buffer, holder.start); } /** @@ -217,28 +206,11 @@ public void set(int index, UuidHolder holder) { * @param holder the holder containing the UUID data */ public void set(int index, NullableUuidHolder holder) { - this.set(index, holder.isSet, holder.buffer); - } - - /** - * Sets the UUID value at the given index with explicit null flag. - * - * @param index the index to set - * @param isSet 1 if the value is set, 0 if null - * @param buffer the buffer containing the 16-byte UUID data - */ - public void set(int index, int isSet, ArrowBuf buffer) { - getUnderlyingVector().set(index, isSet, buffer); - } - - /** - * Sets the UUID value at the given index from an ArrowBuf. - * - * @param index the index to set - * @param value the buffer containing the 16-byte UUID data - */ - public void set(int index, ArrowBuf value) { - getUnderlyingVector().set(index, value); + if (holder.isSet == 0) { + getUnderlyingVector().setNull(index); + } else { + this.set(index, holder.buffer, holder.start); + } } /** @@ -249,10 +221,12 @@ public void set(int index, ArrowBuf value) { * @param sourceOffset the offset in the source buffer where the UUID data starts */ public void set(int index, ArrowBuf source, int sourceOffset) { - // Copy bytes from source buffer to target vector data buffer - ArrowBuf dataBuffer = getUnderlyingVector().getDataBuffer(); - dataBuffer.setBytes((long) index * UUID_BYTE_WIDTH, source, sourceOffset, UUID_BYTE_WIDTH); - getUnderlyingVector().setIndexDefined(index); + Preconditions.checkNotNull(source, "Cannot set UUID vector, the source buffer is null."); + + BitVectorHelper.setBit(getUnderlyingVector().getValidityBuffer(), index); + getUnderlyingVector() + .getDataBuffer() + .setBytes((long) index * UUID_BYTE_WIDTH, source, sourceOffset, UUID_BYTE_WIDTH); } /** @@ -286,10 +260,10 @@ public void setSafe(int index, UUID value) { * @param holder the holder containing the UUID data, or null to set a null value */ public void setSafe(int index, NullableUuidHolder holder) { - if (holder != null) { - getUnderlyingVector().setSafe(index, holder.isSet, holder.buffer); - } else { + if (holder == null || holder.isSet == 0) { getUnderlyingVector().setNull(index); + } else { + this.setSafe(index, holder.buffer, holder.start); } } @@ -297,14 +271,23 @@ public void setSafe(int index, NullableUuidHolder holder) { * Sets the UUID value at the given index from a UuidHolder, expanding capacity if needed. * * @param index the index to set - * @param holder the holder containing the UUID data, or null to set a null value + * @param holder the holder containing the UUID data */ public void setSafe(int index, UuidHolder holder) { - if (holder != null) { - getUnderlyingVector().setSafe(index, holder.isSet, holder.buffer); - } else { - getUnderlyingVector().setNull(index); - } + this.setSafe(index, holder.buffer, holder.start); + } + + /** + * Sets the UUID value at the given index by copying from a source buffer, expanding capacity if + * needed. + * + * @param index the index to set + * @param buffer the source buffer to copy from + * @param start the offset in the source buffer where the UUID data starts + */ + public void setSafe(int index, ArrowBuf buffer, int start) { + getUnderlyingVector().handleSafe(index); + this.set(index, buffer, start); } /** @@ -400,15 +383,9 @@ public TransferPair getTransferPair(BufferAllocator allocator) { return getTransferPair(this.getField().getName(), allocator); } - private ArrowBuf getBufferSlicePostNullCheck(int index) { - return getUnderlyingVector() - .getDataBuffer() - .slice((long) index * UUID_BYTE_WIDTH, UUID_BYTE_WIDTH); - } - @Override public int getTypeWidth() { - return getUnderlyingVector().getTypeWidth(); + return UUID_BYTE_WIDTH; } /** {@link TransferPair} for {@link UuidVector}. */ diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableUuidHolderReaderImpl.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableUuidHolderReaderImpl.java new file mode 100644 index 0000000000..7a5312f6ed --- /dev/null +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableUuidHolderReaderImpl.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.complex.impl; + +import org.apache.arrow.vector.holders.ExtensionHolder; +import org.apache.arrow.vector.holders.NullableUuidHolder; +import org.apache.arrow.vector.holders.UuidHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.util.UuidUtility; + +/** + * Reader implementation for reading UUID values from a {@link NullableUuidHolder}. + * + *

This reader wraps a single UUID holder value and provides methods to read from it. Unlike + * {@link UuidReaderImpl} which reads from a vector, this reader operates on a holder instance. + * + * @see NullableUuidHolder + * @see UuidReaderImpl + */ +public class NullableUuidHolderReaderImpl extends AbstractFieldReader { + private final NullableUuidHolder holder; + + /** + * Constructs a reader for the given UUID holder. + * + * @param holder the UUID holder to read from + */ + public NullableUuidHolderReaderImpl(NullableUuidHolder holder) { + this.holder = holder; + } + + @Override + public int size() { + throw new UnsupportedOperationException( + "size() is not supported on NullableUuidHolderReaderImpl. " + + "This reader wraps a single UUID holder value, not a collection. " + + "Use UuidReaderImpl for vector-based UUID reading."); + } + + @Override + public boolean next() { + throw new UnsupportedOperationException( + "next() is not supported on NullableUuidHolderReaderImpl. " + + "This reader wraps a single UUID holder value, not an iterator. " + + "Use UuidReaderImpl for vector-based UUID reading."); + } + + @Override + public void setPosition(int index) { + throw new UnsupportedOperationException( + "setPosition() is not supported on NullableUuidHolderReaderImpl. " + + "This reader wraps a single UUID holder value, not a vector. " + + "Use UuidReaderImpl for vector-based UUID reading."); + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.EXTENSIONTYPE; + } + + @Override + public boolean isSet() { + return holder.isSet == 1; + } + + @Override + public void read(ExtensionHolder h) { + if (h instanceof NullableUuidHolder) { + NullableUuidHolder nullableHolder = (NullableUuidHolder) h; + nullableHolder.buffer = this.holder.buffer; + nullableHolder.isSet = this.holder.isSet; + nullableHolder.start = this.holder.start; + } else if (h instanceof UuidHolder) { + UuidHolder uuidHolder = (UuidHolder) h; + uuidHolder.buffer = this.holder.buffer; + uuidHolder.start = this.holder.start; + } else { + throw new IllegalArgumentException( + "Unsupported holder type: " + + h.getClass().getName() + + ". " + + "Only NullableUuidHolder and UuidHolder are supported for UUID values. " + + "Provided holder type cannot be used to read UUID data."); + } + } + + @Override + public Object readObject() { + if (!isSet()) { + return null; + } + // Convert UUID bytes to Java UUID object + try { + return UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); + } catch (Exception e) { + throw new RuntimeException( + String.format( + "Failed to read UUID from buffer. Invalid Arrow buffer state: " + + "capacity=%d, readableBytes=%d, readerIndex=%d, writerIndex=%d, refCnt=%d. " + + "The buffer must contain exactly 16 bytes of valid UUID data.", + holder.buffer.capacity(), + holder.buffer.readableBytes(), + holder.buffer.readerIndex(), + holder.buffer.writerIndex(), + holder.buffer.refCnt()), + e); + } + } +} diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java index bb35b960d3..bb7ae13e5b 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidReaderImpl.java @@ -63,9 +63,7 @@ public boolean isSet() { @Override public void read(ExtensionHolder holder) { - if (holder instanceof UuidHolder) { - vector.get(idx(), (UuidHolder) holder); - } else if (holder instanceof NullableUuidHolder) { + if (holder instanceof NullableUuidHolder) { vector.get(idx(), (NullableUuidHolder) holder); } else { throw new IllegalArgumentException( @@ -75,9 +73,7 @@ public void read(ExtensionHolder holder) { @Override public void read(int arrayIndex, ExtensionHolder holder) { - if (holder instanceof UuidHolder) { - vector.get(arrayIndex, (UuidHolder) holder); - } else if (holder instanceof NullableUuidHolder) { + if (holder instanceof NullableUuidHolder) { vector.get(arrayIndex, (NullableUuidHolder) holder); } else { throw new IllegalArgumentException( diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java index ee3c79d5e3..944b7e2e62 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/impl/UuidWriterImpl.java @@ -51,8 +51,15 @@ public void writeExtension(Object value) { vector.setSafe(getPosition(), (ArrowBuf) value); } else if (value instanceof java.util.UUID) { vector.setSafe(getPosition(), (java.util.UUID) value); + } else if (value instanceof ExtensionHolder) { + write((ExtensionHolder) value); } else { - throw new IllegalArgumentException("Unsupported value type for UUID: " + value.getClass()); + throw new IllegalArgumentException( + "Unsupported value type for UUID: " + + value.getClass().getName() + + ". " + + "Supported types are: byte[] (16 bytes), ArrowBuf (16 bytes), or java.util.UUID. " + + "Convert your value to one of these types before writing."); } vector.setValueCount(getPosition() + 1); } diff --git a/vector/src/main/java/org/apache/arrow/vector/holders/NullableUuidHolder.java b/vector/src/main/java/org/apache/arrow/vector/holders/NullableUuidHolder.java index 7fa50ca761..6a2b4ff604 100644 --- a/vector/src/main/java/org/apache/arrow/vector/holders/NullableUuidHolder.java +++ b/vector/src/main/java/org/apache/arrow/vector/holders/NullableUuidHolder.java @@ -35,6 +35,9 @@ public class NullableUuidHolder extends ExtensionHolder { /** Buffer containing 16-byte UUID data. */ public ArrowBuf buffer; + /** Offset in the buffer where the UUID data starts. */ + public int start = 0; + @Override public ArrowType type() { return UuidType.INSTANCE; diff --git a/vector/src/main/java/org/apache/arrow/vector/holders/UuidHolder.java b/vector/src/main/java/org/apache/arrow/vector/holders/UuidHolder.java index 8a0a66e435..9ec0305f30 100644 --- a/vector/src/main/java/org/apache/arrow/vector/holders/UuidHolder.java +++ b/vector/src/main/java/org/apache/arrow/vector/holders/UuidHolder.java @@ -33,6 +33,9 @@ public class UuidHolder extends ExtensionHolder { /** Buffer containing 16-byte UUID data. */ public ArrowBuf buffer; + /** Offset in the buffer where the UUID data starts. */ + public int start = 0; + /** Constructs a UuidHolder with isSet = 1. */ public UuidHolder() { this.isSet = 1; diff --git a/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java index 759c84651d..ccc0d3e176 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java @@ -37,7 +37,7 @@ import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.complex.writer.BaseWriter.ExtensionWriter; import org.apache.arrow.vector.extension.UuidType; -import org.apache.arrow.vector.holders.UuidHolder; +import org.apache.arrow.vector.holders.NullableUuidHolder; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; @@ -1071,14 +1071,14 @@ public void testCopyValueSafeForExtensionType() throws Exception { assertTrue(reader.isSet(), "first list shouldn't be null"); reader.next(); FieldReader uuidReader = reader.reader(); - UuidHolder holder = new UuidHolder(); + NullableUuidHolder holder = new NullableUuidHolder(); uuidReader.read(holder); - UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u1, actualUuid); reader.next(); uuidReader = reader.reader(); uuidReader.read(holder); - actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u2, actualUuid); // Verify second list @@ -1087,12 +1087,12 @@ public void testCopyValueSafeForExtensionType() throws Exception { reader.next(); uuidReader = reader.reader(); uuidReader.read(holder); - actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u3, actualUuid); reader.next(); uuidReader = reader.reader(); uuidReader.read(holder); - actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u4, actualUuid); reader.next(); uuidReader = reader.reader(); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index e96ac3027c..1fe4c59f63 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -40,8 +40,8 @@ import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.holders.DurationHolder; import org.apache.arrow.vector.holders.FixedSizeBinaryHolder; +import org.apache.arrow.vector.holders.NullableUuidHolder; import org.apache.arrow.vector.holders.TimeStampMilliTZHolder; -import org.apache.arrow.vector.holders.UuidHolder; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -1254,14 +1254,14 @@ public void testListVectorReaderForExtensionType() throws Exception { reader.setPosition(0); reader.next(); FieldReader uuidReader = reader.reader(); - UuidHolder holder = new UuidHolder(); + NullableUuidHolder holder = new NullableUuidHolder(); uuidReader.read(holder); - UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u1, actualUuid); reader.next(); uuidReader = reader.reader(); uuidReader.read(holder); - actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u2, actualUuid); } } @@ -1294,14 +1294,14 @@ public void testCopyFromForExtensionType() throws Exception { reader.setPosition(0); reader.next(); FieldReader uuidReader = reader.reader(); - UuidHolder holder = new UuidHolder(); + NullableUuidHolder holder = new NullableUuidHolder(); uuidReader.read(holder); - UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u1, actualUuid); reader.next(); uuidReader = reader.reader(); uuidReader.read(holder); - actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u2, actualUuid); } } @@ -1350,14 +1350,14 @@ public void testCopyValueSafeForExtensionType() throws Exception { assertTrue(reader.isSet(), "first list shouldn't be null"); reader.next(); FieldReader uuidReader = reader.reader(); - UuidHolder holder = new UuidHolder(); + NullableUuidHolder holder = new NullableUuidHolder(); uuidReader.read(holder); - UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u1, actualUuid); reader.next(); uuidReader = reader.reader(); uuidReader.read(holder); - actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u2, actualUuid); // Verify second list @@ -1366,12 +1366,12 @@ public void testCopyValueSafeForExtensionType() throws Exception { reader.next(); uuidReader = reader.reader(); uuidReader.read(holder); - actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u3, actualUuid); reader.next(); uuidReader = reader.reader(); uuidReader.read(holder); - actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u4, actualUuid); reader.next(); uuidReader = reader.reader(); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java index bfac1237a4..274d2973bd 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java @@ -42,7 +42,7 @@ import org.apache.arrow.vector.complex.writer.FieldWriter; import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.holders.FixedSizeBinaryHolder; -import org.apache.arrow.vector.holders.UuidHolder; +import org.apache.arrow.vector.holders.NullableUuidHolder; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; @@ -1299,14 +1299,14 @@ public void testMapVectorWithExtensionType() throws Exception { mapReader.setPosition(0); mapReader.next(); FieldReader uuidReader = mapReader.value(); - UuidHolder holder = new UuidHolder(); + NullableUuidHolder holder = new NullableUuidHolder(); uuidReader.read(holder); - UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u1, actualUuid); mapReader.next(); uuidReader = mapReader.value(); uuidReader.read(holder); - actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u2, actualUuid); } } @@ -1341,14 +1341,14 @@ public void testCopyFromForExtensionType() throws Exception { mapReader.setPosition(0); mapReader.next(); FieldReader uuidReader = mapReader.value(); - UuidHolder holder = new UuidHolder(); + NullableUuidHolder holder = new NullableUuidHolder(); uuidReader.read(holder); - UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u1, actualUuid); mapReader.next(); uuidReader = mapReader.value(); uuidReader.read(holder); - actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(u2, actualUuid); } } @@ -1626,9 +1626,9 @@ public void testMapWithUuidKeyAndListUuidValue() throws Exception { // Read first entry mapReader.next(); FieldReader keyReader = mapReader.key(); - UuidHolder keyHolder = new UuidHolder(); + NullableUuidHolder keyHolder = new NullableUuidHolder(); keyReader.read(keyHolder); - UUID actualKey = UuidUtility.uuidFromArrowBuf(keyHolder.buffer, 0); + UUID actualKey = UuidUtility.uuidFromArrowBuf(keyHolder.buffer, keyHolder.start); assertEquals(key1, actualKey); FieldReader valueReader = mapReader.value(); @@ -1648,7 +1648,7 @@ public void testMapWithUuidKeyAndListUuidValue() throws Exception { mapReader.next(); keyReader = mapReader.key(); keyReader.read(keyHolder); - actualKey = UuidUtility.uuidFromArrowBuf(keyHolder.buffer, 0); + actualKey = UuidUtility.uuidFromArrowBuf(keyHolder.buffer, keyHolder.start); assertEquals(key2, actualKey); valueReader = mapReader.value(); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestUuidType.java b/vector/src/test/java/org/apache/arrow/vector/TestUuidType.java index acf9dd6868..99045d1cba 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestUuidType.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestUuidType.java @@ -233,7 +233,8 @@ void testVectorByteArrayOperations() { // Verify the bytes match byte[] actualBytes = new byte[UuidType.UUID_BYTE_WIDTH]; - uuidVector.get(0).getBytes(0, actualBytes); + int offset = uuidVector.getStartOffset(0); + uuidVector.getDataBuffer().getBytes(offset, actualBytes); assertArrayEquals(uuidBytes, actualBytes); } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestUuidVector.java b/vector/src/test/java/org/apache/arrow/vector/TestUuidVector.java index a3690461cf..b5dd12d89c 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestUuidVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestUuidVector.java @@ -27,6 +27,7 @@ import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.complex.impl.NullableUuidHolderReaderImpl; import org.apache.arrow.vector.complex.impl.UuidReaderImpl; import org.apache.arrow.vector.complex.impl.UuidWriterImpl; import org.apache.arrow.vector.extension.UuidType; @@ -136,8 +137,8 @@ void testWriteExtensionWithUnsupportedType() throws Exception { IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> writer.writeExtension("invalid-type")); - assertEquals( - "Unsupported value type for UUID: class java.lang.String", exception.getMessage()); + assertTrue( + exception.getMessage().contains("Unsupported value type for UUID: java.lang.String")); } } @@ -235,9 +236,9 @@ void testReaderCopyAsValueExtensionVector() throws Exception { UuidReaderImpl reader = (UuidReaderImpl) vectorForRead.getReader(); reader.copyAsValue(writer); UuidReaderImpl reader2 = (UuidReaderImpl) vector.getReader(); - UuidHolder holder = new UuidHolder(); + NullableUuidHolder holder = new NullableUuidHolder(); reader2.read(0, holder); - UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(uuid, actualUuid); } } @@ -252,10 +253,10 @@ void testReaderReadWithUuidHolder() throws Exception { UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); reader.setPosition(0); - UuidHolder holder = new UuidHolder(); + NullableUuidHolder holder = new NullableUuidHolder(); reader.read(holder); - UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(uuid, actualUuid); assertEquals(1, holder.isSet); } @@ -274,7 +275,7 @@ void testReaderReadWithNullableUuidHolder() throws Exception { NullableUuidHolder holder = new NullableUuidHolder(); reader.read(holder); - UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(uuid, actualUuid); assertEquals(1, holder.isSet); } @@ -310,10 +311,10 @@ void testReaderReadWithArrayIndexUuidHolder() throws Exception { UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); - UuidHolder holder = new UuidHolder(); + NullableUuidHolder holder = new NullableUuidHolder(); reader.read(1, holder); - UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, 0); + UUID actualUuid = UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start); assertEquals(uuid2, actualUuid); assertEquals(1, holder.isSet); } @@ -334,7 +335,7 @@ void testReaderReadWithArrayIndexNullableUuidHolder() throws Exception { NullableUuidHolder holder1 = new NullableUuidHolder(); reader.read(0, holder1); - assertEquals(uuid1, UuidUtility.uuidFromArrowBuf(holder1.buffer, 0)); + assertEquals(uuid1, UuidUtility.uuidFromArrowBuf(holder1.buffer, holder1.start)); assertEquals(1, holder1.isSet); NullableUuidHolder holder2 = new NullableUuidHolder(); @@ -343,7 +344,7 @@ void testReaderReadWithArrayIndexNullableUuidHolder() throws Exception { NullableUuidHolder holder3 = new NullableUuidHolder(); reader.read(2, holder3); - assertEquals(uuid2, UuidUtility.uuidFromArrowBuf(holder3.buffer, 0)); + assertEquals(uuid2, UuidUtility.uuidFromArrowBuf(holder3.buffer, holder3.start)); assertEquals(1, holder3.isSet); } } @@ -374,31 +375,6 @@ public ArrowType type() { } } - @Test - void testReaderReadWithArrayIndexUnsupportedHolder() throws Exception { - try (UuidVector vector = new UuidVector("test", allocator)) { - UUID uuid = UUID.randomUUID(); - vector.setSafe(0, uuid); - vector.setValueCount(1); - - UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); - - // Create a mock unsupported holder - ExtensionHolder unsupportedHolder = - new ExtensionHolder() { - @Override - public ArrowType type() { - return null; - } - }; - - IllegalArgumentException exception = - assertThrows(IllegalArgumentException.class, () -> reader.read(0, unsupportedHolder)); - - assertTrue(exception.getMessage().contains("Unsupported holder type for UuidReader")); - } - } - @Test void testReaderIsSet() throws Exception { try (UuidVector vector = new UuidVector("test", allocator)) { @@ -461,4 +437,290 @@ void testReaderGetField() throws Exception { assertEquals("test", reader.getField().getName()); } } + + @Test + void testHolderStartOffsetWithMultipleValues() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid1 = UUID.randomUUID(); + UUID uuid2 = UUID.randomUUID(); + UUID uuid3 = UUID.randomUUID(); + + vector.setSafe(0, uuid1); + vector.setSafe(1, uuid2); + vector.setSafe(2, uuid3); + vector.setValueCount(3); + + // Test UuidHolder with different indices + NullableUuidHolder holder = new NullableUuidHolder(); + vector.get(0, holder); + assertEquals(0, holder.start); + assertEquals(uuid1, UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start)); + + vector.get(1, holder); + assertEquals(16, holder.start); // UUID_BYTE_WIDTH = 16 + assertEquals(uuid2, UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start)); + + vector.get(2, holder); + assertEquals(32, holder.start); // 2 * UUID_BYTE_WIDTH = 32 + assertEquals(uuid3, UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start)); + } + } + + @Test + void testNullableHolderStartOffsetWithMultipleValues() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid1 = UUID.randomUUID(); + UUID uuid2 = UUID.randomUUID(); + + vector.setSafe(0, uuid1); + vector.setNull(1); + vector.setSafe(2, uuid2); + vector.setValueCount(3); + + // Test NullableUuidHolder with different indices + NullableUuidHolder holder1 = new NullableUuidHolder(); + vector.get(0, holder1); + assertEquals(0, holder1.start); + assertEquals(1, holder1.isSet); + assertEquals(uuid1, UuidUtility.uuidFromArrowBuf(holder1.buffer, holder1.start)); + + NullableUuidHolder holder2 = new NullableUuidHolder(); + vector.get(1, holder2); + assertEquals(0, holder2.isSet); + + NullableUuidHolder holder3 = new NullableUuidHolder(); + vector.get(2, holder3); + assertEquals(32, holder3.start); // 2 * UUID_BYTE_WIDTH = 32 + assertEquals(1, holder3.isSet); + assertEquals(uuid2, UuidUtility.uuidFromArrowBuf(holder3.buffer, holder3.start)); + + // Verify all holders share the same buffer + assertEquals(holder1.buffer, holder3.buffer); + } + } + + @Test + void testSetFromHolderWithStartOffset() throws Exception { + try (UuidVector sourceVector = new UuidVector("source", allocator); + UuidVector targetVector = new UuidVector("target", allocator)) { + UUID uuid1 = UUID.randomUUID(); + UUID uuid2 = UUID.randomUUID(); + + sourceVector.setSafe(0, uuid1); + sourceVector.setSafe(1, uuid2); + sourceVector.setValueCount(3); + + // Get holder from index 1 (should have start = 16) + NullableUuidHolder holder = new NullableUuidHolder(); + sourceVector.get(1, holder); + assertEquals(16, holder.start); + + // Set target vector using holder with non-zero start offset + targetVector.setSafe(0, holder); + targetVector.setValueCount(1); + + // Verify the value was copied correctly + assertEquals(uuid2, targetVector.getObject(0)); + } + } + + @Test + void testSetFromNullableHolderWithStartOffset() throws Exception { + try (UuidVector sourceVector = new UuidVector("source", allocator); + UuidVector targetVector = new UuidVector("target", allocator)) { + UUID uuid1 = UUID.randomUUID(); + UUID uuid2 = UUID.randomUUID(); + + sourceVector.setSafe(0, uuid1); + sourceVector.setNull(1); + sourceVector.setSafe(2, uuid2); + sourceVector.setValueCount(3); + + // Get holder from index 2 (should have start = 32) + NullableUuidHolder holder = new NullableUuidHolder(); + sourceVector.get(2, holder); + assertEquals(32, holder.start); + assertEquals(1, holder.isSet); + + // Set target vector using holder with non-zero start offset + targetVector.setSafe(0, holder); + targetVector.setValueCount(1); + + // Verify the value was copied correctly + assertEquals(uuid2, targetVector.getObject(0)); + + // Test with null holder + NullableUuidHolder nullHolder = new NullableUuidHolder(); + sourceVector.get(1, nullHolder); + assertEquals(0, nullHolder.isSet); + + targetVector.setSafe(1, nullHolder); + targetVector.setValueCount(2); + assertTrue(targetVector.isNull(1)); + } + } + + @Test + void testGetStartOffset() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + vector.allocateNew(10); + + // Test getStartOffset for various indices + assertEquals(0, vector.getStartOffset(0)); + assertEquals(16, vector.getStartOffset(1)); + assertEquals(32, vector.getStartOffset(2)); + assertEquals(48, vector.getStartOffset(3)); + assertEquals(160, vector.getStartOffset(10)); + } + } + + @Test + void testReaderWithStartOffsetMultipleReads() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid1 = UUID.randomUUID(); + UUID uuid2 = UUID.randomUUID(); + UUID uuid3 = UUID.randomUUID(); + + vector.setSafe(0, uuid1); + vector.setSafe(1, uuid2); + vector.setSafe(2, uuid3); + vector.setValueCount(3); + + UuidReaderImpl reader = (UuidReaderImpl) vector.getReader(); + NullableUuidHolder holder = new NullableUuidHolder(); + + // Read from different positions and verify start offset + reader.read(0, holder); + assertEquals(0, holder.start); + assertEquals(uuid1, UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start)); + + reader.read(1, holder); + assertEquals(16, holder.start); + assertEquals(uuid2, UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start)); + + reader.read(2, holder); + assertEquals(32, holder.start); + assertEquals(uuid3, UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start)); + } + } + + @Test + void testWriterWithExtensionHolder() throws Exception { + try (UuidVector sourceVector = new UuidVector("source", allocator); + UuidVector targetVector = new UuidVector("target", allocator)) { + UUID uuid = UUID.randomUUID(); + sourceVector.setSafe(0, uuid); + sourceVector.setValueCount(1); + + // Get holder from source + NullableUuidHolder holder = new NullableUuidHolder(); + sourceVector.get(0, holder); + + // Write using UuidWriterImpl with ExtensionHolder + UuidWriterImpl writer = new UuidWriterImpl(targetVector); + writer.setPosition(0); + writer.writeExtension(holder); + + assertEquals(uuid, targetVector.getObject(0)); + } + } + + @Test + void testNullableUuidHolderReaderImpl() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid = UUID.randomUUID(); + vector.setSafe(0, uuid); + vector.setValueCount(1); + + // Get holder from vector + NullableUuidHolder sourceHolder = new NullableUuidHolder(); + vector.get(0, sourceHolder); + assertEquals(1, sourceHolder.isSet); + assertEquals(0, sourceHolder.start); + + // Create reader from holder + NullableUuidHolderReaderImpl reader = new NullableUuidHolderReaderImpl(sourceHolder); + assertTrue(reader.isSet()); + assertEquals(uuid, reader.readObject()); + + // Read into another holder + NullableUuidHolder targetHolder = new NullableUuidHolder(); + reader.read(targetHolder); + assertEquals(1, targetHolder.isSet); + assertEquals(0, targetHolder.start); + assertEquals(uuid, UuidUtility.uuidFromArrowBuf(targetHolder.buffer, targetHolder.start)); + } + } + + @Test + void testNullableUuidHolderReaderImplWithNull() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + vector.setNull(0); + vector.setValueCount(1); + + // Get null holder from vector + NullableUuidHolder sourceHolder = new NullableUuidHolder(); + vector.get(0, sourceHolder); + assertEquals(0, sourceHolder.isSet); + + // Create reader from null holder + NullableUuidHolderReaderImpl reader = new NullableUuidHolderReaderImpl(sourceHolder); + assertFalse(reader.isSet()); + assertNull(reader.readObject()); + + // Read into another holder + NullableUuidHolder targetHolder = new NullableUuidHolder(); + reader.read(targetHolder); + assertEquals(0, targetHolder.isSet); + } + } + + @Test + void testNullableUuidHolderReaderImplReadIntoUuidHolder() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid = UUID.randomUUID(); + vector.setSafe(0, uuid); + vector.setValueCount(1); + + // Get holder from vector + NullableUuidHolder sourceHolder = new NullableUuidHolder(); + vector.get(0, sourceHolder); + + // Create reader from holder + NullableUuidHolderReaderImpl reader = new NullableUuidHolderReaderImpl(sourceHolder); + + // Read into UuidHolder (non-nullable) + UuidHolder targetHolder = new UuidHolder(); + reader.read(targetHolder); + assertEquals(0, targetHolder.start); + assertEquals(uuid, UuidUtility.uuidFromArrowBuf(targetHolder.buffer, targetHolder.start)); + } + } + + @Test + void testNullableUuidHolderReaderImplWithNonZeroStart() throws Exception { + try (UuidVector vector = new UuidVector("test", allocator)) { + UUID uuid1 = UUID.randomUUID(); + UUID uuid2 = UUID.randomUUID(); + vector.setSafe(0, uuid1); + vector.setSafe(1, uuid2); + vector.setValueCount(2); + + // Get holder from index 1 (start = 16) + NullableUuidHolder sourceHolder = new NullableUuidHolder(); + vector.get(1, sourceHolder); + assertEquals(1, sourceHolder.isSet); + assertEquals(16, sourceHolder.start); + + // Create reader from holder + NullableUuidHolderReaderImpl reader = new NullableUuidHolderReaderImpl(sourceHolder); + assertEquals(uuid2, reader.readObject()); + + // Read into another holder and verify start is preserved + NullableUuidHolder targetHolder = new NullableUuidHolder(); + reader.read(targetHolder); + assertEquals(16, targetHolder.start); + assertEquals(uuid2, UuidUtility.uuidFromArrowBuf(targetHolder.buffer, targetHolder.start)); + } + } } diff --git a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index b131bf07e2..80d03cae6d 100644 --- a/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -1169,7 +1169,7 @@ public void simpleUnion() throws Exception { } else if (i % 5 == 4) { NullableUuidHolder holder = new NullableUuidHolder(); unionReader.read(holder); - assertEquals(UuidUtility.uuidFromArrowBuf(holder.buffer, 0), uuid); + assertEquals(UuidUtility.uuidFromArrowBuf(holder.buffer, holder.start), uuid); } else { assertEquals((float) i, unionReader.readFloat(), 1e-12); } @@ -2536,7 +2536,7 @@ public void extensionWriterReader() throws Exception { { FieldReader uuidReader = rootReader.reader("uuid1"); uuidReader.setPosition(0); - UuidHolder uuidHolder = new UuidHolder(); + NullableUuidHolder uuidHolder = new NullableUuidHolder(); uuidReader.read(uuidHolder); UUID actualUuid = UuidUtility.uuidFromArrowBuf(uuidHolder.buffer, 0); assertEquals(u1, actualUuid); From a1d83179cf6d3cce4660f6f0bf8e7f75867e87bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Tue, 20 Jan 2026 13:57:15 +0000 Subject: [PATCH 129/271] GH-929: Add UUID support in JDBC driver (#930) ## What's Changed This PR adds UUID support to the Arrow Flight SQL JDBC driver, enabling JDBC applications to work with UUID data types when connecting to Flight SQL servers that use Arrow's canonical `arrow.uuid` extension type. ### Key Implementation Details - Added `ArrowFlightJdbcUuidVectorAccessor` to handle reading UUID values from `UuidVector` - `getObject()` returns `java.util.UUID` directly - `getString()` returns the standard hyphenated UUID format (e.g., "550e8400-e29b-41d4-a716-446655440000") - `getBytes()` returns the 16-byte binary representation - Added `UuidAvaticaParameterConverter` to handle parameter binding for UUID columns - Supports binding `java.util.UUID` objects directly via `setObject()` - Supports binding UUID string representations via `setString()` - Supports binding 16-byte arrays via `setBytes()` - UUID extension type maps to `java.sql.Types.OTHER`. - Updated `SqlTypes` to recognize `UuidType` and return appropriate SQL type ID **Examples** ``` java try (Statement stmt = conn.createStatement(); ResultSet rs = stmt.executeQuery("SELECT id, session_id FROM sessions")) { while (rs.next()) { int id = rs.getInt("id"); // getObject() returns java.util.UUID directly UUID sessionId = rs.getObject("session_id", UUID.class); // getString() returns hyphenated format: "550e8400-e29b-41d4-a716-..." String sessionIdStr = rs.getString("session_id"); System.out.printf("ID: %d, UUID: %s%n", id, sessionId); } } // Use PreparedStatement to bind UUID parameters String sql = "SELECT * FROM sessions WHERE session_id = ?"; try (PreparedStatement pstmt = conn.prepareStatement(sql)) { UUID targetId = UUID.fromString("550e8400-e29b-41d4-a716-446655440000"); // Bind UUID directly with setObject() pstmt.setObject(1, targetId); // Or bind as string: pstmt.setString(1, targetId.toString()); try (ResultSet rs = pstmt.executeQuery()) { if (rs.next()) { System.out.println("Found: " + rs.getObject("session_id")); } } } ``` Closes #929. --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Joana Hrotko --- docs/source/jdbc.rst | 8 +- .../driver/jdbc/ArrowDatabaseMetadata.java | 5 + .../ArrowFlightJdbcAccessorFactory.java | 5 + .../ArrowFlightJdbcUuidVectorAccessor.java | 88 ++++++++ .../impl/UuidAvaticaParameterConverter.java | 103 ++++++++++ .../jdbc/utils/AvaticaParameterBinder.java | 14 ++ .../arrow/driver/jdbc/utils/ConvertUtils.java | 14 ++ .../arrow/driver/jdbc/utils/SqlTypes.java | 5 + .../arrow/driver/jdbc/ResultSetTest.java | 176 ++++++++++++++++ .../ArrowFlightJdbcAccessorFactoryTest.java | 13 ++ ...ArrowFlightJdbcUuidVectorAccessorTest.java | 188 ++++++++++++++++++ .../UuidAvaticaParameterConverterTest.java | 160 +++++++++++++++ .../jdbc/utils/CoreMockedSqlProducers.java | 122 ++++++++++++ .../utils/RootAllocatorTestExtension.java | 22 ++ .../arrow/driver/jdbc/utils/SqlTypesTest.java | 5 + 15 files changed, 927 insertions(+), 1 deletion(-) create mode 100644 flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/binary/ArrowFlightJdbcUuidVectorAccessor.java create mode 100644 flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/UuidAvaticaParameterConverter.java create mode 100644 flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/binary/ArrowFlightJdbcUuidVectorAccessorTest.java create mode 100644 flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/converter/impl/UuidAvaticaParameterConverterTest.java diff --git a/docs/source/jdbc.rst b/docs/source/jdbc.rst index c0477cb06d..a4c95dbf00 100644 --- a/docs/source/jdbc.rst +++ b/docs/source/jdbc.rst @@ -213,7 +213,8 @@ Type Mapping ------------ The Arrow to JDBC type mapping can be obtained at runtime via -a method on ColumnBinder. +a method on ColumnBinder. The Flight SQL JDBC driver follows the same +mapping, with additional support for the UUID extension type noted below. +----------------------------+----------------------------+-------+ | Arrow Type | JDBC Type | Notes | @@ -232,6 +233,8 @@ a method on ColumnBinder. +----------------------------+----------------------------+-------+ | FixedSizeBinary | BINARY (setBytes) | | +----------------------------+----------------------------+-------+ +| Uuid (extension) | OTHER (setObject) | \(3) | ++----------------------------+----------------------------+-------+ | Float32 | REAL (setFloat) | | +----------------------------+----------------------------+-------+ | Int8 | TINYINT (setByte) | | @@ -276,3 +279,6 @@ a method on ColumnBinder. `_, which will lead to the driver using the "default timezone" (that of the Java VM). +* \(3) For the Flight SQL JDBC driver, the Arrow UUID extension type + (``arrow.uuid``) maps to JDBC ``OTHER`` and is surfaced as + ``java.util.UUID`` values. diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java index 7185ddfe01..502270e1cd 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java @@ -75,10 +75,12 @@ import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.ipc.ReadChannel; import org.apache.arrow.vector.ipc.message.MessageSerializer; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ExtensionTypeRegistry; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.Text; @@ -164,6 +166,9 @@ public class ArrowDatabaseMetadata extends AvaticaDatabaseMetaData { LONGNVARCHAR, SqlSupportsConvert.SQL_CONVERT_LONGVARCHAR_VALUE); sqlTypesToFlightEnumConvertTypes.put(DATE, SqlSupportsConvert.SQL_CONVERT_DATE_VALUE); sqlTypesToFlightEnumConvertTypes.put(TIMESTAMP, SqlSupportsConvert.SQL_CONVERT_TIMESTAMP_VALUE); + + // Register the UUID extension type so it is always available for the driver + ExtensionTypeRegistry.register(UuidType.INSTANCE); } ArrowDatabaseMetadata(final AvaticaConnection connection) { diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactory.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactory.java index bbfe88a78a..8362eb7627 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactory.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactory.java @@ -19,6 +19,7 @@ import java.util.function.IntSupplier; import org.apache.arrow.driver.jdbc.accessor.impl.ArrowFlightJdbcNullVectorAccessor; import org.apache.arrow.driver.jdbc.accessor.impl.binary.ArrowFlightJdbcBinaryVectorAccessor; +import org.apache.arrow.driver.jdbc.accessor.impl.binary.ArrowFlightJdbcUuidVectorAccessor; import org.apache.arrow.driver.jdbc.accessor.impl.calendar.ArrowFlightJdbcDateVectorAccessor; import org.apache.arrow.driver.jdbc.accessor.impl.calendar.ArrowFlightJdbcDurationVectorAccessor; import org.apache.arrow.driver.jdbc.accessor.impl.calendar.ArrowFlightJdbcIntervalVectorAccessor; @@ -65,6 +66,7 @@ import org.apache.arrow.vector.UInt2Vector; import org.apache.arrow.vector.UInt4Vector; import org.apache.arrow.vector.UInt8Vector; +import org.apache.arrow.vector.UuidVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; @@ -138,6 +140,9 @@ public static ArrowFlightJdbcAccessor createAccessor( } else if (vector instanceof LargeVarBinaryVector) { return new ArrowFlightJdbcBinaryVectorAccessor( (LargeVarBinaryVector) vector, getCurrentRow, setCursorWasNull); + } else if (vector instanceof UuidVector) { + return new ArrowFlightJdbcUuidVectorAccessor( + (UuidVector) vector, getCurrentRow, setCursorWasNull); } else if (vector instanceof FixedSizeBinaryVector) { return new ArrowFlightJdbcBinaryVectorAccessor( (FixedSizeBinaryVector) vector, getCurrentRow, setCursorWasNull); diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/binary/ArrowFlightJdbcUuidVectorAccessor.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/binary/ArrowFlightJdbcUuidVectorAccessor.java new file mode 100644 index 0000000000..4bdbcbb63d --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/binary/ArrowFlightJdbcUuidVectorAccessor.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.accessor.impl.binary; + +import java.util.UUID; +import java.util.function.IntSupplier; +import org.apache.arrow.driver.jdbc.accessor.ArrowFlightJdbcAccessor; +import org.apache.arrow.driver.jdbc.accessor.ArrowFlightJdbcAccessorFactory; +import org.apache.arrow.vector.UuidVector; +import org.apache.arrow.vector.util.UuidUtility; + +/** + * Accessor for the Arrow UUID extension type ({@link UuidVector}). + * + *

This accessor provides JDBC-compatible access to UUID values stored in Arrow's canonical UUID + * extension type ('arrow.uuid'). It follows PostgreSQL JDBC driver conventions: + * + *

    + *
  • {@link #getObject()} returns {@link java.util.UUID} + *
  • {@link #getString()} returns the hyphenated string format (e.g., + * "550e8400-e29b-41d4-a716-446655440000") + *
  • {@link #getBytes()} returns the 16-byte binary representation + *
+ */ +public class ArrowFlightJdbcUuidVectorAccessor extends ArrowFlightJdbcAccessor { + + private final UuidVector vector; + + /** + * Creates a new accessor for a UUID vector. + * + * @param vector the UUID vector to access + * @param currentRowSupplier supplier for the current row index + * @param setCursorWasNull consumer to set the wasNull flag + */ + public ArrowFlightJdbcUuidVectorAccessor( + UuidVector vector, + IntSupplier currentRowSupplier, + ArrowFlightJdbcAccessorFactory.WasNullConsumer setCursorWasNull) { + super(currentRowSupplier, setCursorWasNull); + this.vector = vector; + } + + @Override + public Object getObject() { + UUID uuid = vector.getObject(getCurrentRow()); + this.wasNull = uuid == null; + this.wasNullConsumer.setWasNull(this.wasNull); + return uuid; + } + + @Override + public Class getObjectClass() { + return UUID.class; + } + + @Override + public String getString() { + UUID uuid = (UUID) getObject(); + if (uuid == null) { + return null; + } + return uuid.toString(); + } + + @Override + public byte[] getBytes() { + UUID uuid = (UUID) getObject(); + if (uuid == null) { + return null; + } + return UuidUtility.getBytesFromUUID(uuid); + } +} diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/UuidAvaticaParameterConverter.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/UuidAvaticaParameterConverter.java new file mode 100644 index 0000000000..b2157890cf --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/UuidAvaticaParameterConverter.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.converter.impl; + +import static org.apache.arrow.driver.jdbc.utils.SqlTypes.getSqlTypeIdFromArrowType; +import static org.apache.arrow.driver.jdbc.utils.SqlTypes.getSqlTypeNameFromArrowType; + +import java.nio.ByteBuffer; +import java.util.UUID; +import org.apache.arrow.driver.jdbc.converter.AvaticaParameterConverter; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.UuidVector; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.util.UuidUtility; +import org.apache.calcite.avatica.AvaticaParameter; +import org.apache.calcite.avatica.remote.TypedValue; +import org.apache.calcite.avatica.util.ByteString; + +/** + * AvaticaParameterConverter for UUID Arrow extension type. + * + *

Handles conversion of UUID values from JDBC parameters to Arrow's UUID extension type. Accepts + * both {@link UUID} objects and String representations of UUIDs. + */ +public class UuidAvaticaParameterConverter implements AvaticaParameterConverter { + + public UuidAvaticaParameterConverter() {} + + @Override + public boolean bindParameter(FieldVector vector, TypedValue typedValue, int index) { + if (!(vector instanceof UuidVector)) { + return false; + } + + UuidVector uuidVector = (UuidVector) vector; + Object value = typedValue.toJdbc(null); + + if (value == null) { + uuidVector.setNull(index); + return true; + } + + UUID uuid; + if (value instanceof UUID) { + uuid = (UUID) value; + } else if (value instanceof String) { + uuid = UUID.fromString((String) value); + } else if (value instanceof byte[]) { + byte[] bytes = (byte[]) value; + if (bytes.length != 16) { + throw new IllegalArgumentException("UUID byte array must be 16 bytes, got " + bytes.length); + } + uuid = uuidFromBytes(bytes); + } else if (value instanceof ByteString) { + byte[] bytes = ((ByteString) value).getBytes(); + if (bytes.length != 16) { + throw new IllegalArgumentException("UUID byte array must be 16 bytes, got " + bytes.length); + } + uuid = uuidFromBytes(bytes); + } else { + throw new IllegalArgumentException( + "Cannot convert " + value.getClass().getName() + " to UUID"); + } + + uuidVector.setSafe(index, UuidUtility.getBytesFromUUID(uuid)); + return true; + } + + @Override + public AvaticaParameter createParameter(Field field) { + final String name = field.getName(); + final int jdbcType = getSqlTypeIdFromArrowType(field.getType()); + final String typeName = getSqlTypeNameFromArrowType(field.getType()); + final String className = UUID.class.getCanonicalName(); + return new AvaticaParameter(false, 0, 0, jdbcType, typeName, className, name); + } + + private static UUID uuidFromBytes(byte[] bytes) { + final long mostSignificantBits; + final long leastSignificantBits; + ByteBuffer bb = ByteBuffer.wrap(bytes); + // Reads the first eight bytes + mostSignificantBits = bb.getLong(); + // Reads the first eight bytes at this buffer's current + leastSignificantBits = bb.getLong(); + + return new UUID(mostSignificantBits, leastSignificantBits); + } +} diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java index 8c98ee4077..8f40d6698e 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java @@ -41,10 +41,14 @@ import org.apache.arrow.driver.jdbc.converter.impl.UnionAvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.Utf8AvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.Utf8ViewAvaticaParameterConverter; +import org.apache.arrow.driver.jdbc.converter.impl.UuidAvaticaParameterConverter; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeVisitor; +import org.apache.arrow.vector.types.pojo.ArrowType.ExtensionType; import org.apache.calcite.avatica.remote.TypedValue; import org.checkerframework.checker.nullness.qual.Nullable; @@ -290,5 +294,15 @@ public Boolean visit(ArrowType.RunEndEncoded type) { throw new UnsupportedOperationException( "No Avatica parameter binder implemented for type " + type); } + + @Override + public Boolean visit(ExtensionType type) { + if (type instanceof UuidType) { + return new UuidAvaticaParameterConverter().bindParameter(vector, typedValue, index); + } + + // fallback to default implementation + return ArrowTypeVisitor.super.visit(type); + } } } diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java index 5dd4c69c73..dd51ee5361 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java @@ -43,8 +43,12 @@ import org.apache.arrow.driver.jdbc.converter.impl.UnionAvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.Utf8AvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.Utf8ViewAvaticaParameterConverter; +import org.apache.arrow.driver.jdbc.converter.impl.UuidAvaticaParameterConverter; import org.apache.arrow.flight.sql.FlightSqlColumnMetadata; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeVisitor; +import org.apache.arrow.vector.types.pojo.ArrowType.ExtensionType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.calcite.avatica.AvaticaParameter; import org.apache.calcite.avatica.ColumnMetaData; @@ -294,5 +298,15 @@ public AvaticaParameter visit(ArrowType.RunEndEncoded type) { throw new UnsupportedOperationException( "No Avatica parameter binder implemented for type " + type); } + + @Override + public AvaticaParameter visit(ExtensionType type) { + if (type instanceof UuidType) { + return new UuidAvaticaParameterConverter().createParameter(field); + } + + // fallback to default implementation + return ArrowTypeVisitor.super.visit(type); + } } } diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/SqlTypes.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/SqlTypes.java index 5ba3957f8b..7982d5bc73 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/SqlTypes.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/SqlTypes.java @@ -20,11 +20,13 @@ import java.sql.Types; import java.util.HashMap; import java.util.Map; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.types.pojo.ArrowType; /** SQL Types utility functions. */ public class SqlTypes { + private static final Map typeIdToName = new HashMap<>(); static { @@ -110,6 +112,9 @@ public static int getSqlTypeIdFromArrowType(ArrowType arrowType) { case BinaryView: return Types.VARBINARY; case FixedSizeBinary: + if (arrowType instanceof UuidType) { + return Types.OTHER; + } return Types.BINARY; case LargeBinary: return Types.LONGVARBINARY; diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java index 569b5495fe..3a5a39be3d 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java @@ -22,8 +22,10 @@ import static org.hamcrest.CoreMatchers.allOf; import static org.hamcrest.CoreMatchers.anyOf; import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.*; @@ -31,7 +33,9 @@ import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.PreparedStatement; import java.sql.ResultSet; +import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.SQLTimeoutException; import java.sql.Statement; @@ -42,6 +46,7 @@ import java.util.List; import java.util.Random; import java.util.Set; +import java.util.UUID; import java.util.concurrent.CountDownLatch; import org.apache.arrow.driver.jdbc.utils.CoreMockedSqlProducers; import org.apache.arrow.driver.jdbc.utils.FallbackFlightSqlProducer; @@ -61,6 +66,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.arrow.vector.util.UuidUtility; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -795,4 +801,174 @@ public void testResultSetAppMetadata() throws Exception { "foo".getBytes(StandardCharsets.UTF_8)); } } + + @Test + public void testSelectQueryWithUuidColumn() throws SQLException { + // Expectations + final int expectedRowCount = 4; + final UUID[] expectedUuids = + new UUID[] { + CoreMockedSqlProducers.UUID_1, + CoreMockedSqlProducers.UUID_2, + CoreMockedSqlProducers.UUID_3, + null + }; + + final Integer[] expectedIds = new Integer[] {1, 2, 3, 4}; + + final List actualUuids = new ArrayList<>(expectedRowCount); + final List actualIds = new ArrayList<>(expectedRowCount); + + // Query + int actualRowCount = 0; + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(CoreMockedSqlProducers.UUID_SQL_CMD)) { + for (; resultSet.next(); actualRowCount++) { + actualIds.add((Integer) resultSet.getObject("id")); + actualUuids.add((UUID) resultSet.getObject("uuid_col")); + } + } + + // Assertions + int finalActualRowCount = actualRowCount; + assertAll( + "UUID ResultSet values are as expected", + () -> assertThat(finalActualRowCount, is(equalTo(expectedRowCount))), + () -> assertThat(actualIds.toArray(new Integer[0]), is(expectedIds)), + () -> assertThat(actualUuids.toArray(new UUID[0]), is(expectedUuids))); + } + + @Test + public void testGetObjectReturnsUuid() throws SQLException { + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(CoreMockedSqlProducers.UUID_SQL_CMD)) { + resultSet.next(); + Object result = resultSet.getObject("uuid_col"); + assertThat(result, instanceOf(UUID.class)); + assertThat(result, is(CoreMockedSqlProducers.UUID_1)); + } + } + + @Test + public void testGetObjectByIndexReturnsUuid() throws SQLException { + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(CoreMockedSqlProducers.UUID_SQL_CMD)) { + resultSet.next(); + Object result = resultSet.getObject(2); + assertThat(result, instanceOf(UUID.class)); + assertThat(result, is(CoreMockedSqlProducers.UUID_1)); + } + } + + @Test + public void testGetStringReturnsHyphenatedFormat() throws SQLException { + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(CoreMockedSqlProducers.UUID_SQL_CMD)) { + resultSet.next(); + String result = resultSet.getString("uuid_col"); + assertThat(result, is(CoreMockedSqlProducers.UUID_1.toString())); + } + } + + @Test + public void testGetBytesReturns16ByteArray() throws SQLException { + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(CoreMockedSqlProducers.UUID_SQL_CMD)) { + resultSet.next(); + byte[] result = resultSet.getBytes("uuid_col"); + assertThat(result.length, is(16)); + assertThat(result, is(UuidUtility.getBytesFromUUID(CoreMockedSqlProducers.UUID_1))); + } + } + + @Test + public void testNullUuidHandling() throws SQLException { + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(CoreMockedSqlProducers.UUID_SQL_CMD)) { + // Skip to row 4 which has NULL UUID + resultSet.next(); // row 1 + resultSet.next(); // row 2 + resultSet.next(); // row 3 + resultSet.next(); // row 4 (NULL UUID) + + Object objResult = resultSet.getObject("uuid_col"); + assertThat(objResult, nullValue()); + assertThat(resultSet.wasNull(), is(true)); + + String strResult = resultSet.getString("uuid_col"); + assertThat(strResult, nullValue()); + assertThat(resultSet.wasNull(), is(true)); + + byte[] bytesResult = resultSet.getBytes("uuid_col"); + assertThat(bytesResult, nullValue()); + assertThat(resultSet.wasNull(), is(true)); + } + } + + @Test + public void testMultipleUuidRows() throws SQLException { + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(CoreMockedSqlProducers.UUID_SQL_CMD)) { + resultSet.next(); + assertThat(resultSet.getObject("uuid_col"), is(CoreMockedSqlProducers.UUID_1)); + + resultSet.next(); + assertThat(resultSet.getObject("uuid_col"), is(CoreMockedSqlProducers.UUID_2)); + + resultSet.next(); + assertThat(resultSet.getObject("uuid_col"), is(CoreMockedSqlProducers.UUID_3)); + + resultSet.next(); + assertThat(resultSet.getObject("uuid_col"), nullValue()); + } + } + + @Test + public void testUuidExtensionTypeInSchema() throws SQLException { + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(CoreMockedSqlProducers.UUID_SQL_CMD)) { + ResultSetMetaData metaData = resultSet.getMetaData(); + + assertThat(metaData.getColumnCount(), is(2)); + assertThat(metaData.getColumnName(1), is("id")); + assertThat(metaData.getColumnName(2), is("uuid_col")); + + assertThat(metaData.getColumnType(2), is(java.sql.Types.OTHER)); + } + } + + @Test + public void testPreparedStatementWithUuidParameter() throws SQLException { + try (PreparedStatement pstmt = + connection.prepareStatement(CoreMockedSqlProducers.UUID_PREPARED_SELECT_SQL_CMD)) { + pstmt.setObject(1, CoreMockedSqlProducers.UUID_1); + try (ResultSet rs = pstmt.executeQuery()) { + rs.next(); + assertThat(rs.getObject("uuid_col"), is(CoreMockedSqlProducers.UUID_1)); + } + } + } + + @Test + public void testPreparedStatementWithUuidStringParameter() throws SQLException { + try (PreparedStatement pstmt = + connection.prepareStatement(CoreMockedSqlProducers.UUID_PREPARED_SELECT_SQL_CMD)) { + pstmt.setString(1, CoreMockedSqlProducers.UUID_1.toString()); + try (ResultSet rs = pstmt.executeQuery()) { + rs.next(); + assertThat(rs.getObject("uuid_col"), is(CoreMockedSqlProducers.UUID_1)); + } + } + } + + @Test + public void testPreparedStatementUpdateWithUuid() throws SQLException { + try (PreparedStatement pstmt = + connection.prepareStatement(CoreMockedSqlProducers.UUID_PREPARED_UPDATE_SQL_CMD)) { + pstmt.setObject(1, CoreMockedSqlProducers.UUID_3); + pstmt.setInt(2, 1); + int updated = pstmt.executeUpdate(); + assertThat(updated, is(1)); + } + } } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactoryTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactoryTest.java index 8b39041f0c..1fbd2f86a9 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactoryTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorFactoryTest.java @@ -16,10 +16,12 @@ */ package org.apache.arrow.driver.jdbc.accessor; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.function.IntSupplier; import org.apache.arrow.driver.jdbc.accessor.impl.binary.ArrowFlightJdbcBinaryVectorAccessor; +import org.apache.arrow.driver.jdbc.accessor.impl.binary.ArrowFlightJdbcUuidVectorAccessor; import org.apache.arrow.driver.jdbc.accessor.impl.calendar.ArrowFlightJdbcDateVectorAccessor; import org.apache.arrow.driver.jdbc.accessor.impl.calendar.ArrowFlightJdbcDurationVectorAccessor; import org.apache.arrow.driver.jdbc.accessor.impl.calendar.ArrowFlightJdbcIntervalVectorAccessor; @@ -497,4 +499,15 @@ public void createAccessorForMapVector() { assertTrue(accessor instanceof ArrowFlightJdbcMapVectorAccessor); } } + + @Test + public void createAccessorForUuidVector() { + try (ValueVector valueVector = rootAllocatorTestExtension.createUuidVector()) { + ArrowFlightJdbcAccessor accessor = + ArrowFlightJdbcAccessorFactory.createAccessor( + valueVector, GET_CURRENT_ROW, (boolean wasNull) -> {}); + + assertInstanceOf(ArrowFlightJdbcUuidVectorAccessor.class, accessor); + } + } } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/binary/ArrowFlightJdbcUuidVectorAccessorTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/binary/ArrowFlightJdbcUuidVectorAccessorTest.java new file mode 100644 index 0000000000..b7f341240c --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/binary/ArrowFlightJdbcUuidVectorAccessorTest.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.accessor.impl.binary; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.MatcherAssert.assertThat; + +import java.util.UUID; +import org.apache.arrow.driver.jdbc.accessor.ArrowFlightJdbcAccessorFactory; +import org.apache.arrow.driver.jdbc.utils.RootAllocatorTestExtension; +import org.apache.arrow.vector.UuidVector; +import org.apache.arrow.vector.util.UuidUtility; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * Tests for {@link ArrowFlightJdbcUuidVectorAccessor}. + * + *

Verifies that the accessor correctly handles UUID values from Arrow's UUID extension type, + * following PostgreSQL JDBC driver conventions. + */ +public class ArrowFlightJdbcUuidVectorAccessorTest { + + @RegisterExtension + public static RootAllocatorTestExtension rootAllocatorTestExtension = + new RootAllocatorTestExtension(); + + private static final UUID UUID_1 = UUID.fromString("550e8400-e29b-41d4-a716-446655440000"); + private static final UUID UUID_2 = UUID.fromString("6ba7b810-9dad-11d1-80b4-00c04fd430c8"); + private static final UUID UUID_3 = UUID.fromString("f47ac10b-58cc-4372-a567-0e02b2c3d479"); + + private UuidVector vector; + private ArrowFlightJdbcUuidVectorAccessor accessor; + private boolean wasNullCalled; + private boolean wasNullValue; + + @BeforeEach + public void setUp() { + vector = rootAllocatorTestExtension.createUuidVector(); + wasNullCalled = false; + wasNullValue = false; + ArrowFlightJdbcAccessorFactory.WasNullConsumer wasNullConsumer = + (wasNull) -> { + wasNullCalled = true; + wasNullValue = wasNull; + }; + accessor = new ArrowFlightJdbcUuidVectorAccessor(vector, () -> 0, wasNullConsumer); + } + + @AfterEach + public void tearDown() { + vector.close(); + } + + @Test + public void testGetObjectReturnsUuid() { + accessor = new ArrowFlightJdbcUuidVectorAccessor(vector, () -> 0, (wasNull) -> {}); + Object result = accessor.getObject(); + assertThat(result, is(UUID_1)); + assertThat(accessor.wasNull(), is(false)); + } + + @Test + public void testGetObjectReturnsCorrectUuidForEachRow() { + accessor = new ArrowFlightJdbcUuidVectorAccessor(vector, () -> 0, (wasNull) -> {}); + assertThat(accessor.getObject(), is(UUID_1)); + + accessor = new ArrowFlightJdbcUuidVectorAccessor(vector, () -> 1, (wasNull) -> {}); + assertThat(accessor.getObject(), is(UUID_2)); + + accessor = new ArrowFlightJdbcUuidVectorAccessor(vector, () -> 2, (wasNull) -> {}); + assertThat(accessor.getObject(), is(UUID_3)); + } + + @Test + public void testGetObjectReturnsNullForNullValue() { + vector.reset(); + vector.allocateNew(1); + vector.setNull(0); + vector.setValueCount(1); + + accessor = new ArrowFlightJdbcUuidVectorAccessor(vector, () -> 0, (wasNull) -> {}); + Object result = accessor.getObject(); + assertThat(result, nullValue()); + assertThat(accessor.wasNull(), is(true)); + } + + @Test + public void testGetObjectClassReturnsUuidClass() { + assertThat(accessor.getObjectClass(), equalTo(UUID.class)); + } + + @Test + public void testGetStringReturnsHyphenatedFormat() { + accessor = new ArrowFlightJdbcUuidVectorAccessor(vector, () -> 0, (wasNull) -> {}); + String result = accessor.getString(); + assertThat(result, is("550e8400-e29b-41d4-a716-446655440000")); + assertThat(accessor.wasNull(), is(false)); + } + + @Test + public void testGetStringReturnsNullForNullValue() { + vector.reset(); + vector.allocateNew(1); + vector.setNull(0); + vector.setValueCount(1); + + accessor = new ArrowFlightJdbcUuidVectorAccessor(vector, () -> 0, (wasNull) -> {}); + String result = accessor.getString(); + assertThat(result, nullValue()); + assertThat(accessor.wasNull(), is(true)); + } + + @Test + public void testGetBytesReturns16ByteArray() { + accessor = new ArrowFlightJdbcUuidVectorAccessor(vector, () -> 0, (wasNull) -> {}); + byte[] result = accessor.getBytes(); + assertThat(result.length, is(16)); + assertThat(result, is(UuidUtility.getBytesFromUUID(UUID_1))); + assertThat(accessor.wasNull(), is(false)); + } + + @Test + public void testGetBytesReturnsNullForNullValue() { + vector.reset(); + vector.allocateNew(1); + vector.setNull(0); + vector.setValueCount(1); + + accessor = new ArrowFlightJdbcUuidVectorAccessor(vector, () -> 0, (wasNull) -> {}); + byte[] result = accessor.getBytes(); + assertThat(result, nullValue()); + assertThat(accessor.wasNull(), is(true)); + } + + @Test + public void testWasNullConsumerIsCalled() { + accessor = + new ArrowFlightJdbcUuidVectorAccessor( + vector, + () -> 0, + (wasNull) -> { + wasNullCalled = true; + wasNullValue = wasNull; + }); + accessor.getObject(); + assertThat(wasNullCalled, is(true)); + assertThat(wasNullValue, is(false)); + } + + @Test + public void testWasNullConsumerIsCalledWithTrueForNull() { + vector.reset(); + vector.allocateNew(1); + vector.setNull(0); + vector.setValueCount(1); + + accessor = + new ArrowFlightJdbcUuidVectorAccessor( + vector, + () -> 0, + (wasNull) -> { + wasNullCalled = true; + wasNullValue = wasNull; + }); + accessor.getObject(); + assertThat(wasNullCalled, is(true)); + assertThat(wasNullValue, is(true)); + } +} diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/converter/impl/UuidAvaticaParameterConverterTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/converter/impl/UuidAvaticaParameterConverterTest.java new file mode 100644 index 0000000000..07751f0abc --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/converter/impl/UuidAvaticaParameterConverterTest.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.converter.impl; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.sql.Types; +import java.util.UUID; +import org.apache.arrow.driver.jdbc.utils.RootAllocatorTestExtension; +import org.apache.arrow.vector.UuidVector; +import org.apache.arrow.vector.extension.UuidType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.UuidUtility; +import org.apache.calcite.avatica.AvaticaParameter; +import org.apache.calcite.avatica.ColumnMetaData; +import org.apache.calcite.avatica.remote.TypedValue; +import org.apache.calcite.avatica.util.ByteString; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * Tests for {@link UuidAvaticaParameterConverter}. + * + *

Verifies that the converter correctly handles UUID parameter binding from JDBC to Arrow's UUID + * extension type. + */ +public class UuidAvaticaParameterConverterTest { + + @RegisterExtension + public static RootAllocatorTestExtension rootAllocatorTestExtension = + new RootAllocatorTestExtension(); + + private static final UUID TEST_UUID = UUID.fromString("550e8400-e29b-41d4-a716-446655440000"); + + private UuidVector vector; + private UuidAvaticaParameterConverter converter; + + @BeforeEach + public void setUp() { + vector = new UuidVector("uuid_param", rootAllocatorTestExtension.getRootAllocator()); + vector.allocateNew(5); + converter = new UuidAvaticaParameterConverter(); + } + + @AfterEach + public void tearDown() { + vector.close(); + } + + @Test + public void testBindParameterWithUuidObject() { + TypedValue typedValue = TypedValue.ofLocal(ColumnMetaData.Rep.OBJECT, TEST_UUID); + + boolean result = converter.bindParameter(vector, typedValue, 0); + + assertTrue(result); + assertThat(vector.getObject(0), is(TEST_UUID)); + } + + @Test + public void testBindParameterWithUuidString() { + String uuidString = "550e8400-e29b-41d4-a716-446655440000"; + TypedValue typedValue = TypedValue.ofLocal(ColumnMetaData.Rep.STRING, uuidString); + + boolean result = converter.bindParameter(vector, typedValue, 0); + + assertTrue(result); + assertThat(vector.getObject(0), is(TEST_UUID)); + } + + @Test + public void testBindParameterWithByteArray() { + byte[] uuidBytes = UuidUtility.getBytesFromUUID(TEST_UUID); + ByteString byteString = new ByteString(uuidBytes); + TypedValue typedValue = TypedValue.ofLocal(ColumnMetaData.Rep.BYTE_STRING, byteString); + + boolean result = converter.bindParameter(vector, typedValue, 0); + + assertTrue(result); + assertThat(vector.getObject(0), is(TEST_UUID)); + } + + @Test + public void testBindParameterWithNullValue() { + TypedValue typedValue = TypedValue.ofLocal(ColumnMetaData.Rep.OBJECT, null); + + boolean result = converter.bindParameter(vector, typedValue, 0); + + assertTrue(result); + assertTrue(vector.isNull(0)); + assertThat(vector.getObject(0), nullValue()); + } + + @Test + public void testBindParameterWithInvalidByteArrayLength() { + byte[] invalidBytes = new byte[8]; // Should be 16 bytes + ByteString byteString = new ByteString(invalidBytes); + TypedValue typedValue = TypedValue.ofLocal(ColumnMetaData.Rep.BYTE_STRING, byteString); + + assertThrows( + IllegalArgumentException.class, () -> converter.bindParameter(vector, typedValue, 0)); + } + + @Test + public void testBindParameterWithInvalidType() { + TypedValue typedValue = TypedValue.ofLocal(ColumnMetaData.Rep.INTEGER, 12345); + + assertThrows( + IllegalArgumentException.class, () -> converter.bindParameter(vector, typedValue, 0)); + } + + @Test + public void testBindParameterMultipleValues() { + UUID uuid1 = UUID.fromString("550e8400-e29b-41d4-a716-446655440000"); + UUID uuid2 = UUID.fromString("6ba7b810-9dad-11d1-80b4-00c04fd430c8"); + UUID uuid3 = UUID.fromString("f47ac10b-58cc-4372-a567-0e02b2c3d479"); + + converter.bindParameter(vector, TypedValue.ofLocal(ColumnMetaData.Rep.OBJECT, uuid1), 0); + converter.bindParameter(vector, TypedValue.ofLocal(ColumnMetaData.Rep.OBJECT, uuid2), 1); + converter.bindParameter(vector, TypedValue.ofLocal(ColumnMetaData.Rep.OBJECT, uuid3), 2); + + assertThat(vector.getObject(0), is(uuid1)); + assertThat(vector.getObject(1), is(uuid2)); + assertThat(vector.getObject(2), is(uuid3)); + } + + @Test + public void testCreateParameter() { + Field uuidField = new Field("uuid_col", new FieldType(true, UuidType.INSTANCE, null), null); + + AvaticaParameter parameter = converter.createParameter(uuidField); + + assertThat(parameter.name, is("uuid_col")); + assertThat(parameter.parameterType, is(Types.OTHER)); + assertThat(parameter.typeName, is("OTHER")); + assertThat(parameter.className, equalTo(UUID.class.getCanonicalName())); + } +} diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/CoreMockedSqlProducers.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/CoreMockedSqlProducers.java index 8197d7d95f..7c17755693 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/CoreMockedSqlProducers.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/CoreMockedSqlProducers.java @@ -28,8 +28,10 @@ import java.sql.SQLException; import java.sql.Timestamp; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.UUID; import java.util.function.Consumer; import java.util.stream.IntStream; import org.apache.arrow.flight.FlightProducer.ServerStreamListener; @@ -40,10 +42,13 @@ import org.apache.arrow.vector.DateDayVector; import org.apache.arrow.vector.Float4Vector; import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.TimeStampMilliVector; import org.apache.arrow.vector.UInt4Vector; +import org.apache.arrow.vector.UuidVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.types.DateUnit; import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.types.TimeUnit; @@ -52,6 +57,7 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.Text; +import org.apache.arrow.vector.util.UuidUtility; /** Standard {@link MockFlightSqlProducer} instances for tests. */ // TODO Remove this once all tests are refactor to use only the queries they need. @@ -62,6 +68,22 @@ public final class CoreMockedSqlProducers { public static final String LEGACY_CANCELLATION_SQL_CMD = "SELECT * FROM TAKES_FOREVER"; public static final String LEGACY_REGULAR_WITH_EMPTY_SQL_CMD = "SELECT * FROM TEST_EMPTIES"; + public static final String UUID_SQL_CMD = "SELECT * FROM UUID_TABLE"; + public static final String UUID_PREPARED_SELECT_SQL_CMD = + "SELECT * FROM UUID_TABLE WHERE uuid_col = ?"; + public static final String UUID_PREPARED_UPDATE_SQL_CMD = + "UPDATE UUID_TABLE SET uuid_col = ? WHERE id = ?"; + + public static final UUID UUID_1 = UUID.fromString("550e8400-e29b-41d4-a716-446655440000"); + public static final UUID UUID_2 = UUID.fromString("6ba7b810-9dad-11d1-80b4-00c04fd430c8"); + public static final UUID UUID_3 = UUID.fromString("f47ac10b-58cc-4372-a567-0e02b2c3d479"); + + public static final Schema UUID_SCHEMA = + new Schema( + ImmutableList.of( + new Field("id", new FieldType(true, new ArrowType.Int(32, true), null), null), + new Field("uuid_col", new FieldType(true, UuidType.INSTANCE, null), null))); + private CoreMockedSqlProducers() { // Prevent instantiation. } @@ -78,9 +100,109 @@ public static MockFlightSqlProducer getLegacyProducer() { addLegacyMetadataSqlCmdSupport(producer); addLegacyCancellationSqlCmdSupport(producer); addQueryWithEmbeddedEmptyRoot(producer); + addUuidSqlCmdSupport(producer); + addUuidPreparedSelectSqlCmdSupport(producer); + addUuidPreparedUpdateSqlCmdSupport(producer); return producer; } + /** + * Gets a {@link MockFlightSqlProducer} configured with UUID test data. + * + * @return a new producer with UUID support. + */ + public static MockFlightSqlProducer getUuidProducer() { + final MockFlightSqlProducer producer = new MockFlightSqlProducer(); + addUuidSqlCmdSupport(producer); + return producer; + } + + private static void addUuidPreparedUpdateSqlCmdSupport(final MockFlightSqlProducer producer) { + final String query = "UPDATE UUID_TABLE SET uuid_col = ? WHERE id = ?"; + final Schema parameterSchema = + new Schema( + Arrays.asList( + new Field("", new FieldType(true, UuidType.INSTANCE, null), null), + Field.nullable("", new ArrowType.Int(32, true)))); + + producer.addUpdateQuery(query, 1); + producer.addExpectedParameters( + UUID_PREPARED_UPDATE_SQL_CMD, + parameterSchema, + Collections.singletonList(Arrays.asList(CoreMockedSqlProducers.UUID_3, 1))); + } + + private static void addUuidPreparedSelectSqlCmdSupport(final MockFlightSqlProducer producer) { + final Schema parameterSchema = + new Schema( + Collections.singletonList( + new Field("", new FieldType(true, UuidType.INSTANCE, null), null))); + + final Consumer uuidResultProvider = + listener -> { + try (final BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + final VectorSchemaRoot root = VectorSchemaRoot.create(UUID_SCHEMA, allocator)) { + root.allocateNew(); + IntVector idVector = (IntVector) root.getVector("id"); + UuidVector uuidVector = (UuidVector) root.getVector("uuid_col"); + idVector.setSafe(0, 1); + uuidVector.setSafe(0, UuidUtility.getBytesFromUUID(CoreMockedSqlProducers.UUID_1)); + root.setRowCount(1); + listener.start(root); + listener.putNext(); + } catch (final Throwable throwable) { + listener.error(throwable); + } finally { + listener.completed(); + } + }; + + producer.addSelectQuery( + UUID_PREPARED_SELECT_SQL_CMD, UUID_SCHEMA, Collections.singletonList(uuidResultProvider)); + producer.addExpectedParameters( + UUID_PREPARED_SELECT_SQL_CMD, + parameterSchema, + Collections.singletonList(Collections.singletonList(CoreMockedSqlProducers.UUID_1))); + } + + private static void addUuidSqlCmdSupport(final MockFlightSqlProducer producer) { + final Consumer uuidResultProvider = + listener -> { + try (final BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + final VectorSchemaRoot root = VectorSchemaRoot.create(UUID_SCHEMA, allocator)) { + root.allocateNew(); + + IntVector idVector = (IntVector) root.getVector("id"); + UuidVector uuidVector = (UuidVector) root.getVector("uuid_col"); + + // Row 0: id=1, uuid=UUID_1 + idVector.setSafe(0, 1); + uuidVector.setSafe(0, UuidUtility.getBytesFromUUID(UUID_1)); + + // Row 1: id=2, uuid=UUID_2 + idVector.setSafe(1, 2); + uuidVector.setSafe(1, UuidUtility.getBytesFromUUID(UUID_2)); + + // Row 2: id=3, uuid=UUID_3 + idVector.setSafe(2, 3); + uuidVector.setSafe(2, UuidUtility.getBytesFromUUID(UUID_3)); + + // Row 3: id=4, uuid=NULL + idVector.setSafe(3, 4); + uuidVector.setNull(3); + + root.setRowCount(4); + listener.start(root); + listener.putNext(); + } finally { + listener.completed(); + } + }; + + producer.addSelectQuery( + UUID_SQL_CMD, UUID_SCHEMA, Collections.singletonList(uuidResultProvider)); + } + private static void addQueryWithEmbeddedEmptyRoot(final MockFlightSqlProducer producer) { final Schema querySchema = new Schema( diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/RootAllocatorTestExtension.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/RootAllocatorTestExtension.java index 347e92a16c..4b299d63e0 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/RootAllocatorTestExtension.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/RootAllocatorTestExtension.java @@ -19,6 +19,7 @@ import java.math.BigDecimal; import java.nio.charset.StandardCharsets; import java.util.Random; +import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; import org.apache.arrow.memory.BufferAllocator; @@ -53,6 +54,7 @@ import org.apache.arrow.vector.UInt2Vector; import org.apache.arrow.vector.UInt4Vector; import org.apache.arrow.vector.UInt8Vector; +import org.apache.arrow.vector.UuidVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.complex.FixedSizeListVector; import org.apache.arrow.vector.complex.LargeListVector; @@ -60,6 +62,7 @@ import org.apache.arrow.vector.complex.impl.UnionFixedSizeListWriter; import org.apache.arrow.vector.complex.impl.UnionLargeListWriter; import org.apache.arrow.vector.complex.impl.UnionListWriter; +import org.apache.arrow.vector.util.UuidUtility; import org.junit.jupiter.api.extension.AfterAllCallback; import org.junit.jupiter.api.extension.BeforeAllCallback; import org.junit.jupiter.api.extension.ExtensionContext; @@ -811,4 +814,23 @@ public FixedSizeListVector createFixedSizeListVector() { return valueVector; } + + /** + * Create a UuidVector to be used in the accessor tests. + * + * @return UuidVector + */ + public UuidVector createUuidVector() { + UuidVector valueVector = new UuidVector("", this.getRootAllocator()); + valueVector.allocateNew(3); + valueVector.setSafe( + 0, UuidUtility.getBytesFromUUID(UUID.fromString("550e8400-e29b-41d4-a716-446655440000"))); + valueVector.setSafe( + 1, UuidUtility.getBytesFromUUID(UUID.fromString("6ba7b810-9dad-11d1-80b4-00c04fd430c8"))); + valueVector.setSafe( + 2, UuidUtility.getBytesFromUUID(UUID.fromString("f47ac10b-58cc-4372-a567-0e02b2c3d479"))); + valueVector.setValueCount(3); + + return valueVector; + } } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/SqlTypesTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/SqlTypesTest.java index d69c549296..c4858d787d 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/SqlTypesTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/SqlTypesTest.java @@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import java.sql.Types; +import org.apache.arrow.vector.extension.UuidType; import org.apache.arrow.vector.types.DateUnit; import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.types.IntervalUnit; @@ -85,6 +86,8 @@ public void testGetSqlTypeIdFromArrowType() { assertEquals(Types.JAVA_OBJECT, getSqlTypeIdFromArrowType(new ArrowType.Map(true))); assertEquals(Types.NULL, getSqlTypeIdFromArrowType(new ArrowType.Null())); + + assertEquals(Types.OTHER, getSqlTypeIdFromArrowType(UuidType.INSTANCE)); } @Test @@ -140,5 +143,7 @@ public void testGetSqlTypeNameFromArrowType() { assertEquals("JAVA_OBJECT", getSqlTypeNameFromArrowType(new ArrowType.Map(true))); assertEquals("NULL", getSqlTypeNameFromArrowType(new ArrowType.Null())); + + assertEquals("OTHER", getSqlTypeNameFromArrowType(UuidType.INSTANCE)); } } From 7e61d462c094ff9eb3a692176b040a08f81654fe Mon Sep 17 00:00:00 2001 From: Pedro Matias Date: Thu, 22 Jan 2026 14:19:32 +0000 Subject: [PATCH 130/271] GH-932: [JDBC] Fix memory leak on Connection#close due to unclosed Statement(s) (#933) ## What's Changed Closing a Connection when there was one or more ResultSet that matched the following 2 conditions 1. hadn't been fully consumed 2. was obtained via a Statement instance of this Connection instance would generate exceptions due to memory leaks. Now, closing a Connection will first close all the Statement instances obtained via that Connection, which has a side effect of closing all the ResultSet, and then proceed with the old closing logic. This side effect is guaranteed by the JDBC Spec 4.3, chapter 13.1.4 The old closing logic was also slightly refactored to: 1. remove duplicate calls to ArrowFlightSqlClientHandler.close() 5. make sure that any exception generated during Connection.close() would be wrapped in a SQLException. Closes #932. --- .../driver/jdbc/ArrowFlightConnection.java | 39 ++++++++++++++----- .../arrow/driver/jdbc/ConnectionTest.java | 38 ++++++++++++++++++ 2 files changed, 68 insertions(+), 9 deletions(-) diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java index f6f17770f1..f81233ec33 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java @@ -20,6 +20,7 @@ import io.netty.util.concurrent.DefaultThreadFactory; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -180,19 +181,39 @@ public Properties getClientInfo() { @Override public void close() throws SQLException { - clientHandler.close(); - if (executorService != null) { - executorService.shutdown(); + Exception topLevelException = null; + try { + if (executorService != null) { + executorService.shutdown(); + } + } catch (final Exception e) { + topLevelException = e; + } + ArrayList closeables = new ArrayList<>(statementMap.values()); + closeables.add(clientHandler); + closeables.addAll(allocator.getChildAllocators()); + closeables.add(allocator); + try { + AutoCloseables.close(closeables); + } catch (final Exception e) { + if (topLevelException == null) { + topLevelException = e; + } else { + topLevelException.addSuppressed(e); + } } - try { - AutoCloseables.close(clientHandler); - allocator.getChildAllocators().forEach(AutoCloseables::closeNoChecked); - AutoCloseables.close(allocator); - super.close(); } catch (final Exception e) { - throw AvaticaConnection.HELPER.createException(e.getMessage(), e); + if (topLevelException == null) { + topLevelException = e; + } else { + topLevelException.addSuppressed(e); + } + } + if (topLevelException != null) { + throw AvaticaConnection.HELPER.createException( + topLevelException.getMessage(), topLevelException); } } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java index 46762f3319..dbedbe9d36 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java @@ -17,6 +17,7 @@ package org.apache.arrow.driver.jdbc; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -27,6 +28,7 @@ import java.sql.Driver; import java.sql.DriverManager; import java.sql.SQLException; +import java.sql.Statement; import java.util.Map; import java.util.Properties; import org.apache.arrow.driver.jdbc.authentication.UserPasswordAuthentication; @@ -660,4 +662,40 @@ public String visit(String value) { assertEquals(catalog, actualCatalog); } } + + @Test + public void testStatementsClosedOnConnectionClose() throws Exception { + // create a connection + final Properties properties = new Properties(); + properties.put(ArrowFlightConnectionProperty.HOST.camelName(), "localhost"); + properties.put( + ArrowFlightConnectionProperty.PORT.camelName(), FLIGHT_SERVER_TEST_EXTENSION.getPort()); + properties.put(ArrowFlightConnectionProperty.USER.camelName(), userTest); + properties.put(ArrowFlightConnectionProperty.PASSWORD.camelName(), passTest); + properties.put("useEncryption", false); + + Connection connection = + DriverManager.getConnection( + "jdbc:arrow-flight-sql://" + + FLIGHT_SERVER_TEST_EXTENSION.getHost() + + ":" + + FLIGHT_SERVER_TEST_EXTENSION.getPort(), + properties); + + // create some statements + int numStatements = 3; + Statement[] statements = new Statement[numStatements]; + for (int i = 0; i < numStatements; i++) { + statements[i] = connection.createStatement(); + assertFalse(statements[i].isClosed()); + } + + // close the connection + connection.close(); + + // assert the statements are closed + for (int i = 0; i < numStatements; i++) { + assertTrue(statements[i].isClosed()); + } + } } From 44c49baf6c2fdfcf20c8611f45c627c9438b2adb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Thu, 22 Jan 2026 14:19:58 +0000 Subject: [PATCH 131/271] GH-952: Add OAuth support (#953) ## What's Changed - Add OAuth 2.0 support to the Flight SQL JDBC driver, including client credentials and token exchange flows - Integrate OAuth token acquisition into connection setup, wiring tokens through OAuthCredentialWriter and updating gRPC credential handling to fail fast on writer errors. - Document new OAuth connection properties and add example clients for both OAuth flows. - Connection Properties Added - oauth.flow - oauth.tokenUri - oauth.clientId - oauth.clientSecret - oauth.scope - oauth.resource - oauth.exchange.subjectToken - oauth.exchange.subjectTokenType - oauth.exchange.actorToken - oauth.exchange.actorTokenType - oauth.exchange.aud - oauth.exchange.requestedTokenType - Connection config now recognizes oauth.* and oauth.exchange.* properties and builds OAuth providers when oauth.flow is specified. - Adds com.nimbusds:oauth2-oidc-sdk dependency and mockwebserver for tests. Closes #952. --- docs/source/flight_sql_jdbc_driver.rst | 123 +++++ .../flight/grpc/CallCredentialAdapter.java | 12 +- flight/flight-sql-jdbc-core/pom.xml | 32 ++ .../driver/jdbc/ArrowFlightConnection.java | 1 + .../client/ArrowFlightSqlClientHandler.java | 23 +- .../oauth/AbstractOAuthTokenProvider.java | 108 ++++ .../oauth/ClientCredentialsTokenProvider.java | 58 +++ .../jdbc/client/oauth/OAuthConfiguration.java | 240 +++++++++ .../client/oauth/OAuthCredentialWriter.java | 42 ++ .../client/oauth/OAuthTokenException.java | 31 ++ .../jdbc/client/oauth/OAuthTokenProvider.java | 33 ++ .../client/oauth/OAuthTokenProviders.java | 419 ++++++++++++++++ .../oauth/TokenExchangeTokenProvider.java | 81 +++ .../driver/jdbc/client/oauth/TokenInfo.java | 45 ++ .../ArrowFlightConnectionConfigImpl.java | 52 ++ .../driver/jdbc/OAuthIntegrationTest.java | 474 ++++++++++++++++++ .../client/oauth/OAuthConfigurationTest.java | 296 +++++++++++ .../oauth/OAuthCredentialWriterTest.java | 95 ++++ .../src/shade/LICENSE.txt | 8 + .../driver/jdbc/ITDriverJarValidation.java | 4 + 20 files changed, 2173 insertions(+), 4 deletions(-) create mode 100644 flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/AbstractOAuthTokenProvider.java create mode 100644 flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/ClientCredentialsTokenProvider.java create mode 100644 flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthConfiguration.java create mode 100644 flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthCredentialWriter.java create mode 100644 flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthTokenException.java create mode 100644 flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthTokenProvider.java create mode 100644 flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthTokenProviders.java create mode 100644 flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/TokenExchangeTokenProvider.java create mode 100644 flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/TokenInfo.java create mode 100644 flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/OAuthIntegrationTest.java create mode 100644 flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthConfigurationTest.java create mode 100644 flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthCredentialWriterTest.java diff --git a/docs/source/flight_sql_jdbc_driver.rst b/docs/source/flight_sql_jdbc_driver.rst index 1806930943..4deb726b33 100644 --- a/docs/source/flight_sql_jdbc_driver.rst +++ b/docs/source/flight_sql_jdbc_driver.rst @@ -173,3 +173,126 @@ DriverManager#getConnection() `_, the username and password supplied on the URI supercede the username and password arguments to the function call. + +OAuth 2.0 Authentication +======================== + +The driver supports OAuth 2.0 authentication for obtaining access tokens +from an authorization server. Two OAuth flows are currently supported: + +* **Client Credentials** - For service-to-service authentication where no + user interaction is required. The application authenticates using its own + credentials (client ID and client secret). + +* **Token Exchange** (RFC 8693) - For exchanging one token for another, + commonly used for federated authentication, delegation, or impersonation + scenarios. + +OAuth Connection Properties +--------------------------- + +The following properties configure OAuth authentication. These properties +should be provided via the ``Properties`` object when connecting, as they +may contain special characters that are difficult to encode in a URI. + +**Common OAuth Properties** + +.. list-table:: + :header-rows: 1 + + * - Parameter + - Type + - Required + - Default + - Description + + * - oauth.flow + - String + - Yes (to enable OAuth) + - null + - The OAuth grant type. Supported values: ``client_credentials``, + ``token_exchange`` + + * - oauth.tokenUri + - String + - Yes + - null + - The OAuth 2.0 token endpoint URL (e.g., + ``https://auth.example.com/oauth/token``) + + * - oauth.clientId + - String + - Conditional + - null + - The OAuth 2.0 client ID. Required for ``client_credentials`` flow, + optional for ``token_exchange`` + + * - oauth.clientSecret + - String + - Conditional + - null + - The OAuth 2.0 client secret. Required for ``client_credentials`` flow, + optional for ``token_exchange`` + + * - oauth.scope + - String + - No + - null + - Space-separated list of OAuth scopes to request + + * - oauth.resource + - String + - No + - null + - The resource indicator for the token request (RFC 8707) + +**Token Exchange Properties** + +These properties are specific to the ``token_exchange`` flow: + +.. list-table:: + :header-rows: 1 + + * - Parameter + - Type + - Required + - Default + - Description + + * - oauth.exchange.subjectToken + - String + - Yes + - null + - The subject token to exchange (e.g., a JWT from an identity provider) + + * - oauth.exchange.subjectTokenType + - String + - Yes + - null + - The token type URI of the subject token. Common values: + ``urn:ietf:params:oauth:token-type:access_token``, + ``urn:ietf:params:oauth:token-type:jwt`` + + * - oauth.exchange.actorToken + - String + - No + - null + - The actor token for delegation/impersonation scenarios + + * - oauth.exchange.actorTokenType + - String + - No + - null + - The token type URI of the actor token + + * - oauth.exchange.aud + - String + - No + - null + - The target audience for the exchanged token + + * - oauth.exchange.requestedTokenType + - String + - No + - null + - The desired token type for the exchanged token diff --git a/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/CallCredentialAdapter.java b/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/CallCredentialAdapter.java index f33e9b2f94..fe81f3fb23 100644 --- a/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/CallCredentialAdapter.java +++ b/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/CallCredentialAdapter.java @@ -18,6 +18,7 @@ import io.grpc.CallCredentials; import io.grpc.Metadata; +import io.grpc.Status; import java.util.concurrent.Executor; import java.util.function.Consumer; import org.apache.arrow.flight.CallHeaders; @@ -36,9 +37,14 @@ public void applyRequestMetadata( RequestInfo requestInfo, Executor executor, MetadataApplier metadataApplier) { executor.execute( () -> { - final Metadata headers = new Metadata(); - credentialWriter.accept(new MetadataAdapter(headers)); - metadataApplier.apply(headers); + try { + final Metadata headers = new Metadata(); + credentialWriter.accept(new MetadataAdapter(headers)); + metadataApplier.apply(headers); + } catch (Throwable t) { + metadataApplier.fail( + Status.UNAUTHENTICATED.withCause(t).withDescription(t.getMessage())); + } }); } diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index bb191ca9ed..da00baf32a 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -120,6 +120,31 @@ under the License. test + + com.squareup.okhttp3 + mockwebserver3 + 5.3.2 + test + + + com.squareup.okhttp3 + mockwebserver3-junit5 + 5.3.2 + test + + + com.squareup.okhttp3 + okhttp-jvm + 5.3.2 + test + + + com.squareup.okio + okio-jvm + 3.16.4 + test + + io.netty netty-common @@ -153,6 +178,13 @@ under the License. caffeine 3.2.3 + + + com.nimbusds + oauth2-oidc-sdk + 11.20.1 + + diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java index f81233ec33..0e9c198f52 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java @@ -122,6 +122,7 @@ private static ArrowFlightSqlClientHandler createNewClientHandler( .withClientCache(config.useClientCache() ? new FlightClientCache() : null) .withConnectTimeout(config.getConnectTimeout()) .withDriverVersion(driverVersion) + .withOAuthConfiguration(config.getOauthConfiguration()) .build(); } catch (final SQLException e) { try { diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java index 666996cd95..f0ea284239 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java @@ -32,6 +32,9 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import org.apache.arrow.driver.jdbc.client.oauth.OAuthConfiguration; +import org.apache.arrow.driver.jdbc.client.oauth.OAuthCredentialWriter; +import org.apache.arrow.driver.jdbc.client.oauth.OAuthTokenProvider; import org.apache.arrow.driver.jdbc.client.utils.ClientAuthenticationUtils; import org.apache.arrow.driver.jdbc.client.utils.FlightClientCache; import org.apache.arrow.driver.jdbc.client.utils.FlightLocationQueue; @@ -675,6 +678,8 @@ public static final class Builder { @VisibleForTesting @Nullable Duration connectTimeout; + @VisibleForTesting @Nullable OAuthConfiguration oauthConfig; + // These two middleware are for internal use within build() and should not be // exposed by builder // APIs. @@ -714,6 +719,7 @@ public Builder() {} this.clientKeyPath = original.clientKeyPath; this.allocator = original.allocator; this.catalog = original.catalog; + this.oauthConfig = original.oauthConfig; if (original.retainCookies) { this.cookieFactory = original.cookieFactory; @@ -983,6 +989,17 @@ public Builder withDriverVersion(DriverVersion driverVersion) { return this; } + /** + * Sets the OAuth configuration for this handler. + * + * @param oauthConfig the OAuth configuration + * @return this builder instance + */ + public Builder withOAuthConfiguration(final OAuthConfiguration oauthConfig) { + this.oauthConfig = oauthConfig; + return this; + } + public String getCacheKey() { return getLocation().toString(); } @@ -1070,7 +1087,11 @@ public ArrowFlightSqlClientHandler build() throws SQLException { FlightGrpcUtils.createFlightClient( allocator, channelBuilder.build(), clientBuilder.middleware()); final ArrayList credentialOptions = new ArrayList<>(); - if (isUsingUserPasswordAuth) { + // Authentication priority: OAuth > token > username/password + if (oauthConfig != null) { + OAuthTokenProvider tokenProvider = oauthConfig.createTokenProvider(); + credentialOptions.add(new CredentialCallOption(new OAuthCredentialWriter(tokenProvider))); + } else if (isUsingUserPasswordAuth) { // If the authFactory has already been used for a handshake, use the existing // token. // This can occur if the authFactory is being re-used for a new connection diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/AbstractOAuthTokenProvider.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/AbstractOAuthTokenProvider.java new file mode 100644 index 0000000000..9c377a5850 --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/AbstractOAuthTokenProvider.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.client.oauth; + +import com.nimbusds.oauth2.sdk.ParseException; +import com.nimbusds.oauth2.sdk.Scope; +import com.nimbusds.oauth2.sdk.TokenErrorResponse; +import com.nimbusds.oauth2.sdk.TokenRequest; +import com.nimbusds.oauth2.sdk.TokenResponse; +import com.nimbusds.oauth2.sdk.auth.ClientAuthentication; +import com.nimbusds.oauth2.sdk.token.AccessToken; +import java.io.IOException; +import java.net.URI; +import java.sql.SQLException; +import java.time.Instant; +import org.apache.arrow.util.VisibleForTesting; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** + * Abstract base class for OAuth token providers that handles token caching, refresh logic, and + * common request/response handling. + */ +public abstract class AbstractOAuthTokenProvider implements OAuthTokenProvider { + protected static final int EXPIRATION_BUFFER_SECONDS = 30; + protected static final int DEFAULT_EXPIRATION_SECONDS = 3600; + + private final Object tokenLock = new Object(); + private volatile @Nullable TokenInfo cachedToken; + + @VisibleForTesting URI tokenUri; + + @VisibleForTesting @Nullable ClientAuthentication clientAuth; + + @VisibleForTesting @Nullable Scope scope; + + @Override + public String getValidToken() throws SQLException { + TokenInfo token = cachedToken; + if (token != null && !token.isExpired(EXPIRATION_BUFFER_SECONDS)) { + return token.getAccessToken(); + } + + synchronized (tokenLock) { + token = cachedToken; + if (token != null && !token.isExpired(EXPIRATION_BUFFER_SECONDS)) { + return token.getAccessToken(); + } + cachedToken = fetchNewToken(); + return cachedToken.getAccessToken(); + } + } + + /** + * Fetches a new token from the authorization server. This method handles the common + * request/response logic while delegating flow-specific request building to subclasses. + * + * @return the new token information + * @throws SQLException if token cannot be obtained + */ + protected TokenInfo fetchNewToken() throws SQLException { + try { + TokenRequest request = buildTokenRequest(); + TokenResponse response = TokenResponse.parse(request.toHTTPRequest().send()); + + if (!response.indicatesSuccess()) { + TokenErrorResponse errorResponse = response.toErrorResponse(); + String errorMsg = + String.format( + "OAuth request failed: %s - %s", + errorResponse.getErrorObject().getCode(), + errorResponse.getErrorObject().getDescription()); + throw new SQLException(errorMsg); + } + + AccessToken accessToken = response.toSuccessResponse().getTokens().getAccessToken(); + long expiresIn = + accessToken.getLifetime() > 0 ? accessToken.getLifetime() : DEFAULT_EXPIRATION_SECONDS; + Instant expiresAt = Instant.now().plusSeconds(expiresIn); + + return new TokenInfo(accessToken.getValue(), expiresAt); + } catch (ParseException e) { + throw new SQLException("Failed to parse OAuth token response", e); + } catch (IOException e) { + throw new SQLException("Failed to send OAuth token request", e); + } + } + + /** + * Builds the flow-specific token request. + * + * @return the token request to send to the authorization server + */ + protected abstract TokenRequest buildTokenRequest(); +} diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/ClientCredentialsTokenProvider.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/ClientCredentialsTokenProvider.java new file mode 100644 index 0000000000..7e6289819c --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/ClientCredentialsTokenProvider.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.client.oauth; + +import com.nimbusds.oauth2.sdk.ClientCredentialsGrant; +import com.nimbusds.oauth2.sdk.Scope; +import com.nimbusds.oauth2.sdk.TokenRequest; +import com.nimbusds.oauth2.sdk.auth.ClientSecretBasic; +import com.nimbusds.oauth2.sdk.auth.Secret; +import com.nimbusds.oauth2.sdk.id.ClientID; +import java.net.URI; +import java.util.Objects; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** + * OAuth 2.0 Client Credentials flow token provider (RFC 6749 Section 4.4). + * + *

This provider handles service-to-service authentication where no user interaction is required. + * Tokens are cached and automatically refreshed before expiration. + */ +public class ClientCredentialsTokenProvider extends AbstractOAuthTokenProvider { + + /** + * Creates a new ClientCredentialsTokenProvider. + * + * @param tokenUri the OAuth token endpoint URI + * @param clientId the OAuth client ID + * @param clientSecret the OAuth client secret + * @param scope optional OAuth scopes (space-separated) + */ + ClientCredentialsTokenProvider( + URI tokenUri, String clientId, String clientSecret, @Nullable String scope) { + this.tokenUri = Objects.requireNonNull(tokenUri, "tokenUri cannot be null"); + Objects.requireNonNull(clientId, "clientId cannot be null"); + Objects.requireNonNull(clientSecret, "clientSecret cannot be null"); + this.clientAuth = new ClientSecretBasic(new ClientID(clientId), new Secret(clientSecret)); + this.scope = (scope != null && !scope.isEmpty()) ? Scope.parse(scope) : null; + } + + @Override + protected TokenRequest buildTokenRequest() { + return new TokenRequest(tokenUri, clientAuth, new ClientCredentialsGrant(), scope); + } +} diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthConfiguration.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthConfiguration.java new file mode 100644 index 0000000000..cba9d4c2e6 --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthConfiguration.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.client.oauth; + +import com.nimbusds.oauth2.sdk.GrantType; +import java.net.URI; +import java.net.URISyntaxException; +import java.sql.SQLException; +import java.util.Locale; +import java.util.Objects; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** Configuration class for OAuth settings parsed from connection properties. */ +public class OAuthConfiguration { + + private final GrantType grantType; + private final URI tokenUri; + private final @Nullable String clientId; + private final @Nullable String clientSecret; + private final @Nullable String scope; + private final @Nullable String subjectToken; + private final @Nullable String subjectTokenType; + private final @Nullable String actorToken; + private final @Nullable String actorTokenType; + private final @Nullable String audience; + private final @Nullable String resource; + private final @Nullable String requestedTokenType; + + private OAuthConfiguration(Builder builder) throws SQLException { + this.grantType = builder.grantType; + this.tokenUri = builder.tokenUri; + this.clientId = builder.clientId; + this.clientSecret = builder.clientSecret; + this.scope = builder.scope; + this.subjectToken = builder.subjectToken; + this.subjectTokenType = builder.subjectTokenType; + this.actorToken = builder.actorToken; + this.actorTokenType = builder.actorTokenType; + this.audience = builder.audience; + this.resource = builder.resource; + this.requestedTokenType = builder.requestedTokenType; + + validate(); + } + + private void validate() throws SQLException { + Objects.requireNonNull(grantType, "OAuth grant type is required"); + Objects.requireNonNull(tokenUri, "Token URI is required"); + + if (GrantType.CLIENT_CREDENTIALS.equals(grantType)) { + if (clientId == null || clientId.isEmpty()) { + throw new SQLException("clientId is required for client_credentials flow"); + } + if (clientSecret == null || clientSecret.isEmpty()) { + throw new SQLException("clientSecret is required for client_credentials flow"); + } + } else if (GrantType.TOKEN_EXCHANGE.equals(grantType)) { + if (subjectToken == null || subjectToken.isEmpty()) { + throw new SQLException("subjectToken is required for token_exchange flow"); + } + if (subjectTokenType == null || subjectTokenType.isEmpty()) { + throw new SQLException("subjectTokenType is required for token_exchange flow"); + } + } else { + throw new SQLException("Unsupported OAuth grant type: " + grantType); + } + } + + /** + * Creates an OAuthTokenProvider based on the configured grant type. + * + * @return the token provider + * @throws SQLException if the grant type is not supported or configuration is invalid + */ + public OAuthTokenProvider createTokenProvider() throws SQLException { + if (GrantType.CLIENT_CREDENTIALS.equals(grantType)) { + return OAuthTokenProviders.clientCredentials() + .tokenUri(tokenUri) + .clientId(clientId) + .clientSecret(clientSecret) + .scope(scope) + .build(); + } else if (GrantType.TOKEN_EXCHANGE.equals(grantType)) { + OAuthTokenProviders.TokenExchangeBuilder builder = + OAuthTokenProviders.tokenExchange() + .tokenUri(tokenUri) + .subjectToken(subjectToken) + .subjectTokenType(subjectTokenType) + .actorToken(actorToken) + .actorTokenType(actorTokenType) + .audience(audience) + .requestedTokenType(requestedTokenType) + .scope(scope) + .resource(resource); + + if (clientId != null && clientSecret != null) { + builder.clientCredentials(clientId, clientSecret); + } + + return builder.build(); + } else { + throw new SQLException("Unsupported OAuth grant type: " + grantType); + } + } + + /** Builder for OAuthConfiguration. */ + public static class Builder { + private GrantType grantType; + private URI tokenUri; + private @Nullable String clientId; + private @Nullable String clientSecret; + private @Nullable String scope; + private @Nullable String subjectToken; + private @Nullable String subjectTokenType; + private @Nullable String actorToken; + private @Nullable String actorTokenType; + private @Nullable String audience; + private @Nullable String resource; + private @Nullable String requestedTokenType; + + /** + * Sets the OAuth grant type from a string value. + * + *

Accepts either user-friendly names ("client_credentials", "token_exchange") or the full + * URN format as defined in RFC 6749 and RFC 8693. + * + * @param flowStr the flow type string (e.g., "client_credentials", "token_exchange") + * @return this builder + * @throws SQLException if the flow string is invalid + */ + public Builder flow(String flowStr) throws SQLException { + if (flowStr == null || flowStr.isEmpty()) { + throw new SQLException("OAuth flow cannot be null or empty"); + } + try { + String normalized = flowStr.toLowerCase(Locale.ROOT); + // Map user-friendly names to URN format for token_exchange + if ("token_exchange".equals(normalized)) { + normalized = GrantType.TOKEN_EXCHANGE.getValue(); + } + GrantType parsed = GrantType.parse(normalized); + if (!parsed.equals(GrantType.CLIENT_CREDENTIALS) + && !parsed.equals(GrantType.TOKEN_EXCHANGE)) { + throw new SQLException("Unsupported OAuth flow: " + flowStr); + } + this.grantType = parsed; + } catch (com.nimbusds.oauth2.sdk.ParseException e) { + throw new SQLException("Invalid OAuth flow: " + flowStr, e); + } + return this; + } + + /** + * Sets the token URI. + * + * @param tokenUri the OAuth token endpoint URI + * @return this builder + * @throws SQLException if the URI is invalid + */ + public Builder tokenUri(String tokenUri) throws SQLException { + if (tokenUri == null || tokenUri.isEmpty()) { + throw new SQLException("Token URI cannot be null or empty"); + } + try { + this.tokenUri = new URI(tokenUri); + } catch (URISyntaxException e) { + throw new SQLException("Invalid token URI: " + tokenUri, e); + } + return this; + } + + public Builder clientId(@Nullable String clientId) { + this.clientId = clientId; + return this; + } + + public Builder clientSecret(@Nullable String clientSecret) { + this.clientSecret = clientSecret; + return this; + } + + public Builder scope(@Nullable String scope) { + this.scope = scope; + return this; + } + + public Builder subjectToken(@Nullable String subjectToken) { + this.subjectToken = subjectToken; + return this; + } + + public Builder subjectTokenType(@Nullable String subjectTokenType) { + this.subjectTokenType = subjectTokenType; + return this; + } + + public Builder actorToken(@Nullable String actorToken) { + this.actorToken = actorToken; + return this; + } + + public Builder actorTokenType(@Nullable String actorTokenType) { + this.actorTokenType = actorTokenType; + return this; + } + + public Builder audience(@Nullable String audience) { + this.audience = audience; + return this; + } + + public Builder resource(@Nullable String resource) { + this.resource = resource; + return this; + } + + public Builder requestedTokenType(@Nullable String requestedTokenType) { + this.requestedTokenType = requestedTokenType; + return this; + } + + public OAuthConfiguration build() throws SQLException { + return new OAuthConfiguration(this); + } + } +} diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthCredentialWriter.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthCredentialWriter.java new file mode 100644 index 0000000000..0d4ad4689f --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthCredentialWriter.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.client.oauth; + +import java.sql.SQLException; +import java.util.Objects; +import java.util.function.Consumer; +import org.apache.arrow.flight.CallHeaders; +import org.apache.arrow.flight.auth2.Auth2Constants; + +/** Writes OAuth bearer tokens to Flight call headers. */ +public class OAuthCredentialWriter implements Consumer { + private final OAuthTokenProvider tokenProvider; + + public OAuthCredentialWriter(OAuthTokenProvider tokenProvider) { + this.tokenProvider = Objects.requireNonNull(tokenProvider, "tokenProvider cannot be null"); + } + + @Override + public void accept(CallHeaders headers) { + try { + String token = tokenProvider.getValidToken(); + headers.insert(Auth2Constants.AUTHORIZATION_HEADER, Auth2Constants.BEARER_PREFIX + token); + } catch (SQLException e) { + throw new OAuthTokenException("Failed to obtain OAuth token", e); + } + } +} diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthTokenException.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthTokenException.java new file mode 100644 index 0000000000..aceadb327b --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthTokenException.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.client.oauth; + +/** + * Runtime exception thrown when OAuth token operations fail. Used to wrap checked exceptions in + * contexts that don't allow them. + */ +public class OAuthTokenException extends RuntimeException { + public OAuthTokenException(String message) { + super(message); + } + + public OAuthTokenException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthTokenProvider.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthTokenProvider.java new file mode 100644 index 0000000000..241611e432 --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthTokenProvider.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.client.oauth; + +import java.sql.SQLException; + +/** + * Interface for OAuth token providers that handle token acquisition and refresh. Implementations + * should cache tokens and automatically refresh them before expiration. + */ +public interface OAuthTokenProvider { + /** + * Gets a valid OAuth access token, refreshing if necessary. + * + * @return a valid access token string + * @throws SQLException if token cannot be obtained + */ + String getValidToken() throws SQLException; +} diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthTokenProviders.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthTokenProviders.java new file mode 100644 index 0000000000..bbf7072d39 --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthTokenProviders.java @@ -0,0 +1,419 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.client.oauth; + +import com.nimbusds.oauth2.sdk.ParseException; +import com.nimbusds.oauth2.sdk.Scope; +import com.nimbusds.oauth2.sdk.auth.ClientAuthentication; +import com.nimbusds.oauth2.sdk.auth.ClientSecretBasic; +import com.nimbusds.oauth2.sdk.auth.Secret; +import com.nimbusds.oauth2.sdk.id.Audience; +import com.nimbusds.oauth2.sdk.id.ClientID; +import com.nimbusds.oauth2.sdk.token.TokenTypeURI; +import com.nimbusds.oauth2.sdk.token.TypelessAccessToken; +import com.nimbusds.oauth2.sdk.tokenexchange.TokenExchangeGrant; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** + * Unified factory for creating OAuth token providers. + * + *

This class provides a single entry point for creating all OAuth token providers with a + * consistent builder API. It supports: + * + *

    + *
  • Client Credentials flow (RFC 6749 Section 4.4) + *
  • Token Exchange flow (RFC 8693) + *
+ * + *

Example usage: + * + * 

{@code
+ * // Client Credentials flow
+ * OAuthTokenProvider provider = OAuthTokenProviders.clientCredentials()
+ *     .tokenUri("https://auth.example.com/token")
+ *     .clientId("my-client")
+ *     .clientSecret("my-secret")
+ *     .scope("read write")
+ *     .build();
+ *
+ * // Token Exchange flow
+ * OAuthTokenProvider provider = OAuthTokenProviders.tokenExchange()
+ *     .tokenUri("https://auth.example.com/token")
+ *     .subjectToken("user-token")
+ *     .subjectTokenType("urn:ietf:params:oauth:token-type:access_token")
+ *     .build();
+ * }
+ */ +public final class OAuthTokenProviders { + + private OAuthTokenProviders() {} + + /** + * Creates a new builder for Client Credentials flow. + * + * @return a new ClientCredentialsBuilder instance + */ + public static ClientCredentialsBuilder clientCredentials() { + return new ClientCredentialsBuilder(); + } + + /** + * Creates a new builder for Token Exchange flow. + * + * @return a new TokenExchangeBuilder instance + */ + public static TokenExchangeBuilder tokenExchange() { + return new TokenExchangeBuilder(); + } + + /** Builder for creating {@link ClientCredentialsTokenProvider} instances. */ + public static class ClientCredentialsBuilder { + private @Nullable URI tokenUri; + private @Nullable String clientId; + private @Nullable String clientSecret; + private @Nullable String scope; + + ClientCredentialsBuilder() {} + + /** + * Sets the OAuth token endpoint URI (required). + * + * @param tokenUri the token endpoint URI + * @return this builder + */ + public ClientCredentialsBuilder tokenUri(URI tokenUri) { + this.tokenUri = Objects.requireNonNull(tokenUri, "tokenUri cannot be null"); + return this; + } + + /** + * Sets the OAuth token endpoint URI from a string (required). + * + * @param tokenUri the token endpoint URI string + * @return this builder + * @throws IllegalArgumentException if the URI is invalid + */ + public ClientCredentialsBuilder tokenUri(String tokenUri) { + Objects.requireNonNull(tokenUri, "tokenUri cannot be null"); + try { + this.tokenUri = new URI(tokenUri); + } catch (URISyntaxException e) { + throw new IllegalArgumentException("Invalid token URI: " + tokenUri, e); + } + return this; + } + + /** + * Sets the OAuth client ID (required). + * + * @param clientId the client ID + * @return this builder + */ + public ClientCredentialsBuilder clientId(String clientId) { + this.clientId = Objects.requireNonNull(clientId, "clientId cannot be null"); + return this; + } + + /** + * Sets the OAuth client secret (required). + * + * @param clientSecret the client secret + * @return this builder + */ + public ClientCredentialsBuilder clientSecret(String clientSecret) { + this.clientSecret = Objects.requireNonNull(clientSecret, "clientSecret cannot be null"); + return this; + } + + /** + * Sets the OAuth scopes (optional). + * + * @param scope the space-separated scope string + * @return this builder + */ + public ClientCredentialsBuilder scope(@Nullable String scope) { + this.scope = scope; + return this; + } + + /** + * Builds a new ClientCredentialsTokenProvider instance. + * + * @return the configured ClientCredentialsTokenProvider + * @throws IllegalStateException if required parameters are missing + */ + public ClientCredentialsTokenProvider build() { + if (tokenUri == null) { + throw new IllegalStateException("tokenUri is required"); + } + if (clientId == null) { + throw new IllegalStateException("clientId is required"); + } + if (clientSecret == null) { + throw new IllegalStateException("clientSecret is required"); + } + return new ClientCredentialsTokenProvider(tokenUri, clientId, clientSecret, scope); + } + } + + /** Builder for creating {@link TokenExchangeTokenProvider} instances. */ + public static class TokenExchangeBuilder { + private @Nullable URI tokenUri; + private @Nullable String subjectToken; + private @Nullable String subjectTokenType; + private @Nullable String actorToken; + private @Nullable String actorTokenType; + private @Nullable String audience; + private @Nullable String requestedTokenType; + private @Nullable Scope scope; + private @Nullable List resources; + private @Nullable ClientAuthentication clientAuth; + + TokenExchangeBuilder() {} + + /** + * Sets the OAuth token endpoint URI (required). + * + * @param tokenUri the token endpoint URI + * @return this builder + */ + public TokenExchangeBuilder tokenUri(URI tokenUri) { + this.tokenUri = Objects.requireNonNull(tokenUri, "tokenUri cannot be null"); + return this; + } + + /** + * Sets the OAuth token endpoint URI from a string (required). + * + * @param tokenUri the token endpoint URI string + * @return this builder + * @throws IllegalArgumentException if the URI is invalid + */ + public TokenExchangeBuilder tokenUri(String tokenUri) { + Objects.requireNonNull(tokenUri, "tokenUri cannot be null"); + try { + this.tokenUri = new URI(tokenUri); + } catch (URISyntaxException e) { + throw new IllegalArgumentException("Invalid token URI: " + tokenUri, e); + } + return this; + } + + /** + * Sets the subject token to exchange (required). + * + * @param subjectToken the subject token value + * @return this builder + */ + public TokenExchangeBuilder subjectToken(String subjectToken) { + this.subjectToken = Objects.requireNonNull(subjectToken, "subjectToken cannot be null"); + return this; + } + + /** + * Sets the type of the subject token (required). + * + * @param subjectTokenType the subject token type URI + * @return this builder + */ + public TokenExchangeBuilder subjectTokenType(String subjectTokenType) { + this.subjectTokenType = + Objects.requireNonNull(subjectTokenType, "subjectTokenType cannot be null"); + return this; + } + + /** + * Sets the optional actor token for delegation scenarios. + * + * @param actorToken the actor token value + * @return this builder + */ + public TokenExchangeBuilder actorToken(@Nullable String actorToken) { + this.actorToken = actorToken; + return this; + } + + /** + * Sets the type of the actor token. + * + * @param actorTokenType the actor token type URI + * @return this builder + */ + public TokenExchangeBuilder actorTokenType(@Nullable String actorTokenType) { + this.actorTokenType = actorTokenType; + return this; + } + + /** + * Sets the target audience for the exchanged token. + * + * @param audience the target audience + * @return this builder + */ + public TokenExchangeBuilder audience(@Nullable String audience) { + this.audience = audience; + return this; + } + + /** + * Sets the requested token type for the exchanged token. + * + * @param requestedTokenType the requested token type URI + * @return this builder + */ + public TokenExchangeBuilder requestedTokenType(@Nullable String requestedTokenType) { + this.requestedTokenType = requestedTokenType; + return this; + } + + /** + * Sets the OAuth scopes for the token request. + * + * @param scope the OAuth scope object + * @return this builder + */ + public TokenExchangeBuilder scope(@Nullable Scope scope) { + this.scope = scope; + return this; + } + + /** + * Sets the OAuth scopes from a space-separated string. + * + * @param scope the space-separated scope string + * @return this builder + */ + public TokenExchangeBuilder scope(@Nullable String scope) { + this.scope = (scope != null && !scope.isEmpty()) ? Scope.parse(scope) : null; + return this; + } + + /** + * Sets the target resource URIs (RFC 8707). + * + * @param resources the list of resource URIs + * @return this builder + */ + public TokenExchangeBuilder resources(@Nullable List resources) { + this.resources = resources; + return this; + } + + /** + * Sets a single target resource URI (RFC 8707). + * + * @param resource the resource URI + * @return this builder + */ + public TokenExchangeBuilder resource(@Nullable URI resource) { + this.resources = resource != null ? Collections.singletonList(resource) : null; + return this; + } + + /** + * Sets a single target resource URI from a string (RFC 8707). + * + * @param resource the resource URI string + * @return this builder + */ + public TokenExchangeBuilder resource(@Nullable String resource) { + if (resource != null && !resource.isEmpty()) { + this.resources = Collections.singletonList(URI.create(resource)); + } else { + this.resources = null; + } + return this; + } + + /** + * Sets the client authentication. + * + * @param clientAuth the client authentication object + * @return this builder + */ + public TokenExchangeBuilder clientAuthentication(@Nullable ClientAuthentication clientAuth) { + this.clientAuth = clientAuth; + return this; + } + + /** + * Sets client authentication using client ID and secret. + * + * @param clientId the client ID + * @param clientSecret the client secret + * @return this builder + */ + public TokenExchangeBuilder clientCredentials(String clientId, String clientSecret) { + Objects.requireNonNull(clientId, "clientId cannot be null"); + Objects.requireNonNull(clientSecret, "clientSecret cannot be null"); + this.clientAuth = new ClientSecretBasic(new ClientID(clientId), new Secret(clientSecret)); + return this; + } + + /** + * Builds a new TokenExchangeTokenProvider instance. + * + * @return the configured TokenExchangeTokenProvider + * @throws IllegalStateException if required parameters are missing + */ + public TokenExchangeTokenProvider build() { + if (tokenUri == null) { + throw new IllegalStateException("tokenUri is required"); + } + if (subjectToken == null) { + throw new IllegalStateException("subjectToken is required"); + } + if (subjectTokenType == null) { + throw new IllegalStateException("subjectTokenType is required"); + } + + TokenExchangeGrant grant = createGrant(); + return new TokenExchangeTokenProvider(tokenUri, grant, clientAuth, scope, resources); + } + + private TokenExchangeGrant createGrant() { + try { + TypelessAccessToken subjectAccessToken = new TypelessAccessToken(subjectToken); + TokenTypeURI subjectTypeUri = TokenTypeURI.parse(subjectTokenType); + + TypelessAccessToken actorAccessToken = + actorToken != null ? new TypelessAccessToken(actorToken) : null; + TokenTypeURI actorTypeUri = + actorTokenType != null ? TokenTypeURI.parse(actorTokenType) : null; + TokenTypeURI requestedTypeUri = + requestedTokenType != null ? TokenTypeURI.parse(requestedTokenType) : null; + List audienceList = + audience != null ? Collections.singletonList(new Audience(audience)) : null; + + return new TokenExchangeGrant( + subjectAccessToken, + subjectTypeUri, + actorAccessToken, + actorTypeUri, + requestedTypeUri, + audienceList); + } catch (ParseException e) { + throw new IllegalStateException("Failed to create TokenExchangeGrant", e); + } + } + } +} diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/TokenExchangeTokenProvider.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/TokenExchangeTokenProvider.java new file mode 100644 index 0000000000..af433a2712 --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/TokenExchangeTokenProvider.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.client.oauth; + +import com.nimbusds.oauth2.sdk.Scope; +import com.nimbusds.oauth2.sdk.TokenRequest; +import com.nimbusds.oauth2.sdk.auth.ClientAuthentication; +import com.nimbusds.oauth2.sdk.tokenexchange.TokenExchangeGrant; +import java.net.URI; +import java.util.List; +import java.util.Objects; +import org.apache.arrow.util.VisibleForTesting; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** + * OAuth 2.0 Token Exchange flow token provider (RFC 8693). + * + *

This provider exchanges one token for another, commonly used for federated authentication, + * delegation, or impersonation scenarios. Tokens are cached and automatically refreshed. + */ +public class TokenExchangeTokenProvider extends AbstractOAuthTokenProvider { + + @VisibleForTesting TokenExchangeGrant grant; + + @VisibleForTesting @Nullable List resources; + + /** + * Creates a new TokenExchangeTokenProvider with full configuration. + * + * @param tokenUri the OAuth token endpoint URI + * @param grant the token exchange grant containing subject/actor token information + * @param clientAuth optional client authentication + * @param scope optional OAuth scopes + * @param resource optional target resource URI (RFC 8707) + */ + TokenExchangeTokenProvider( + URI tokenUri, + TokenExchangeGrant grant, + @Nullable ClientAuthentication clientAuth, + @Nullable Scope scope, + @Nullable List resource) { + this.tokenUri = Objects.requireNonNull(tokenUri, "tokenUri cannot be null"); + this.grant = Objects.requireNonNull(grant, "grant cannot be null"); + this.scope = scope; + this.resources = resource; + this.clientAuth = clientAuth; + } + + @Override + protected TokenRequest buildTokenRequest() { + TokenRequest.Builder builder; + if (clientAuth != null) { + builder = new TokenRequest.Builder(tokenUri, clientAuth, grant); + } else { + builder = new TokenRequest.Builder(tokenUri, grant); + } + + if (scope != null) { + builder.scope(scope); + } + if (resources != null) { + builder.resources(resources.toArray(new URI[0])); + } + + return builder.build(); + } +} diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/TokenInfo.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/TokenInfo.java new file mode 100644 index 0000000000..f47cc8b053 --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/oauth/TokenInfo.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.client.oauth; + +import java.time.Instant; +import java.util.Objects; + +/** Holds OAuth token information including the access token and expiration time. */ +public class TokenInfo { + private final String accessToken; + private final Instant expiresAt; + + public TokenInfo(String accessToken, Instant expiresAt) { + this.accessToken = Objects.requireNonNull(accessToken, "accessToken cannot be null"); + this.expiresAt = Objects.requireNonNull(expiresAt, "expiresAt cannot be null"); + } + + public String getAccessToken() { + return accessToken; + } + + /** + * Checks if the token is expired or will expire within the buffer period. + * + * @param bufferSeconds seconds before actual expiration to consider token expired + * @return true if token should be refreshed + */ + public boolean isExpired(int bufferSeconds) { + return Instant.now().plusSeconds(bufferSeconds).isAfter(expiresAt); + } +} diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java index 76ba964a53..d0ba74dbcc 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.driver.jdbc.utils; +import java.sql.SQLException; import java.time.Duration; import java.util.Arrays; import java.util.HashMap; @@ -23,6 +24,7 @@ import java.util.Objects; import java.util.Properties; import org.apache.arrow.driver.jdbc.ArrowFlightConnection; +import org.apache.arrow.driver.jdbc.client.oauth.OAuthConfiguration; import org.apache.arrow.flight.CallHeaders; import org.apache.arrow.flight.CallOption; import org.apache.arrow.flight.FlightCallHeaders; @@ -31,6 +33,7 @@ import org.apache.calcite.avatica.ConnectionConfig; import org.apache.calcite.avatica.ConnectionConfigImpl; import org.apache.calcite.avatica.ConnectionProperty; +import org.checkerframework.checker.nullness.qual.Nullable; /** A {@link ConnectionConfig} for the {@link ArrowFlightConnection}. */ public final class ArrowFlightConnectionConfigImpl extends ConnectionConfigImpl { @@ -211,6 +214,38 @@ public Map getHeaderAttributes() { return headers; } + /** + * Returns OAuth configuration if oauth.flow is specified, null otherwise. + * + * @return the OAuth configuration or null + * @throws SQLException if the OAuth configuration is invalid + */ + public @Nullable OAuthConfiguration getOauthConfiguration() throws SQLException { + String flow = ArrowFlightConnectionProperty.OAUTH_FLOW.getString(properties); + if (flow == null) { + return null; + } + + return new OAuthConfiguration.Builder() + .flow(flow) + .clientId(ArrowFlightConnectionProperty.OAUTH_CLIENT_ID.getString(properties)) + .clientSecret(ArrowFlightConnectionProperty.OAUTH_CLIENT_SECRET.getString(properties)) + .tokenUri(ArrowFlightConnectionProperty.OAUTH_TOKEN_URI.getString(properties)) + .scope(ArrowFlightConnectionProperty.OAUTH_SCOPE.getString(properties)) + .resource(ArrowFlightConnectionProperty.OAUTH_RESOURCE.getString(properties)) + .subjectToken( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_SUBJECT_TOKEN.getString(properties)) + .subjectTokenType( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_SUBJECT_TOKEN_TYPE.getString(properties)) + .actorToken(ArrowFlightConnectionProperty.OAUTH_EXCHANGE_ACTOR_TOKEN.getString(properties)) + .actorTokenType( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_ACTOR_TOKEN_TYPE.getString(properties)) + .audience(ArrowFlightConnectionProperty.OAUTH_EXCHANGE_AUDIENCE.getString(properties)) + .requestedTokenType( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_REQUESTED_TOKEN_TYPE.getString(properties)) + .build(); + } + /** Custom {@link ConnectionProperty} for the {@link ArrowFlightConnectionConfigImpl}. */ public enum ArrowFlightConnectionProperty implements ConnectionProperty { HOST("host", null, Type.STRING, true), @@ -232,6 +267,23 @@ public enum ArrowFlightConnectionProperty implements ConnectionProperty { CATALOG("catalog", null, Type.STRING, false), CONNECT_TIMEOUT_MILLIS("connectTimeoutMs", 10000, Type.NUMBER, false), USE_CLIENT_CACHE("useClientCache", true, Type.BOOLEAN, false), + + // OAuth configuration properties + OAUTH_FLOW("oauth.flow", null, Type.STRING, false), + OAUTH_CLIENT_ID("oauth.clientId", null, Type.STRING, false), + OAUTH_CLIENT_SECRET("oauth.clientSecret", null, Type.STRING, false), + OAUTH_TOKEN_URI("oauth.tokenUri", null, Type.STRING, false), + OAUTH_SCOPE("oauth.scope", null, Type.STRING, false), + OAUTH_RESOURCE("oauth.resource", null, Type.STRING, false), + + // Token exchange specific properties + OAUTH_EXCHANGE_SUBJECT_TOKEN("oauth.exchange.subjectToken", null, Type.STRING, false), + OAUTH_EXCHANGE_SUBJECT_TOKEN_TYPE("oauth.exchange.subjectTokenType", null, Type.STRING, false), + OAUTH_EXCHANGE_ACTOR_TOKEN("oauth.exchange.actorToken", null, Type.STRING, false), + OAUTH_EXCHANGE_ACTOR_TOKEN_TYPE("oauth.exchange.actorTokenType", null, Type.STRING, false), + OAUTH_EXCHANGE_AUDIENCE("oauth.exchange.aud", null, Type.STRING, false), + OAUTH_EXCHANGE_REQUESTED_TOKEN_TYPE( + "oauth.exchange.requestedTokenType", null, Type.STRING, false), ; private final String camelName; diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/OAuthIntegrationTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/OAuthIntegrationTest.java new file mode 100644 index 0000000000..5e782db031 --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/OAuthIntegrationTest.java @@ -0,0 +1,474 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.URI; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import mockwebserver3.MockResponse; +import mockwebserver3.MockWebServer; +import mockwebserver3.RecordedRequest; +import mockwebserver3.junit5.StartStop; +import org.apache.arrow.driver.jdbc.authentication.TokenAuthentication; +import org.apache.arrow.driver.jdbc.utils.ArrowFlightConnectionConfigImpl.ArrowFlightConnectionProperty; +import org.apache.arrow.driver.jdbc.utils.MockFlightSqlProducer; +import org.apache.arrow.flight.sql.FlightSqlProducer.Schemas; +import org.apache.arrow.flight.sql.impl.FlightSql.CommandGetCatalogs; +import org.apache.arrow.flight.sql.impl.FlightSql.CommandGetDbSchemas; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.util.AutoCloseables; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * Integration tests for OAuth authentication flows in the JDBC driver. + * + *

These tests verify that OAuth tokens obtained from an OAuth server are correctly used in + * Flight SQL requests. + */ +public class OAuthIntegrationTest { + + private static final String VALID_ACCESS_TOKEN = "valid-oauth-access-token-12345"; + private static final String CLIENT_ID = "test-client-id"; + private static final String CLIENT_SECRET = "test-client-secret"; + private static final String SUBJECT_TOKEN = "original-subject-token"; + private static final String SUBJECT_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:jwt"; + private static final String TEST_SCOPE = "dremio.all"; + + private static final MockFlightSqlProducer FLIGHT_SQL_PRODUCER = new MockFlightSqlProducer(); + + @RegisterExtension public static FlightServerTestExtension FLIGHT_SERVER_TEST_EXTENSION; + + static { + FLIGHT_SERVER_TEST_EXTENSION = + new FlightServerTestExtension.Builder() + .authentication(new TokenAuthentication.Builder().token(VALID_ACCESS_TOKEN).build()) + .producer(FLIGHT_SQL_PRODUCER) + .build(); + } + + @StartStop private final MockWebServer oauthServer = new MockWebServer(); + private URI tokenEndpoint; + + @BeforeAll + public static void setUpClass() { + // Register a simple catalog query handler + FLIGHT_SQL_PRODUCER.addCatalogQuery( + CommandGetCatalogs.getDefaultInstance(), + listener -> { + try (BufferAllocator allocator = new RootAllocator(); + VectorSchemaRoot root = + VectorSchemaRoot.create(Schemas.GET_CATALOGS_SCHEMA, allocator)) { + root.setRowCount(0); + listener.start(root); + listener.putNext(); + } catch (Throwable t) { + listener.error(t); + } finally { + listener.completed(); + } + }); + + // Register a simple schema query handler for getSchemas() + FLIGHT_SQL_PRODUCER.addCatalogQuery( + CommandGetDbSchemas.getDefaultInstance(), + listener -> { + try (BufferAllocator allocator = new RootAllocator(); + VectorSchemaRoot root = + VectorSchemaRoot.create(Schemas.GET_SCHEMAS_SCHEMA, allocator)) { + root.setRowCount(0); + listener.start(root); + listener.putNext(); + } catch (Throwable t) { + listener.error(t); + } finally { + listener.completed(); + } + }); + } + + @AfterAll + public static void tearDownClass() { + AutoCloseables.closeNoChecked(FLIGHT_SQL_PRODUCER); + } + + @BeforeEach + public void setUp() { + tokenEndpoint = oauthServer.url("/oauth/token").uri(); + } + + @AfterEach + public void tearDown() { + oauthServer.close(); + } + + // Helper methods for mock OAuth responses + + private void enqueueSuccessfulTokenResponse() { + enqueueSuccessfulTokenResponse(VALID_ACCESS_TOKEN, 3600); + } + + private void enqueueSuccessfulTokenResponse(String token, int expiresIn) { + String body = + String.format( + "{\"access_token\":\"%s\",\"token_type\":\"Bearer\",\"expires_in\":%d}", + token, expiresIn); + oauthServer.enqueue( + new MockResponse.Builder() + .code(200) + .setHeader("Content-Type", "application/json") + .body(body) + .build()); + } + + private void enqueueErrorResponse(String error, String description) { + String body = + String.format("{\"error\":\"%s\",\"error_description\":\"%s\"}", error, description); + oauthServer.enqueue( + new MockResponse.Builder() + .code(400) + .setHeader("Content-Type", "application/json") + .body(body) + .build()); + } + + private Properties createBaseProperties() { + Properties props = new Properties(); + props.put(ArrowFlightConnectionProperty.HOST.camelName(), "localhost"); + props.put( + ArrowFlightConnectionProperty.PORT.camelName(), FLIGHT_SERVER_TEST_EXTENSION.getPort()); + props.put(ArrowFlightConnectionProperty.USE_ENCRYPTION.camelName(), false); + return props; + } + + private String getJdbcUrl() { + return String.format( + "jdbc:arrow-flight-sql://localhost:%d", FLIGHT_SERVER_TEST_EXTENSION.getPort()); + } + + // ==================== Client Credentials Flow Tests ==================== + + @Test + public void testClientCredentialsFlowSuccess() throws Exception { + enqueueSuccessfulTokenResponse(); + + Properties props = createBaseProperties(); + props.put(ArrowFlightConnectionProperty.OAUTH_FLOW.camelName(), "client_credentials"); + props.put(ArrowFlightConnectionProperty.OAUTH_TOKEN_URI.camelName(), tokenEndpoint.toString()); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_ID.camelName(), CLIENT_ID); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_SECRET.camelName(), CLIENT_SECRET); + props.put(ArrowFlightConnectionProperty.OAUTH_SCOPE.camelName(), TEST_SCOPE); + + try (Connection conn = DriverManager.getConnection(getJdbcUrl(), props)) { + assertFalse(conn.isClosed()); + // Trigger a Flight call to force OAuth token retrieval + conn.getMetaData().getCatalogs().close(); + } + + // Verify OAuth request was made + RecordedRequest oauthRequest = oauthServer.takeRequest(5, TimeUnit.SECONDS); + assertNotNull(oauthRequest, "OAuth request should have been made"); + assertEquals("POST", oauthRequest.getMethod()); + String body = oauthRequest.getBody().utf8(); + assertTrue(body.contains("grant_type=client_credentials")); + assertTrue(body.contains("scope=" + TEST_SCOPE)); + } + + @Test + public void testClientCredentialsFlowWithUrlParameters() throws Exception { + enqueueSuccessfulTokenResponse(); + + String url = + String.format( + "jdbc:arrow-flight-sql://localhost:%d?useEncryption=false" + + "&oauth.flow=client_credentials" + + "&oauth.tokenUri=%s" + + "&oauth.clientId=%s" + + "&oauth.clientSecret=%s", + FLIGHT_SERVER_TEST_EXTENSION.getPort(), + tokenEndpoint.toString(), + CLIENT_ID, + CLIENT_SECRET); + + try (Connection conn = DriverManager.getConnection(url)) { + conn.getMetaData().getCatalogs().close(); + } + + assertEquals(1, oauthServer.getRequestCount()); + } + + @Test + public void testClientCredentialsFlowInvalidCredentials() throws Exception { + enqueueErrorResponse("invalid_client", "Client authentication failed"); + + Properties props = createBaseProperties(); + props.put(ArrowFlightConnectionProperty.OAUTH_FLOW.camelName(), "client_credentials"); + props.put(ArrowFlightConnectionProperty.OAUTH_TOKEN_URI.camelName(), tokenEndpoint.toString()); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_ID.camelName(), "wrong-client"); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_SECRET.camelName(), "wrong-secret"); + + Exception ex = + assertThrows( + Exception.class, + () -> { + try (Connection conn = DriverManager.getConnection(getJdbcUrl(), props)) { + conn.getMetaData().getCatalogs().close(); + } + }); + // Verify the error message contains the OAuth error somewhere in the exception chain + assertTrue( + containsInExceptionChain(ex, "invalid_client"), + "Exception chain should contain 'invalid_client'"); + } + + private boolean containsInExceptionChain(Throwable t, String message) { + while (t != null) { + if (t.getMessage() != null && t.getMessage().contains(message)) { + return true; + } + t = t.getCause(); + } + return false; + } + + // ==================== Token Exchange Flow Tests ==================== + + @Test + public void testTokenExchangeFlowMinimalParameters() throws Exception { + enqueueSuccessfulTokenResponse(); + + Properties props = createBaseProperties(); + props.put(ArrowFlightConnectionProperty.OAUTH_FLOW.camelName(), "token_exchange"); + props.put(ArrowFlightConnectionProperty.OAUTH_TOKEN_URI.camelName(), tokenEndpoint.toString()); + props.put( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_SUBJECT_TOKEN.camelName(), SUBJECT_TOKEN); + props.put( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_SUBJECT_TOKEN_TYPE.camelName(), + SUBJECT_TOKEN_TYPE); + + try (Connection conn = DriverManager.getConnection(getJdbcUrl(), props)) { + conn.getMetaData().getCatalogs().close(); + } + + RecordedRequest oauthRequest = oauthServer.takeRequest(5, TimeUnit.SECONDS); + assertNotNull(oauthRequest, "OAuth request should have been made"); + String body = oauthRequest.getBody().utf8(); + assertTrue( + body.contains("grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Atoken-exchange"), + "Should contain token exchange grant type"); + assertTrue(body.contains("subject_token=" + SUBJECT_TOKEN)); + } + + @Test + public void testTokenExchangeFlowWithAllParameters() throws Exception { + enqueueSuccessfulTokenResponse(); + + String actorToken = "actor-token-value"; + String actorTokenType = "urn:ietf:params:oauth:token-type:access_token"; + String audience = "https://api.example.com"; + String resource = "https://api.example.com/resource"; + String requestedTokenType = "urn:ietf:params:oauth:token-type:access_token"; + + Properties props = createBaseProperties(); + props.put(ArrowFlightConnectionProperty.OAUTH_FLOW.camelName(), "token_exchange"); + props.put(ArrowFlightConnectionProperty.OAUTH_TOKEN_URI.camelName(), tokenEndpoint.toString()); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_ID.camelName(), CLIENT_ID); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_SECRET.camelName(), CLIENT_SECRET); + props.put(ArrowFlightConnectionProperty.OAUTH_SCOPE.camelName(), TEST_SCOPE); + props.put( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_SUBJECT_TOKEN.camelName(), SUBJECT_TOKEN); + props.put( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_SUBJECT_TOKEN_TYPE.camelName(), + SUBJECT_TOKEN_TYPE); + props.put(ArrowFlightConnectionProperty.OAUTH_EXCHANGE_ACTOR_TOKEN.camelName(), actorToken); + props.put( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_ACTOR_TOKEN_TYPE.camelName(), actorTokenType); + props.put(ArrowFlightConnectionProperty.OAUTH_EXCHANGE_AUDIENCE.camelName(), audience); + props.put(ArrowFlightConnectionProperty.OAUTH_RESOURCE.camelName(), resource); + props.put( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_REQUESTED_TOKEN_TYPE.camelName(), + requestedTokenType); + + try (Connection conn = DriverManager.getConnection(getJdbcUrl(), props)) { + conn.getMetaData().getCatalogs().close(); + } + + RecordedRequest oauthRequest = oauthServer.takeRequest(5, TimeUnit.SECONDS); + assertNotNull(oauthRequest, "OAuth request should have been made"); + String body = oauthRequest.getBody().utf8(); + assertTrue(body.contains("subject_token=" + SUBJECT_TOKEN)); + assertTrue(body.contains("actor_token=" + actorToken)); + } + + @Test + public void testTokenExchangeFlowWithClientAuthentication() throws Exception { + enqueueSuccessfulTokenResponse(); + + Properties props = createBaseProperties(); + props.put(ArrowFlightConnectionProperty.OAUTH_FLOW.camelName(), "token_exchange"); + props.put(ArrowFlightConnectionProperty.OAUTH_TOKEN_URI.camelName(), tokenEndpoint.toString()); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_ID.camelName(), CLIENT_ID); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_SECRET.camelName(), CLIENT_SECRET); + props.put( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_SUBJECT_TOKEN.camelName(), SUBJECT_TOKEN); + props.put( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_SUBJECT_TOKEN_TYPE.camelName(), + SUBJECT_TOKEN_TYPE); + + try (Connection conn = DriverManager.getConnection(getJdbcUrl(), props)) { + conn.getMetaData().getCatalogs().close(); + } + + RecordedRequest oauthRequest = oauthServer.takeRequest(5, TimeUnit.SECONDS); + assertNotNull(oauthRequest, "OAuth request should have been made"); + String authHeader = oauthRequest.getHeaders().get("Authorization"); + assertNotNull(authHeader, "Should have Basic auth header for client authentication"); + assertTrue(authHeader.startsWith("Basic ")); + } + + // ==================== Token Caching Tests ==================== + + @Test + public void testTokenCachingAcrossMultipleOperations() throws Exception { + enqueueSuccessfulTokenResponse(); + + Properties props = createBaseProperties(); + props.put(ArrowFlightConnectionProperty.OAUTH_FLOW.camelName(), "client_credentials"); + props.put(ArrowFlightConnectionProperty.OAUTH_TOKEN_URI.camelName(), tokenEndpoint.toString()); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_ID.camelName(), CLIENT_ID); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_SECRET.camelName(), CLIENT_SECRET); + + try (Connection conn = DriverManager.getConnection(getJdbcUrl(), props)) { + // Execute multiple operations + conn.isValid(5); + conn.getMetaData().getCatalogs().close(); + conn.getMetaData().getSchemas().close(); + } + + // Should only have made one OAuth request due to caching + assertEquals(1, oauthServer.getRequestCount()); + } + + @Test + public void testTokenRefreshAfterExpiration() throws Exception { + enqueueSuccessfulTokenResponse(VALID_ACCESS_TOKEN, 1); + enqueueSuccessfulTokenResponse(VALID_ACCESS_TOKEN, 3600); + + Properties props = createBaseProperties(); + props.put(ArrowFlightConnectionProperty.OAUTH_FLOW.camelName(), "token_exchange"); + props.put(ArrowFlightConnectionProperty.OAUTH_TOKEN_URI.camelName(), tokenEndpoint.toString()); + props.put( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_SUBJECT_TOKEN.camelName(), SUBJECT_TOKEN); + props.put( + ArrowFlightConnectionProperty.OAUTH_EXCHANGE_SUBJECT_TOKEN_TYPE.camelName(), + SUBJECT_TOKEN_TYPE); + + try (Connection conn = DriverManager.getConnection(getJdbcUrl(), props)) { + // First operation triggers initial token fetch + conn.getMetaData().getCatalogs().close(); + + // Token with 1s expiry is immediately considered expired (due to 30s buffer) + // so the next operation should trigger a refresh + conn.getMetaData().getCatalogs().close(); + } + + // Should have made exactly 2 OAuth requests: initial + refresh + assertEquals(2, oauthServer.getRequestCount()); + } + + // ==================== Error Handling Tests ==================== + + @Test + public void testMissingRequiredParametersClientCredentials() { + Properties props = createBaseProperties(); + props.put(ArrowFlightConnectionProperty.OAUTH_FLOW.camelName(), "client_credentials"); + props.put(ArrowFlightConnectionProperty.OAUTH_TOKEN_URI.camelName(), tokenEndpoint.toString()); + // Missing client_id and client_secret + + assertThrows(SQLException.class, () -> DriverManager.getConnection(getJdbcUrl(), props)); + } + + @Test + public void testMissingRequiredParametersTokenExchange() { + Properties props = createBaseProperties(); + props.put(ArrowFlightConnectionProperty.OAUTH_FLOW.camelName(), "token_exchange"); + props.put(ArrowFlightConnectionProperty.OAUTH_TOKEN_URI.camelName(), tokenEndpoint.toString()); + // Missing subject_token and subject_token_type + + assertThrows(SQLException.class, () -> DriverManager.getConnection(getJdbcUrl(), props)); + } + + @Test + public void testInvalidOAuthFlow() { + Properties props = createBaseProperties(); + props.put(ArrowFlightConnectionProperty.OAUTH_FLOW.camelName(), "invalid_flow"); + props.put(ArrowFlightConnectionProperty.OAUTH_TOKEN_URI.camelName(), tokenEndpoint.toString()); + + assertThrows(SQLException.class, () -> DriverManager.getConnection(getJdbcUrl(), props)); + } + + @Test + public void testMalformedTokenEndpoint() { + Properties props = createBaseProperties(); + props.put(ArrowFlightConnectionProperty.OAUTH_FLOW.camelName(), "client_credentials"); + props.put(ArrowFlightConnectionProperty.OAUTH_TOKEN_URI.camelName(), "not-a-valid-uri://"); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_ID.camelName(), CLIENT_ID); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_SECRET.camelName(), CLIENT_SECRET); + + assertThrows(SQLException.class, () -> DriverManager.getConnection(getJdbcUrl(), props)); + } + + // ==================== Authorization Header Verification ==================== + + @Test + public void testOAuthTokenSentAsBearer() throws Exception { + enqueueSuccessfulTokenResponse(); + + Properties props = createBaseProperties(); + props.put(ArrowFlightConnectionProperty.OAUTH_FLOW.camelName(), "client_credentials"); + props.put(ArrowFlightConnectionProperty.OAUTH_TOKEN_URI.camelName(), tokenEndpoint.toString()); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_ID.camelName(), CLIENT_ID); + props.put(ArrowFlightConnectionProperty.OAUTH_CLIENT_SECRET.camelName(), CLIENT_SECRET); + + try (Connection conn = DriverManager.getConnection(getJdbcUrl(), props)) { + conn.getMetaData().getCatalogs().close(); + } + + // Verify the Flight server received the bearer token + String authHeader = + FLIGHT_SERVER_TEST_EXTENSION + .getInterceptorFactory() + .getHeader(org.apache.arrow.flight.FlightMethod.GET_FLIGHT_INFO, "authorization"); + assertNotNull(authHeader, "Authorization header should be present in Flight requests"); + assertEquals("Bearer " + VALID_ACCESS_TOKEN, authHeader); + } +} diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthConfigurationTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthConfigurationTest.java new file mode 100644 index 0000000000..c258a7c652 --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthConfigurationTest.java @@ -0,0 +1,296 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.client.oauth; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.nimbusds.oauth2.sdk.Scope; +import java.net.URI; +import java.sql.SQLException; +import java.util.Collections; +import java.util.stream.Stream; +import org.junit.jupiter.api.Named; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** Tests for {@link OAuthConfiguration}. */ +public class OAuthConfigurationTest { + + private static final String TOKEN_URI = "https://auth.example.com/token"; + private static final String CLIENT_ID = "test-client-id"; + private static final String CLIENT_SECRET = "test-client-secret"; + private static final String SCOPE = "read write"; + private static final String SUBJECT_TOKEN = "subject-token-value"; + public static final String RESOURCE = "https://api.example.com/resource"; + + @FunctionalInterface + interface BuilderConfigurer { + void configure(OAuthConfiguration.Builder builder) throws SQLException; + } + + static Stream createFlowCases() { + return Stream.of( + Arguments.of( + Named.of( + "string flow", (BuilderConfigurer) builder -> builder.flow("client_credentials"))), + Arguments.of( + Named.of( + "uppercase string flow", + (BuilderConfigurer) builder -> builder.flow("CLIENT_CREDENTIALS")))); + } + + @ParameterizedTest + @MethodSource("createFlowCases") + public void testCreateFlowConfiguration(BuilderConfigurer flowConfigurer) throws SQLException { + OAuthConfiguration.Builder builder = new OAuthConfiguration.Builder(); + flowConfigurer.configure(builder); + OAuthConfiguration config = + builder.tokenUri(TOKEN_URI).clientId(CLIENT_ID).clientSecret(CLIENT_SECRET).build(); + + // Verify configuration creates correct provider type + OAuthTokenProvider provider = config.createTokenProvider(); + assertInstanceOf(ClientCredentialsTokenProvider.class, provider); + } + + @Test + public void testCreateClientCredentialsTokenProvider() throws SQLException { + OAuthConfiguration config = + new OAuthConfiguration.Builder() + .flow("client_credentials") + .tokenUri(TOKEN_URI) + .clientId(CLIENT_ID) + .clientSecret(CLIENT_SECRET) + .scope(SCOPE) + .build(); + + OAuthTokenProvider provider = config.createTokenProvider(); + + assertNotNull(provider); + assertInstanceOf(ClientCredentialsTokenProvider.class, provider); + + ClientCredentialsTokenProvider ccProvider = (ClientCredentialsTokenProvider) provider; + assertEquals(URI.create(TOKEN_URI), ccProvider.tokenUri); + assertEquals(CLIENT_ID, ccProvider.clientAuth.getClientID().getValue()); + assertEquals(Scope.parse(SCOPE), ccProvider.scope); + } + + @Test + public void testCreateTokenExchangeTokenProviderWithAllOptions() throws SQLException { + String subjectTokenType = "urn:ietf:params:oauth:token-type:access_token"; + String actorToken = "actor-token-value"; + String actorTokenType = "urn:ietf:params:oauth:token-type:jwt"; + String audience = "https://api.example.com"; + String requestedTokenType = "urn:ietf:params:oauth:token-type:access_token"; + + OAuthConfiguration config = + new OAuthConfiguration.Builder() + .flow("token_exchange") + .tokenUri(TOKEN_URI) + .scope(SCOPE) + .clientId(CLIENT_ID) + .clientSecret(CLIENT_SECRET) + .resource(RESOURCE) + .subjectToken(SUBJECT_TOKEN) + .subjectTokenType(subjectTokenType) + .actorToken(actorToken) + .actorTokenType(actorTokenType) + .audience(audience) + .requestedTokenType(requestedTokenType) + .build(); + + OAuthTokenProvider provider = config.createTokenProvider(); + + assertNotNull(provider); + assertInstanceOf(TokenExchangeTokenProvider.class, provider); + + TokenExchangeTokenProvider teProvider = (TokenExchangeTokenProvider) provider; + assertEquals(URI.create(TOKEN_URI), teProvider.tokenUri); + assertNotNull(teProvider.grant); + assertEquals(SUBJECT_TOKEN, teProvider.grant.getSubjectToken().getValue()); + assertEquals(subjectTokenType, teProvider.grant.getSubjectTokenType().getURI().toString()); + assertEquals(actorToken, teProvider.grant.getActorToken().getValue()); + assertEquals(actorTokenType, teProvider.grant.getActorTokenType().getURI().toString()); + assertNotNull(teProvider.grant.getAudience()); + assertEquals(1, teProvider.grant.getAudience().size()); + assertEquals(audience, teProvider.grant.getAudience().get(0).getValue()); + assertEquals(requestedTokenType, teProvider.grant.getRequestedTokenType().getURI().toString()); + assertEquals(Scope.parse(SCOPE), teProvider.scope); + assertEquals(Collections.singletonList(URI.create(RESOURCE)), teProvider.resources); + + assertEquals(CLIENT_ID, teProvider.clientAuth.getClientID().getValue()); + } + + static Stream generalValidationErrorCases() { + return Stream.of( + Arguments.of( + Named.of( + "null flow", + (BuilderConfigurer) builder -> builder.flow((String) null).tokenUri(TOKEN_URI)), + "OAuth flow cannot be null or empty"), + Arguments.of( + Named.of( + "empty flow", (BuilderConfigurer) builder -> builder.flow("").tokenUri(TOKEN_URI)), + "OAuth flow cannot be null or empty"), + Arguments.of( + Named.of( + "invalid flow", + (BuilderConfigurer) builder -> builder.flow("invalid_flow").tokenUri(TOKEN_URI)), + "Unsupported OAuth flow: invalid_flow"), + Arguments.of( + Named.of( + "null tokenUri", + (BuilderConfigurer) + builder -> + builder + .flow("client_credentials") + .tokenUri((String) null) + .clientId(CLIENT_ID) + .clientSecret(CLIENT_SECRET)), + "Token URI cannot be null or empty"), + Arguments.of( + Named.of( + "empty tokenUri", + (BuilderConfigurer) + builder -> + builder + .flow("client_credentials") + .tokenUri("") + .clientId(CLIENT_ID) + .clientSecret(CLIENT_SECRET)), + "Token URI cannot be null or empty"), + Arguments.of( + Named.of( + "invalid tokenUri", + (BuilderConfigurer) + builder -> + builder + .flow("client_credentials") + .tokenUri("not a valid uri ://") + .clientId(CLIENT_ID) + .clientSecret(CLIENT_SECRET)), + null), + Arguments.of( + Named.of( + "invalid tokenUri", + (BuilderConfigurer) + builder -> + builder.flow("client_credentials").tokenUri(TOKEN_URI).clientId(CLIENT_ID)), + // null means verify exception has message and cause + "clientSecret is required for client_credentials flow")); + } + + @ParameterizedTest + @MethodSource("generalValidationErrorCases") + public void testGeneralValidationErrors(BuilderConfigurer configurer, String expectedMessage) { + SQLException exception = + assertThrows( + SQLException.class, + () -> { + OAuthConfiguration.Builder builder = new OAuthConfiguration.Builder(); + configurer.configure(builder); + builder.build(); + }); + + if (expectedMessage != null) { + assertEquals(expectedMessage, exception.getMessage()); + } else { + assertNotNull(exception.getMessage()); + assertNotNull(exception.getCause()); + } + } + + static Stream flowSpecificValidationErrorCases() { + return Stream.of( + // client_credentials flow validation + Arguments.of( + Named.of( + "client_credentials: missing clientId", + (BuilderConfigurer) + builder -> + builder + .flow("client_credentials") + .tokenUri(TOKEN_URI) + .clientSecret(CLIENT_SECRET)), + "clientId is required for client_credentials flow"), + Arguments.of( + Named.of( + "client_credentials: missing clientSecret", + (BuilderConfigurer) + builder -> + builder.flow("client_credentials").tokenUri(TOKEN_URI).clientId(CLIENT_ID)), + "clientSecret is required for client_credentials flow"), + // token_exchange flow validation + Arguments.of( + Named.of( + "token_exchange: missing subjectToken", + (BuilderConfigurer) builder -> builder.flow("token_exchange").tokenUri(TOKEN_URI)), + "subjectToken is required for token_exchange flow"), + Arguments.of( + Named.of( + "token_exchange: empty subjectToken", + (BuilderConfigurer) + builder -> + builder + .flow("token_exchange") + .tokenUri(TOKEN_URI) + .subjectToken("") + .subjectTokenType("urn:ietf:params:oauth:token-type:access_token")), + "subjectToken is required for token_exchange flow"), + Arguments.of( + Named.of( + "token_exchange: missing subjectTokenType", + (BuilderConfigurer) + builder -> + builder + .flow("token_exchange") + .tokenUri(TOKEN_URI) + .subjectToken(SUBJECT_TOKEN)), + "subjectTokenType is required for token_exchange flow"), + Arguments.of( + Named.of( + "token_exchange: empty subjectTokenType", + (BuilderConfigurer) + builder -> + builder + .flow("token_exchange") + .tokenUri(TOKEN_URI) + .subjectToken(SUBJECT_TOKEN) + .subjectTokenType("")), + "subjectTokenType is required for token_exchange flow")); + } + + @ParameterizedTest + @MethodSource("flowSpecificValidationErrorCases") + public void testFlowSpecificValidationErrors( + BuilderConfigurer configurer, String expectedMessage) { + SQLException exception = + assertThrows( + SQLException.class, + () -> { + OAuthConfiguration.Builder builder = new OAuthConfiguration.Builder(); + configurer.configure(builder); + builder.build(); + }); + + assertEquals(expectedMessage, exception.getMessage()); + } +} diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthCredentialWriterTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthCredentialWriterTest.java new file mode 100644 index 0000000000..1a33f7f0ae --- /dev/null +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/client/oauth/OAuthCredentialWriterTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.driver.jdbc.client.oauth; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.SQLException; +import org.apache.arrow.flight.CallHeaders; +import org.apache.arrow.flight.FlightCallHeaders; +import org.apache.arrow.flight.auth2.Auth2Constants; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +/** Tests for {@link OAuthCredentialWriter}. */ +@ExtendWith(MockitoExtension.class) +public class OAuthCredentialWriterTest { + + @Mock private OAuthTokenProvider mockTokenProvider; + + @Test + public void testConstructorRejectsNullTokenProvider() { + assertThrows(NullPointerException.class, () -> new OAuthCredentialWriter(null)); + } + + @Test + public void testAcceptWritesBearerTokenToHeaders() throws SQLException { + String testToken = "test-access-token-12345"; + when(mockTokenProvider.getValidToken()).thenReturn(testToken); + + OAuthCredentialWriter writer = new OAuthCredentialWriter(mockTokenProvider); + CallHeaders headers = new FlightCallHeaders(); + + writer.accept(headers); + + verify(mockTokenProvider).getValidToken(); + assertEquals( + Auth2Constants.BEARER_PREFIX + testToken, headers.get(Auth2Constants.AUTHORIZATION_HEADER)); + } + + @Test + public void testAcceptThrowsOAuthTokenExceptionOnSQLException() throws SQLException { + SQLException sqlException = new SQLException("Token fetch failed"); + when(mockTokenProvider.getValidToken()).thenThrow(sqlException); + + OAuthCredentialWriter writer = new OAuthCredentialWriter(mockTokenProvider); + CallHeaders headers = new FlightCallHeaders(); + + OAuthTokenException exception = + assertThrows(OAuthTokenException.class, () -> writer.accept(headers)); + + assertEquals("Failed to obtain OAuth token", exception.getMessage()); + assertEquals(sqlException, exception.getCause()); + } + + @Test + public void testAcceptCallsTokenProviderEachTime() throws SQLException { + when(mockTokenProvider.getValidToken()) + .thenReturn("token1") + .thenReturn("token2") + .thenReturn("token3"); + + OAuthCredentialWriter writer = new OAuthCredentialWriter(mockTokenProvider); + + CallHeaders headers1 = new FlightCallHeaders(); + writer.accept(headers1); + assertEquals("Bearer token1", headers1.get(Auth2Constants.AUTHORIZATION_HEADER)); + + CallHeaders headers2 = new FlightCallHeaders(); + writer.accept(headers2); + assertEquals("Bearer token2", headers2.get(Auth2Constants.AUTHORIZATION_HEADER)); + + CallHeaders headers3 = new FlightCallHeaders(); + writer.accept(headers3); + assertEquals("Bearer token3", headers3.get(Auth2Constants.AUTHORIZATION_HEADER)); + } +} diff --git a/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt b/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt index 8bc43cbe0f..8476bd9995 100644 --- a/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt +++ b/flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt @@ -345,6 +345,14 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- +This binary artifact contains Nimbus OAuth 2.0 SDK with OpenID Connect extensions 11.20.1. + +Copyright: Copyright 2012-2024 Connect2id Ltd. +Home page: https://connect2id.com/products/nimbus-oauth-openid-connect-sdk +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + This binary artifact contains Bouncycastle 1.80. Copyright: Copyright (c) 2000-2024 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org). diff --git a/flight/flight-sql-jdbc-driver/src/test/java/org/apache/arrow/driver/jdbc/ITDriverJarValidation.java b/flight/flight-sql-jdbc-driver/src/test/java/org/apache/arrow/driver/jdbc/ITDriverJarValidation.java index c1bd111fb9..145744ad38 100644 --- a/flight/flight-sql-jdbc-driver/src/test/java/org/apache/arrow/driver/jdbc/ITDriverJarValidation.java +++ b/flight/flight-sql-jdbc-driver/src/test/java/org/apache/arrow/driver/jdbc/ITDriverJarValidation.java @@ -70,6 +70,10 @@ public class ITDriverJarValidation { "LICENSE.txt", "NOTICE.txt", "arrow-git.properties", + "iso3166_1alpha2-codes.properties", + "iso3166_1alpha3-codes.properties", + "iso3166_1alpha-2-3-map.properties", + "iso3166_3-codes.properties", "properties/flight.properties", "META-INF/io.netty.versions.properties", "META-INF/MANIFEST.MF", From ccaac9ad688265d272d4c3d9d824426ef7681669 Mon Sep 17 00:00:00 2001 From: Kaustav Sarkar <70840177+Kaustav-Sarkar@users.noreply.github.com> Date: Thu, 22 Jan 2026 23:26:49 +0530 Subject: [PATCH 132/271] GH-125: Allow null timestamp holder sans timezone (#941) ## Description Fixes an `IllegalArgumentException` in `TimeStamp*TZVector.set/setSafe` when unsetting values using a holder with a `null` timezone. The validation logic now correctly ignores the timezone check when `holder.isSet <= 0`, allowing default-constructed holders to be used for unsetting values as expected. Comprehensive tests added for all timestamp precisions (Micro, Milli, Nano, Sec) to verify the fix and ensure the existing workaround (setting explicit timezone) remains supported. Closes #125 . --- .../arrow/vector/TimeStampMicroTZVector.java | 11 +- .../arrow/vector/TimeStampMilliTZVector.java | 11 +- .../arrow/vector/TimeStampNanoTZVector.java | 11 +- .../arrow/vector/TimeStampSecTZVector.java | 11 +- .../apache/arrow/vector/TestValueVector.java | 193 ++++++++++++++++++ 5 files changed, 217 insertions(+), 20 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java b/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java index abaefcfc12..50f2f066cc 100644 --- a/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java @@ -155,12 +155,13 @@ public void set(int index, NullableTimeStampMicroTZHolder holder) throws IllegalArgumentException { if (holder.isSet < 0) { throw new IllegalArgumentException(); - } else if (!this.timeZone.equals(holder.timezone)) { - throw new IllegalArgumentException( - String.format( - "holder.timezone: %s not equal to vector timezone: %s", - holder.timezone, this.timeZone)); } else if (holder.isSet > 0) { + if (!this.timeZone.equals(holder.timezone)) { + throw new IllegalArgumentException( + String.format( + "holder.timezone: %s not equal to vector timezone: %s", + holder.timezone, this.timeZone)); + } BitVectorHelper.setBit(validityBuffer, index); setValue(index, holder.value); } else { diff --git a/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java b/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java index b5e5fb1be1..9e4998396c 100644 --- a/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java @@ -155,12 +155,13 @@ public void set(int index, NullableTimeStampMilliTZHolder holder) throws IllegalArgumentException { if (holder.isSet < 0) { throw new IllegalArgumentException(); - } else if (!this.timeZone.equals(holder.timezone)) { - throw new IllegalArgumentException( - String.format( - "holder.timezone: %s not equal to vector timezone: %s", - holder.timezone, this.timeZone)); } else if (holder.isSet > 0) { + if (!this.timeZone.equals(holder.timezone)) { + throw new IllegalArgumentException( + String.format( + "holder.timezone: %s not equal to vector timezone: %s", + holder.timezone, this.timeZone)); + } BitVectorHelper.setBit(validityBuffer, index); setValue(index, holder.value); } else { diff --git a/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java b/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java index 2386b3a859..b44b3da8d3 100644 --- a/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java @@ -154,12 +154,13 @@ public Long getObject(int index) { public void set(int index, NullableTimeStampNanoTZHolder holder) throws IllegalArgumentException { if (holder.isSet < 0) { throw new IllegalArgumentException(); - } else if (!this.timeZone.equals(holder.timezone)) { - throw new IllegalArgumentException( - String.format( - "holder.timezone: %s not equal to vector timezone: %s", - holder.timezone, this.timeZone)); } else if (holder.isSet > 0) { + if (!this.timeZone.equals(holder.timezone)) { + throw new IllegalArgumentException( + String.format( + "holder.timezone: %s not equal to vector timezone: %s", + holder.timezone, this.timeZone)); + } BitVectorHelper.setBit(validityBuffer, index); setValue(index, holder.value); } else { diff --git a/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java b/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java index f1774f2703..a64a87f699 100644 --- a/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java @@ -150,12 +150,13 @@ public Long getObject(int index) { public void set(int index, NullableTimeStampSecTZHolder holder) throws IllegalArgumentException { if (holder.isSet < 0) { throw new IllegalArgumentException(); - } else if (!this.timeZone.equals(holder.timezone)) { - throw new IllegalArgumentException( - String.format( - "holder.timezone: %s not equal to vector timezone: %s", - holder.timezone, this.timeZone)); } else if (holder.isSet > 0) { + if (!this.timeZone.equals(holder.timezone)) { + throw new IllegalArgumentException( + String.format( + "holder.timezone: %s not equal to vector timezone: %s", + holder.timezone, this.timeZone)); + } BitVectorHelper.setBit(validityBuffer, index); setValue(index, holder.value); } else { diff --git a/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index ac82246671..df42d04e60 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -57,6 +57,10 @@ import org.apache.arrow.vector.complex.impl.UnionListViewWriter; import org.apache.arrow.vector.complex.impl.UnionListWriter; import org.apache.arrow.vector.holders.NullableIntHolder; +import org.apache.arrow.vector.holders.NullableTimeStampMicroTZHolder; +import org.apache.arrow.vector.holders.NullableTimeStampMilliTZHolder; +import org.apache.arrow.vector.holders.NullableTimeStampNanoTZHolder; +import org.apache.arrow.vector.holders.NullableTimeStampSecTZHolder; import org.apache.arrow.vector.holders.NullableUInt4Holder; import org.apache.arrow.vector.holders.NullableVarBinaryHolder; import org.apache.arrow.vector.holders.NullableVarCharHolder; @@ -2567,6 +2571,195 @@ public void testSetNullableVarCharHolderSafe() { } } + @Test + public void testTimeStampTZVectorSetSafeUnset() { + // reproduction of https://github.com/apache/arrow/issues/45084 + try (TimeStampMicroTZVector vector = new TimeStampMicroTZVector("vector", allocator, "UTC")) { + vector.allocateNew(); + // Set a valid value + NullableTimeStampMicroTZHolder validHolder = new NullableTimeStampMicroTZHolder(); + validHolder.isSet = 1; + validHolder.value = 1000L; + validHolder.timezone = "UTC"; + vector.setSafe(0, validHolder); + + assertEquals(1000L, vector.get(0)); + + // Unset the value using a holder with default (null) timezone + // The bug used to throw IllegalArgumentException because holder.timezone (null) != + // vector.timezone ("UTC") + // The correct behaviour is to not throw an exception and to unset the value. + NullableTimeStampMicroTZHolder unsetHolder = new NullableTimeStampMicroTZHolder(); + unsetHolder.isSet = 0; + vector.setSafe(0, unsetHolder); + + assertNull(vector.getObject(0)); + } + } + + @Test + public void testTimeStampMilliTZVectorSetSafeUnset() { + // reproduction of https://github.com/apache/arrow/issues/45084 + try (TimeStampMilliTZVector vector = new TimeStampMilliTZVector("vector", allocator, "UTC")) { + vector.allocateNew(); + + NullableTimeStampMilliTZHolder validHolder = new NullableTimeStampMilliTZHolder(); + validHolder.isSet = 1; + validHolder.value = 1000L; + validHolder.timezone = "UTC"; + vector.setSafe(0, validHolder); + + assertEquals(1000L, vector.get(0)); + + NullableTimeStampMilliTZHolder unsetHolder = new NullableTimeStampMilliTZHolder(); + unsetHolder.isSet = 0; + vector.setSafe(0, unsetHolder); + + assertNull(vector.getObject(0)); + } + } + + @Test + public void testTimeStampNanoTZVectorSetSafeUnset() { + // reproduction of https://github.com/apache/arrow/issues/45084 + try (TimeStampNanoTZVector vector = new TimeStampNanoTZVector("vector", allocator, "UTC")) { + vector.allocateNew(); + + NullableTimeStampNanoTZHolder validHolder = new NullableTimeStampNanoTZHolder(); + validHolder.isSet = 1; + validHolder.value = 1000L; + validHolder.timezone = "UTC"; + vector.setSafe(0, validHolder); + + assertEquals(1000L, vector.get(0)); + + NullableTimeStampNanoTZHolder unsetHolder = new NullableTimeStampNanoTZHolder(); + unsetHolder.isSet = 0; + vector.setSafe(0, unsetHolder); + + assertNull(vector.getObject(0)); + } + } + + @Test + public void testTimeStampSecTZVectorSetSafeUnset() { + // reproduction of https://github.com/apache/arrow/issues/45084 + try (TimeStampSecTZVector vector = new TimeStampSecTZVector("vector", allocator, "UTC")) { + vector.allocateNew(); + + NullableTimeStampSecTZHolder validHolder = new NullableTimeStampSecTZHolder(); + validHolder.isSet = 1; + validHolder.value = 1000L; + validHolder.timezone = "UTC"; + vector.setSafe(0, validHolder); + + assertEquals(1000L, vector.get(0)); + + NullableTimeStampSecTZHolder unsetHolder = new NullableTimeStampSecTZHolder(); + unsetHolder.isSet = 0; + vector.setSafe(0, unsetHolder); + + assertNull(vector.getObject(0)); + } + } + + @Test + public void testTimeStampMicroTZVectorSetSafeUnsetExplicitTimezone() { + // Test to ensure fix added for https://github.com/apache/arrow/issues/45084 does not break + // workaround. + try (TimeStampMicroTZVector vector = new TimeStampMicroTZVector("vector", allocator, "UTC")) { + vector.allocateNew(); + + NullableTimeStampMicroTZHolder validHolder = new NullableTimeStampMicroTZHolder(); + validHolder.isSet = 1; + validHolder.value = 1000L; + validHolder.timezone = "UTC"; + vector.setSafe(0, validHolder); + + assertEquals(1000L, vector.get(0)); + + NullableTimeStampMicroTZHolder unsetHolder = new NullableTimeStampMicroTZHolder(); + unsetHolder.isSet = 0; + unsetHolder.timezone = "UTC"; + + vector.setSafe(0, unsetHolder); + + assertNull(vector.getObject(0)); + } + } + + @Test + public void testTimeStampMilliTZVectorSetSafeUnsetExplicitTimezone() { + // Test to ensure fix added for https://github.com/apache/arrow/issues/45084 does not break + // workaround. + try (TimeStampMilliTZVector vector = new TimeStampMilliTZVector("vector", allocator, "UTC")) { + vector.allocateNew(); + + NullableTimeStampMilliTZHolder validHolder = new NullableTimeStampMilliTZHolder(); + validHolder.isSet = 1; + validHolder.value = 1000L; + validHolder.timezone = "UTC"; + vector.setSafe(0, validHolder); + + assertEquals(1000L, vector.get(0)); + + NullableTimeStampMilliTZHolder unsetHolder = new NullableTimeStampMilliTZHolder(); + unsetHolder.isSet = 0; + unsetHolder.timezone = "UTC"; + vector.setSafe(0, unsetHolder); + + assertNull(vector.getObject(0)); + } + } + + @Test + public void testTimeStampNanoTZVectorSetSafeUnsetExplicitTimezone() { + // Test to ensure fix added for https://github.com/apache/arrow/issues/45084 does not break + // workaround. + try (TimeStampNanoTZVector vector = new TimeStampNanoTZVector("vector", allocator, "UTC")) { + vector.allocateNew(); + + NullableTimeStampNanoTZHolder validHolder = new NullableTimeStampNanoTZHolder(); + validHolder.isSet = 1; + validHolder.value = 1000L; + validHolder.timezone = "UTC"; + vector.setSafe(0, validHolder); + + assertEquals(1000L, vector.get(0)); + + NullableTimeStampNanoTZHolder unsetHolder = new NullableTimeStampNanoTZHolder(); + unsetHolder.isSet = 0; + unsetHolder.timezone = "UTC"; + vector.setSafe(0, unsetHolder); + + assertNull(vector.getObject(0)); + } + } + + @Test + public void testTimeStampSecTZVectorSetSafeUnsetExplicitTimezone() { + // Test to ensure fix added for https://github.com/apache/arrow/issues/45084 does not break + // workaround. + try (TimeStampSecTZVector vector = new TimeStampSecTZVector("vector", allocator, "UTC")) { + vector.allocateNew(); + + NullableTimeStampSecTZHolder validHolder = new NullableTimeStampSecTZHolder(); + validHolder.isSet = 1; + validHolder.value = 1000L; + validHolder.timezone = "UTC"; + vector.setSafe(0, validHolder); + + assertEquals(1000L, vector.get(0)); + + NullableTimeStampSecTZHolder unsetHolder = new NullableTimeStampSecTZHolder(); + unsetHolder.isSet = 0; + unsetHolder.timezone = "UTC"; + vector.setSafe(0, unsetHolder); + + assertNull(vector.getObject(0)); + } + } + @Test public void testSetNullableVarBinaryHolder() { try (VarBinaryVector vector = new VarBinaryVector("", allocator)) { From 0f8a0808fd9cf0bd22d3c6b40a2016ee724ce185 Mon Sep 17 00:00:00 2001 From: Yicong Huang <17627829+Yicong-Huang@users.noreply.github.com> Date: Fri, 23 Jan 2026 01:18:13 -0800 Subject: [PATCH 133/271] GH-343: Fix ListVector offset buffer not properly serialized for nested empty arrays (#967) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What's Changed Fix `ListVector`/`LargeListVector` IPC serialization when `valueCount` is 0. ### Problem When `valueCount == 0`, `setReaderAndWriterIndex()` was setting `offsetBuffer.writerIndex(0)`, which means `readableBytes() == 0`. IPC serializer uses `readableBytes()` to determine buffer size, so 0 bytes were written to the IPC stream. This crashes IPC readers in other libraries because Arrow spec requires offset buffer to have at least one entry `[0]`. @viirya: > The offset buffers are allocated properly. But during IPC serialization, they are ignored. > ``` > public long readableBytes() { > return writerIndex - readerIndex; > } > ``` > So when ListVector.setReaderAndWriterIndex() sets writerIndex(0) and readerIndex(0), readableBytes() returns 0 - 0 = 0. > > Then when MessageSerializer.writeBatchBuffers() calls WriteChannel.write(buffer), it writes 0 bytes. > > So the flow is: > > valueCount=0 → ListVector.setReaderAndWriterIndex() sets offsetBuffer.writerIndex(0) > VectorUnloader.getFieldBuffers() returns the buffer with writerIndex=0 > MessageSerializer.writeBatchBuffers() writes the buffer > WriteChannel.write(buffer) checks buffer.readableBytes() which is 0 > 0 bytes are written to the IPC stream > PyArrow read the batch with the missing buffer → crash when other libraries to read ### Fix Simplify `setReaderAndWriterIndex()` to always use `(valueCount + 1) * OFFSET_WIDTH` for offset buffer's `writerIndex`. When `valueCount == 0`, this correctly sets `writerIndex` to `OFFSET_WIDTH`, ensuring `offset[0]` is included in serialization. ### Testing Added tests for nested empty lists verifying offset buffer has correct `readableBytes()`. Closes #343. --------- Co-authored-by: Yicong Huang --- .../arrow/vector/complex/LargeListVector.java | 7 +++++-- .../arrow/vector/complex/ListVector.java | 7 +++++-- .../arrow/vector/TestLargeListVector.java | 20 +++++++++++++++++++ .../apache/arrow/vector/TestListVector.java | 20 +++++++++++++++++++ 4 files changed, 50 insertions(+), 4 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java index 997b5a8b78..92dd3eaef7 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java @@ -309,11 +309,14 @@ private void setReaderAndWriterIndex() { offsetBuffer.readerIndex(0); if (valueCount == 0) { validityBuffer.writerIndex(0); - offsetBuffer.writerIndex(0); } else { validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); - offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH); } + // IPC serializer will determine readable bytes based on `readerIndex` and `writerIndex`. + // Both are set to 0 means 0 bytes are written to the IPC stream which will crash IPC readers + // in other libraries. According to Arrow spec, we should still output the offset buffer which + // is [0]. + offsetBuffer.writerIndex((long) (valueCount + 1) * OFFSET_WIDTH); } /** diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 93a313ef4f..6c3993df63 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -267,11 +267,14 @@ private void setReaderAndWriterIndex() { offsetBuffer.readerIndex(0); if (valueCount == 0) { validityBuffer.writerIndex(0); - offsetBuffer.writerIndex(0); } else { validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); - offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH); } + // IPC serializer will determine readable bytes based on `readerIndex` and `writerIndex`. + // Both are set to 0 means 0 bytes are written to the IPC stream which will crash IPC readers + // in other libraries. According to Arrow spec, we should still output the offset buffer which + // is [0]. + offsetBuffer.writerIndex((long) (valueCount + 1) * OFFSET_WIDTH); } /** diff --git a/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java index ccc0d3e176..bf9bba9c78 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java @@ -1100,6 +1100,26 @@ public void testCopyValueSafeForExtensionType() throws Exception { } } + @Test + public void testEmptyLargeListOffsetBuffer() { + // Test that LargeListVector has correct readableBytes after allocation. + // According to Arrow spec, offset buffer must have N+1 entries. + // Even when N=0, it should contain [0]. + try (LargeListVector list = LargeListVector.empty("list", allocator)) { + list.addOrGetVector(FieldType.nullable(MinorType.INT.getType())); + list.allocateNew(); + list.setValueCount(0); + + List buffers = list.getFieldBuffers(); + assertTrue( + buffers.get(1).readableBytes() >= LargeListVector.OFFSET_WIDTH, + "Offset buffer should have at least " + + LargeListVector.OFFSET_WIDTH + + " bytes for offset[0]"); + assertEquals(0L, list.getOffsetBuffer().getLong(0)); + } + } + private void writeIntValues(UnionLargeListWriter writer, int[] values) { writer.startList(); for (int v : values) { diff --git a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index 1fe4c59f63..0c90b32abc 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -1379,6 +1379,26 @@ public void testCopyValueSafeForExtensionType() throws Exception { } } + @Test + public void testEmptyListOffsetBuffer() { + // Test that ListVector has correct readableBytes after allocation. + // According to Arrow spec, offset buffer must have N+1 entries. + // Even when N=0, it should contain [0]. + try (ListVector list = ListVector.empty("list", allocator)) { + list.addOrGetVector(FieldType.nullable(MinorType.INT.getType())); + list.allocateNew(); + list.setValueCount(0); + + List buffers = list.getFieldBuffers(); + assertTrue( + buffers.get(1).readableBytes() >= BaseRepeatedValueVector.OFFSET_WIDTH, + "Offset buffer should have at least " + + BaseRepeatedValueVector.OFFSET_WIDTH + + " bytes for offset[0]"); + assertEquals(0, list.getOffsetBuffer().getInt(0)); + } + } + private void writeIntValues(UnionListWriter writer, int[] values) { writer.startList(); for (int v : values) { From 3325625662fa0412fccacc9d416289ac93b5b318 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jan 2026 08:49:41 +0100 Subject: [PATCH 134/271] MINOR: Bump org.apache.commons:commons-dbcp2 from 2.13.0 to 2.14.0 (#983) Bumps org.apache.commons:commons-dbcp2 from 2.13.0 to 2.14.0. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.commons:commons-dbcp2&package-manager=maven&previous-version=2.13.0&new-version=2.14.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---

Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql/pom.xml b/flight/flight-sql/pom.xml index 4175ff70d3..fe03406738 100644 --- a/flight/flight-sql/pom.xml +++ b/flight/flight-sql/pom.xml @@ -95,7 +95,7 @@ under the License. org.apache.commons commons-dbcp2 - 2.13.0 + 2.14.0 test From 096f582ae7e7f39af8f80f23470c4802a27aeac8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jan 2026 09:06:41 +0100 Subject: [PATCH 135/271] MINOR: Bump org.apache.commons:commons-compress from 1.27.1 to 1.28.0 (#985) Bumps [org.apache.commons:commons-compress](https://github.com/apache/commons-compress) from 1.27.1 to 1.28.0.
Changelog

Sourced from org.apache.commons:commons-compress's changelog.

Apache Commons Compress 1.28.0 Release Notes

The Apache Commons Compress team is pleased to announce the release of Apache Commons Compress 1.28.0.

Apache Commons Compress defines an API for working with compression and archive formats. These include bzip2, gzip, pack200, LZMA, XZ, Snappy, traditional Unix Compress, DEFLATE, DEFLATE64, LZ4, Brotli, Zstandard and ar, cpio, jar, tar, zip, dump, 7z, arj.

This is a feature and maintenance release. Java 8 or later is required.

This release updates Apache Commons Lang to 3.18.0 to pick up the fix for CVE-2025-48924 (https://nvd.nist.gov/vuln/detail/CVE-2025-48924), but is not affected by it.

Changes in this version

Changes in this version include the following.

New Features

  •  Add GzipParameters.getModificationInstant(). Thanks to Gary
Gregory.

  •  Add GzipParameters.setModificationInstant(Instant). Thanks
to Gary Gregory.

  •  Add GzipParameters.OS, setOS(OS), getOS(). Thanks to Gary
Gregory.

  •  Add GzipParameters.toString(). Thanks to Gary Gregory.
  • COMPRESS-638: Add GzipParameters.setFileNameCharset(Charset) and getFileNameCharset() to override the default ISO-8859-1 Charset #602. Thanks to vincexjl, Gary Gregory, Piotr P. Karwasz.
  •  Add support for gzip extra subfields, see
GzipParameters.setExtra(HeaderExtraField)
[#604](https://github.com/apache/commons-compress/issues/604). Thanks to
ddeschenes-1, Gary Gregory.

  •  Add CompressFilterOutputStream and refactor to use. Thanks
to Gary Gregory.

  •        Add ZipFile.stream(). Thanks to Gary Gregory.

  •  GzipCompressorInputStream reads the modification time
(MTIME) and stores its value incorrectly multiplied by 1,000. Thanks to
Danny Deschenes, Gary Gregory.

  •  GzipCompressorInputStream writes the modification time
(MTIME) the value incorrectly divided by 1,000. Thanks to Danny
Deschenes, Gary Gregory.

  •  Add optional FHCRC to GZIP header
[#627](https://github.com/apache/commons-compress/issues/627). Thanks to
Danny Deschenes, Gary Gregory.

  •  Add GzipCompressorInputStream.Builder allowing to customize
the file name and comment Charsets. Thanks to Gary Gregory.

  •  Add
GzipCompressorInputStream.Builder.setOnMemberStart(IOConsumer) to
monitor member parsing. Thanks to Gary Gregory.

  •  Add
GzipCompressorInputStream.Builder.setOnMemberEnd(IOConsumer) to monitor
member parsing. Thanks to Gary Gregory.

  •  Add PMD check to default Maven goal. Thanks to Gary Gregory.

  •  Add SevenZFile.Builder.setMaxMemoryLimitKiB(int). Thanks to
Gary Gregory.

  •  Add MemoryLimitException.MemoryLimitException(long, int,
Throwable) and deprecate MemoryLimitException.MemoryLimitException(long,
int, Exception). Thanks to Gary Gregory.
  • COMPRESS-692: Add support for zstd compression in zip archives. Thanks to Mehmet Karaman, Andrey Loskutov, Gary Gregory.
  •  Add support for XZ compression in ZIP archives. Thanks to
Gary Gregory.
  • COMPRESS-695: Add ZipArchiveInputStream.createZstdInputStream(InputStream) to provide a different InputStream implementation for Zstandard (Zstd) #649. Thanks to Gary Gregory.
  •  Add
org.apache.commons.compress.harmony.pack200.Pack200Exception.Pack200Exception(String,
Throwable). Thanks to Gary Gregory.
  • COMPRESS-697: Move BitStream.nextBit() method to BitInputStream #663. Thanks to Fredrik Kjellberg, Gary Gregory.
  •  Add
org.apache.commons.compress.compressors.lzma.LZMACompressorInputStream.builder/Builder().
Thanks to Gary Gregory.

  •  Add
org.apache.commons.compress.compressors.lzma.LZMACompressorOutputStream.builder/Builder().
Thanks to Gary Gregory.

  •  Add
org.apache.commons.compress.compressors.xz.XZCompressorInputStream.builder/Builder().
Thanks to Gary Gregory.

  •  Add
org.apache.commons.compress.compressors.xz.XZCompressorOutputStream.builder/Builder().
Thanks to Gary Gregory.

  •  Add
org.apache.commons.compress.compressors.xz.ZstdCompressorOutputStream.builder/Builder()
[#666](https://github.com/apache/commons-compress/issues/666). Thanks to
Gary Gregory, David Walluck, Piotr P. Karwasz.

  •  Add org.apache.commons.compress.compressors.xz.ZstdConstants
[#666](https://github.com/apache/commons-compress/issues/666). Thanks to
Gary Gregory, David Walluck, Piotr P. Karwasz.

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.commons:commons-compress&package-manager=maven&previous-version=1.27.1&new-version=1.28.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- compression/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compression/pom.xml b/compression/pom.xml index ba13156243..29f8b41788 100644 --- a/compression/pom.xml +++ b/compression/pom.xml @@ -50,7 +50,7 @@ under the License. org.apache.commons commons-compress - 1.27.1 + 1.28.0 com.github.luben From 3db456262fe00fc980f4846f31ed0e3cea1e6379 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Jan 2026 17:21:30 +0100 Subject: [PATCH 136/271] MINOR: Bump org.assertj:assertj-core from 3.27.3 to 3.27.7 (#988) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.assertj:assertj-core](https://github.com/assertj/assertj) from 3.27.3 to 3.27.7.
Release notes

Sourced from org.assertj:assertj-core's releases.

v3.27.7

:lock: Security

Core

:no_entry_sign: Deprecated

Core

  • Deprecate XmlStringPrettyFormatter with no replacement

:bug: Bug Fixes

Guava

  • Navigation to assertj-core or guava types from assertj-guava Javadoc site has unnecessary header #3478

:hammer: Dependency Upgrades

Core

  • Upgrade to Byte Buddy 1.18.3
  • Upgrade to JUnit BOM 5.14.1

Guava

  • Upgrade to Guava 33.5.0-jre

v3.27.6

:bug: Bug Fixes

Core

  • Add missing export for org.assertj.core.annotation #3951

:heart: Contributors

Thanks to all the contributors who worked on this release:

@​duponter

v3.27.5

:zap: Improvements

Core

  • ByteBuddy in AssertJ 3.27.4 not compatible with Java 25 #3946

... (truncated)

Commits
  • e840716 [maven-release-plugin] prepare release assertj-build-3.27.7
  • 85ca7eb Deprecate XmlStringPrettyFormatter
  • 77081dc Merge commit from fork
  • b68fc24 Bump github/codeql-action from 4.31.9 to 4.31.10 in the github-actions group ...
  • 0cf5bb6 Bump kotlin.version from 2.1.0 to 2.2.21
  • d393ef1 Abort tests when symbolic links cannot be created (#3788)
  • 2212433 Add IntelliJ custom inspection for test class names
  • 5717d02 Update JetBrains icon
  • a8ec20b Add icon for JetBrains products
  • c05fb3d Bump Maven to 3.9.12 and Wrapper to 3.3.4
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.assertj:assertj-core&package-manager=maven&previous-version=3.27.3&new-version=3.27.7)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/apache/arrow-java/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b3141d5cd8..51920947c8 100644 --- a/pom.xml +++ b/pom.xml @@ -175,7 +175,7 @@ under the License. org.assertj assertj-core - 3.27.3 + 3.27.7 test From 9b1f946db4927fa49c6ba4d713d7e83bd712a026 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Jan 2026 17:57:43 +0100 Subject: [PATCH 137/271] MINOR: Bump org.apache.commons:commons-pool2 from 2.12.1 to 2.13.1 (#987) Bumps org.apache.commons:commons-pool2 from 2.12.1 to 2.13.1. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.commons:commons-pool2&package-manager=maven&previous-version=2.12.1&new-version=2.13.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql/pom.xml b/flight/flight-sql/pom.xml index fe03406738..56c47f64dd 100644 --- a/flight/flight-sql/pom.xml +++ b/flight/flight-sql/pom.xml @@ -107,7 +107,7 @@ under the License. org.apache.commons commons-pool2 - 2.12.1 + 2.13.1 test From 923a5df0d668aa90f0d322f618314118ea0399e6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Jan 2026 18:26:34 +0100 Subject: [PATCH 138/271] MINOR: Bump logback.version from 1.5.25 to 1.5.26 (#981) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps `logback.version` from 1.5.25 to 1.5.26. Updates `ch.qos.logback:logback-classic` from 1.5.25 to 1.5.26
Release notes

Sourced from ch.qos.logback:logback-classic's releases.

Logback 1.5.26

2026-01-25 Release of logback version 1.5.26

• InsertFromJNDIModelHandler was accessing javax.naming package forcing the inclusion of the optional java.naming module. This problem was raised in issues/1003 by Marius Hanl who also provided the relevant PR.

• In applications using shadow/fat/shade jars, module or package information could be lost. Thus, in the absence of version information, logback-classic would warn about version mismatches. Logback components now ship with properties files containing version information that survive shadow/fat/shade jars. This issue was reporteed in issues/1002 by Christoph Gritschenberger.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit 33deb54506bbfaf1ff151f26f3a5f86936011619 associated with the tag v_1.5.26. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits
  • 33deb54 prepare release 1.5.26
  • d38a3e2 refactoring based on usage in logback-access
  • 4368333 move VersionUtil.getCoreVersionBySelfDeclaredProperties to CoreVersionUtil
  • 8bd5660 modify VersionCheckTest to use logback-core 1.5.25
  • 7a8f0b6 version information is self declared by modules.
  • 00d272f Do not use javax.naming namespace in the catch block, so that Logback can be ...
  • 420d67c mention country only, add missing 2016-03-29
  • 033aba4 fix javadoc errors
  • 6d52744 start work on 1.5.26-SNAPSHOT
  • See full diff in compare view

Updates `ch.qos.logback:logback-core` from 1.5.25 to 1.5.26
Release notes

Sourced from ch.qos.logback:logback-core's releases.

Logback 1.5.26

2026-01-25 Release of logback version 1.5.26

• InsertFromJNDIModelHandler was accessing javax.naming package forcing the inclusion of the optional java.naming module. This problem was raised in issues/1003 by Marius Hanl who also provided the relevant PR.

• In applications using shadow/fat/shade jars, module or package information could be lost. Thus, in the absence of version information, logback-classic would warn about version mismatches. Logback components now ship with properties files containing version information that survive shadow/fat/shade jars. This issue was reporteed in issues/1002 by Christoph Gritschenberger.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit 33deb54506bbfaf1ff151f26f3a5f86936011619 associated with the tag v_1.5.26. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits
  • 33deb54 prepare release 1.5.26
  • d38a3e2 refactoring based on usage in logback-access
  • 4368333 move VersionUtil.getCoreVersionBySelfDeclaredProperties to CoreVersionUtil
  • 8bd5660 modify VersionCheckTest to use logback-core 1.5.25
  • 7a8f0b6 version information is self declared by modules.
  • 00d272f Do not use javax.naming namespace in the catch block, so that Logback can be ...
  • 420d67c mention country only, add missing 2016-03-29
  • 033aba4 fix javadoc errors
  • 6d52744 start work on 1.5.26-SNAPSHOT
  • See full diff in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 51920947c8..4bab9b06c5 100644 --- a/pom.xml +++ b/pom.xml @@ -111,7 +111,7 @@ under the License. true 2.42.0 3.53.0 - 1.5.25 + 1.5.26 none -Xdoclint:none From 0eb50b5e840d7538d10478934929cc6b7ea40429 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Jan 2026 18:44:08 +0100 Subject: [PATCH 139/271] MINOR: Bump com.google.protobuf:protobuf-bom from 4.33.1 to 4.33.4 (#984) Bumps [com.google.protobuf:protobuf-bom](https://github.com/protocolbuffers/protobuf) from 4.33.1 to 4.33.4.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.protobuf:protobuf-bom&package-manager=maven&previous-version=4.33.1&new-version=4.33.4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4bab9b06c5..d64df1ade3 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,7 @@ under the License. 33.4.8-jre 4.2.9.Final 1.78.0 - 4.33.1 + 4.33.4 2.21.0 3.4.2 25.2.10 From ad59035ec880920f285158a140467d8b8d41789c Mon Sep 17 00:00:00 2001 From: Pedro Matias Date: Tue, 27 Jan 2026 22:25:29 +0000 Subject: [PATCH 140/271] GH-990: [JDBC] Fix memory leak on Connection#close due to unclosed ResultSet(s) (#991) ## What's Changed Closing a Connection when there was one or more unclosed ResultSet that had been obtained via methods of the interface DatabaseMetaData would generate exceptions due to memory leaks. Now, closing a Connection will first close all the ResultSet instances obtained from DatabaseMetadata instances associated with that Connection. Closes #990. --- .../driver/jdbc/ArrowFlightConnection.java | 32 +++++++++ .../ArrowFlightJdbcFlightStreamResultSet.java | 7 ++ ...owFlightJdbcVectorSchemaRootResultSet.java | 11 ++- .../arrow/driver/jdbc/ConnectionTest.java | 70 +++++++++++++++++++ 4 files changed, 113 insertions(+), 7 deletions(-) diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java index 0e9c198f52..623c2b81be 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightConnection.java @@ -21,6 +21,8 @@ import io.netty.util.concurrent.DefaultThreadFactory; import java.sql.SQLException; import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -42,6 +44,8 @@ public final class ArrowFlightConnection extends AvaticaConnection { private final ArrowFlightSqlClientHandler clientHandler; private final ArrowFlightConnectionConfigImpl config; private ExecutorService executorService; + private int metadataResultSetCount; + private Map metadataResultSetMap = new HashMap<>(); /** * Creates a new {@link ArrowFlightConnection}. @@ -66,6 +70,7 @@ private ArrowFlightConnection( this.config = Preconditions.checkNotNull(config, "Config cannot be null."); this.allocator = Preconditions.checkNotNull(allocator, "Allocator cannot be null."); this.clientHandler = Preconditions.checkNotNull(clientHandler, "Handler cannot be null."); + this.metadataResultSetCount = 0; } /** @@ -173,6 +178,31 @@ synchronized ExecutorService getExecutorService() { : executorService; } + /** + * Registers a new metadata ResultSet and assigns it a unique ID. Metadata ResultSets are those + * created without an associated Statement. + * + * @param resultSet the ResultSet to register + * @return the assigned ID + */ + int getNewMetadataResultSetId(ArrowFlightJdbcFlightStreamResultSet resultSet) { + metadataResultSetMap.put(metadataResultSetCount, resultSet); + return metadataResultSetCount++; + } + + /** + * Unregisters a metadata ResultSet when it is closed. This method is called by metadata + * ResultSets during their close operation to remove themselves from the tracking map. + * + * @param id the ID of the ResultSet to unregister, or null if not a metadata ResultSet + */ + void onResultSetClose(Integer id) { + if (id == null) { + return; + } + metadataResultSetMap.remove(id); + } + @Override public Properties getClientInfo() { final Properties copy = new Properties(); @@ -190,7 +220,9 @@ public void close() throws SQLException { } catch (final Exception e) { topLevelException = e; } + // copies of the collections are used to avoid concurrent modification problems ArrayList closeables = new ArrayList<>(statementMap.values()); + closeables.addAll(new ArrayList<>(metadataResultSetMap.values())); closeables.add(clientHandler); closeables.addAll(allocator.getChildAllocators()); closeables.add(allocator); diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcFlightStreamResultSet.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcFlightStreamResultSet.java index aabaf01e63..2885f7895b 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcFlightStreamResultSet.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcFlightStreamResultSet.java @@ -54,6 +54,7 @@ public final class ArrowFlightJdbcFlightStreamResultSet private VectorSchemaRoot currentVectorSchemaRoot; private Schema schema; + private Integer id = null; // used for metadata result sets only /** Public constructor used by ArrowFlightJdbcFactory. */ ArrowFlightJdbcFlightStreamResultSet( @@ -82,6 +83,7 @@ private ArrowFlightJdbcFlightStreamResultSet( super(null, state, signature, resultSetMetaData, timeZone, firstFrame); this.connection = connection; this.flightInfo = flightInfo; + this.id = connection.getNewMetadataResultSetId(this); } /** @@ -234,7 +236,12 @@ protected void cancel() { @Override public synchronized void close() { + try { + if (isClosed()) { + return; + } + this.connection.onResultSetClose(id); if (flightEndpointDataQueue != null) { // flightStreamQueue should close currentFlightStream internally flightEndpointDataQueue.close(); diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java index 622e5fe7f6..49334951de 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java @@ -22,7 +22,6 @@ import java.sql.Types; import java.util.HashSet; import java.util.List; -import java.util.Objects; import java.util.Set; import java.util.TimeZone; import org.apache.arrow.driver.jdbc.utils.ConvertUtils; @@ -159,12 +158,10 @@ public void close() { } catch (final Exception e) { exceptions.add(e); } - if (!Objects.isNull(statement)) { - try { - super.close(); - } catch (final Exception e) { - exceptions.add(e); - } + try { + super.close(); + } catch (final Exception e) { + exceptions.add(e); } exceptions.parallelStream().forEach(e -> LOGGER.error(e.getMessage(), e)); exceptions.stream() diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java index dbedbe9d36..55722f60fb 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionTest.java @@ -16,6 +16,8 @@ */ package org.apache.arrow.driver.jdbc; +import static java.lang.String.format; +import static java.util.stream.IntStream.range; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -23,24 +25,33 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +import com.google.protobuf.Message; import java.net.URISyntaxException; import java.sql.Connection; import java.sql.Driver; import java.sql.DriverManager; +import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.Map; import java.util.Properties; +import java.util.function.Consumer; import org.apache.arrow.driver.jdbc.authentication.UserPasswordAuthentication; import org.apache.arrow.driver.jdbc.client.ArrowFlightSqlClientHandler; import org.apache.arrow.driver.jdbc.utils.ArrowFlightConnectionConfigImpl.ArrowFlightConnectionProperty; import org.apache.arrow.driver.jdbc.utils.MockFlightSqlProducer; import org.apache.arrow.flight.FlightMethod; +import org.apache.arrow.flight.FlightProducer.ServerStreamListener; import org.apache.arrow.flight.NoOpSessionOptionValueVisitor; import org.apache.arrow.flight.SessionOptionValue; +import org.apache.arrow.flight.sql.FlightSqlProducer.Schemas; +import org.apache.arrow.flight.sql.impl.FlightSql.CommandGetTableTypes; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.util.AutoCloseables; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.util.Text; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -698,4 +709,63 @@ public void testStatementsClosedOnConnectionClose() throws Exception { assertTrue(statements[i].isClosed()); } } + + @Test + public void testResultSetsFromDatabaseMetadataClosedOnConnectionClose() throws Exception { + // set up the FlightProducer to respond to metadata queries + // getTableTypes() is being used, but any other method would work + int rowCount = 3; + final Message commandGetTableTypes = CommandGetTableTypes.getDefaultInstance(); + final Consumer commandGetTableTypesResultProducer = + listener -> { + try (final BufferAllocator allocator = new RootAllocator(); + final VectorSchemaRoot root = + VectorSchemaRoot.create(Schemas.GET_TABLE_TYPES_SCHEMA, allocator)) { + final VarCharVector tableType = (VarCharVector) root.getVector("table_type"); + range(0, rowCount) + .forEach(i -> tableType.setSafe(i, new Text(format("table_type #%d", i)))); + root.setRowCount(rowCount); + listener.start(root); + listener.putNext(); + } catch (final Throwable throwable) { + listener.error(throwable); + } finally { + listener.completed(); + } + }; + PRODUCER.addCatalogQuery(commandGetTableTypes, commandGetTableTypesResultProducer); + + // create a connection + final Properties properties = new Properties(); + properties.put(ArrowFlightConnectionProperty.HOST.camelName(), "localhost"); + properties.put( + ArrowFlightConnectionProperty.PORT.camelName(), FLIGHT_SERVER_TEST_EXTENSION.getPort()); + properties.put(ArrowFlightConnectionProperty.USER.camelName(), userTest); + properties.put(ArrowFlightConnectionProperty.PASSWORD.camelName(), passTest); + properties.put("useEncryption", false); + + Connection connection = + DriverManager.getConnection( + "jdbc:arrow-flight-sql://" + + FLIGHT_SERVER_TEST_EXTENSION.getHost() + + ":" + + FLIGHT_SERVER_TEST_EXTENSION.getPort(), + properties); + + // create ResultSets from DatabaseMetadata + int numResultSets = 3; + ResultSet[] resultSets = new ResultSet[numResultSets]; + for (int i = 0; i < numResultSets; i++) { + resultSets[i] = connection.getMetaData().getTableTypes(); + assertFalse(resultSets[i].isClosed()); + } + + // close the connection + connection.close(); + + // assert the ResultSets are closed + for (int i = 0; i < numResultSets; i++) { + assertTrue(resultSets[i].isClosed()); + } + } } From ce1f3d75b25038af068b62cb9ebd35817b7a04a3 Mon Sep 17 00:00:00 2001 From: Tamas Mate <50709850+tmater@users.noreply.github.com> Date: Wed, 28 Jan 2026 22:58:30 +0100 Subject: [PATCH 141/271] GH-993: Fix missing pipe in milestone assignment script (#992) The `head -n1` command was not piped to grep output, causing all matching milestones to be captured instead of just the first one. Example failure: ``` Assigning milestone: 19.0.0 20.0.0 '19.0.0 20.0.0' not found ``` Closes #993 --- .github/workflows/dev_pr_milestone.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dev_pr_milestone.sh b/.github/workflows/dev_pr_milestone.sh index b6876b4b08..4a77eb1f73 100755 --- a/.github/workflows/dev_pr_milestone.sh +++ b/.github/workflows/dev_pr_milestone.sh @@ -37,8 +37,8 @@ main() { local -r milestone=$( gh api "/repos/${repo}/milestones" | jq --raw-output '.[] | .title' | - grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' - head -n1 + grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | + head -n1 ) echo "Assigning milestone: ${milestone}" From b3113ab797020a8cfd6ea83ab3c549eb808b2d09 Mon Sep 17 00:00:00 2001 From: Aleksei Starikov Date: Sun, 8 Feb 2026 14:07:13 +0100 Subject: [PATCH 142/271] GH-1011: [Docs] Fix broken Java API reference links in documentation (#1012) ## What's Changed Fix Java API references in docs. For example: https://arrow.apache.org/docs/java/reference/org/apache/arrow/flight/FlightClient.html -> https://arrow.apache.org/java/current/reference/org.apache.arrow.flight.core/org/apache/arrow/flight/FlightClient.html Closes #1011. --- docs/source/flight.rst | 10 +++++----- docs/source/flight_sql.rst | 2 +- docs/source/jdbc.rst | 6 +++--- docs/source/memory.rst | 18 +++++++++--------- docs/source/table.rst | 24 ++++++++++++------------ docs/source/vector_schema_root.rst | 16 ++++++++-------- 6 files changed, 38 insertions(+), 38 deletions(-) diff --git a/docs/source/flight.rst b/docs/source/flight.rst index fabced8094..fd0fdf07bc 100644 --- a/docs/source/flight.rst +++ b/docs/source/flight.rst @@ -232,8 +232,8 @@ Servers can add other gRPC services. For example, to add the `Health Check servi See the :external+arrow:ref:`best practices for C++ `. -.. _`FlightClient`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/flight/FlightClient.html -.. _`FlightProducer`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/flight/FlightProducer.html -.. _`FlightServer`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/flight/FlightServer.html -.. _`NoOpFlightProducer`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/flight/NoOpFlightProducer.html -.. _`Location`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/flight/Location.html +.. _`FlightClient`: https://arrow.apache.org/java/current/reference/org.apache.arrow.flight.core/org/apache/arrow/flight/FlightClient.html +.. _`FlightProducer`: https://arrow.apache.org/java/current/reference/org.apache.arrow.flight.core/org/apache/arrow/flight/FlightProducer.html +.. _`FlightServer`: https://arrow.apache.org/java/current/reference/org.apache.arrow.flight.core/org/apache/arrow/flight/FlightServer.html +.. _`NoOpFlightProducer`: https://arrow.apache.org/java/current/reference/org.apache.arrow.flight.core/org/apache/arrow/flight/NoOpFlightProducer.html +.. _`Location`: https://arrow.apache.org/java/current/reference/org.apache.arrow.flight.core/org/apache/arrow/flight/Location.html diff --git a/docs/source/flight_sql.rst b/docs/source/flight_sql.rst index 169a0e24bf..09ce1dda0d 100644 --- a/docs/source/flight_sql.rst +++ b/docs/source/flight_sql.rst @@ -29,4 +29,4 @@ over the network. For usage information, see the `API documentation`_. -.. _API documentation: https://arrow.apache.org/docs/java/reference/org/apache/arrow/flight/sql/package-summary.html +.. _API documentation: https://arrow.apache.org/java/current/reference/org.apache.arrow.flight.sql/org/apache/arrow/flight/sql/package-summary.html diff --git a/docs/source/jdbc.rst b/docs/source/jdbc.rst index a4c95dbf00..2f57c34bf8 100644 --- a/docs/source/jdbc.rst +++ b/docs/source/jdbc.rst @@ -95,7 +95,7 @@ Type Mapping The JDBC to Arrow type mapping can be obtained at runtime from `JdbcToArrowUtils.getArrowTypeFromJdbcType`_. -.. _JdbcToArrowUtils.getArrowTypeFromJdbcType: https://arrow.apache.org/docs/java/reference/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.html#getArrowTypeFromJdbcType-org.apache.arrow.adapter.jdbc.JdbcFieldInfo-java.util.Calendar- +.. _JdbcToArrowUtils.getArrowTypeFromJdbcType: https://arrow.apache.org/java/current/reference/org.apache.arrow.adapter.jdbc/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.html#getArrowTypeFromJdbcType-org.apache.arrow.adapter.jdbc.JdbcFieldInfo-java.util.Calendar- +--------------------+--------------------+-------+ | JDBC Type | Arrow Type | Notes | @@ -171,8 +171,8 @@ The JDBC to Arrow type mapping can be obtained at runtime from timezone of the calendar, else it will be a timestamp without timezone. -.. _setArraySubTypeByColumnIndexMap: https://arrow.apache.org/docs/java/reference/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigBuilder.html#setArraySubTypeByColumnIndexMap-java.util.Map- -.. _setArraySubTypeByColumnNameMap: https://arrow.apache.org/docs/java/reference/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigBuilder.html#setArraySubTypeByColumnNameMap-java.util.Map- +.. _setArraySubTypeByColumnIndexMap: https://arrow.apache.org/java/current/reference/org.apache.arrow.adapter.jdbc/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigBuilder.html#setArraySubTypeByColumnIndexMap-java.util.Map- +.. _setArraySubTypeByColumnNameMap: https://arrow.apache.org/java/current/reference/org.apache.arrow.adapter.jdbc/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigBuilder.html#setArraySubTypeByColumnNameMap-java.util.Map- .. _ARROW-17006: https://issues.apache.org/jira/browse/ARROW-17006 VectorSchemaRoot to PreparedStatement Parameter Conversion diff --git a/docs/source/memory.rst b/docs/source/memory.rst index 58ef382dc9..4a71ed846a 100644 --- a/docs/source/memory.rst +++ b/docs/source/memory.rst @@ -333,18 +333,18 @@ How this works: } } -.. _`ArrowBuf`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/ArrowBuf.html -.. _`ArrowBuf.print()`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/ArrowBuf.html#print-java.lang.StringBuilder-int-org.apache.arrow.memory.BaseAllocator.Verbosity- -.. _`BufferAllocator`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/BufferAllocator.html -.. _`BufferLedger`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/BufferLedger.html -.. _`RootAllocator`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/RootAllocator.html -.. _`newChildAllocator`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/RootAllocator.html#newChildAllocator-java.lang.String-org.apache.arrow.memory.AllocationListener-long-long- +.. _`ArrowBuf`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/ArrowBuf.html +.. _`ArrowBuf.print()`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/ArrowBuf.html#print-java.lang.StringBuilder-int-org.apache.arrow.memory.BaseAllocator.Verbosity- +.. _`BufferAllocator`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/BufferAllocator.html +.. _`BufferLedger`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/BufferLedger.html +.. _`RootAllocator`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/RootAllocator.html +.. _`newChildAllocator`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/RootAllocator.html#newChildAllocator-java.lang.String-org.apache.arrow.memory.AllocationListener-long-long- .. _`Netty`: https://netty.io/wiki/ .. _`sun.misc.unsafe`: https://web.archive.org/web/20210929024401/http://www.docjar.com/html/api/sun/misc/Unsafe.java.html .. _`Direct Memory`: https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/nio/ByteBuffer.html -.. _`ReferenceManager`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/ReferenceManager.html -.. _`ReferenceManager.release`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/ReferenceManager.html#release-- -.. _`ReferenceManager.retain`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/ReferenceManager.html#retain-- +.. _`ReferenceManager`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/ReferenceManager.html +.. _`ReferenceManager.release`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/ReferenceManager.html#release-- +.. _`ReferenceManager.retain`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/ReferenceManager.html#retain-- Arrow Memory In-Depth ===================== diff --git a/docs/source/table.rst b/docs/source/table.rst index 5aa95e153c..880ef84d29 100644 --- a/docs/source/table.rst +++ b/docs/source/table.rst @@ -364,15 +364,15 @@ If the table contains dictionary-encoded vectors and was constructed with a ``Di Data.exportTable(bufferAllocator, table, outArrowArray); -.. _`ArrowBuf`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/memory/ArrowBuf.html -.. _`Data`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/c/Data.html -.. _`DictionaryProvider`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/dictionary/DictionaryProvider.html -.. _`Field`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/types/pojo/Field.html -.. _`FieldReader`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/complex/reader/FieldReader.html -.. _`FieldVector`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/FieldVector.html -.. _`Row`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/table/Row.html -.. _`Schema`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/types/pojo/Schema.html -.. _`Table`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/table/Table.html -.. _`ValueHolder`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/holders/ValueHolder.html -.. _`ValueVector`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/ValueVector.html -.. _`VectorSchemaRoot`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/VectorSchemaRoot.html +.. _`ArrowBuf`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/ArrowBuf.html +.. _`Data`: https://arrow.apache.org/java/current/reference/org.apache.arrow.c/org/apache/arrow/c/Data.html +.. _`DictionaryProvider`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/dictionary/DictionaryProvider.html +.. _`Field`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/types/pojo/Field.html +.. _`FieldReader`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/complex/reader/FieldReader.html +.. _`FieldVector`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/FieldVector.html +.. _`Row`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/table/Row.html +.. _`Schema`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/types/pojo/Schema.html +.. _`Table`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/table/Table.html +.. _`ValueHolder`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/holders/ValueHolder.html +.. _`ValueVector`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/ValueVector.html +.. _`VectorSchemaRoot`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/VectorSchemaRoot.html diff --git a/docs/source/vector_schema_root.rst b/docs/source/vector_schema_root.rst index 3119122d9a..f4a497c4e5 100644 --- a/docs/source/vector_schema_root.rst +++ b/docs/source/vector_schema_root.rst @@ -153,11 +153,11 @@ A `Table`_ is an immutable tabular data structure, very similar to VectorSchemaR See the :doc:`table` documentation for more information. -.. _`ArrowRecordBatch`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/ipc/message/ArrowRecordBatch.html -.. _`Field`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/types/pojo/Field.html -.. _`Flight`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/flight/package-summary.html -.. _`Schema`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/types/pojo/Schema.html -.. _`Table`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/table/Table.html -.. _`VectorLoader`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/VectorLoader.html -.. _`VectorSchemaRoot`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/VectorSchemaRoot.html -.. _`VectorUnloader`: https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/VectorUnloader.html +.. _`ArrowRecordBatch`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/ipc/message/ArrowRecordBatch.html +.. _`Field`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/types/pojo/Field.html +.. _`Flight`: https://arrow.apache.org/java/current/reference/org.apache.arrow.flight.core/org/apache/arrow/flight/package-summary.html +.. _`Schema`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/types/pojo/Schema.html +.. _`Table`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/table/Table.html +.. _`VectorLoader`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/VectorLoader.html +.. _`VectorSchemaRoot`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/VectorSchemaRoot.html +.. _`VectorUnloader`: https://arrow.apache.org/java/current/reference/org.apache.arrow.vector/org/apache/arrow/vector/VectorUnloader.html From 776466e904f3ae44f52c3baa019e795e8a68c527 Mon Sep 17 00:00:00 2001 From: Aleksei Starikov Date: Sun, 8 Feb 2026 14:08:20 +0100 Subject: [PATCH 143/271] GH-141: Correct capacity behavior in BufferAllocator.buffer docstrings (#1010) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What's Changed Update the `BufferAllocator.buffer(long)` and `BufferAllocator.buffer(long, BufferManager)` docstrings so they match the actual behavior: the returned buffer’s capacity is the allocated (possibly rounded) size, not the requested size. The previous text said the capacity would be set to the configured size, which was incorrect. The new text also mentions that callers can use `ArrowBuf#capacity(long)` to set the capacity to the requested size when needed. Documentation-only change; no code or behavioral changes. Closes #141. --- .../java/org/apache/arrow/memory/ArrowBuf.java | 2 +- .../org/apache/arrow/memory/BufferAllocator.java | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java b/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java index 775a8925ad..b8012fe643 100644 --- a/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java +++ b/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java @@ -136,7 +136,7 @@ public long capacity() { /** * Adjusts the capacity of this buffer. Size increases are NOT supported. * - * @param newCapacity Must be in in the range [0, length). + * @param newCapacity Must be in the range [0, length). */ public synchronized ArrowBuf capacity(long newCapacity) { diff --git a/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferAllocator.java b/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferAllocator.java index 4f9d3c61c6..dbd6da3291 100644 --- a/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferAllocator.java +++ b/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferAllocator.java @@ -25,9 +25,10 @@ public interface BufferAllocator extends AutoCloseable { /** - * Allocate a new or reused buffer of the provided size. Note that the buffer may technically be - * larger than the requested size for rounding purposes. However, the buffer's capacity will be - * set to the configured size. + * Allocate a new or reused buffer of the provided size. The buffer may be larger than the + * requested size for rounding purposes (e.g. to a power of two), and the buffer's capacity will + * reflect the actual allocated size. Use {@link ArrowBuf#capacity(long)} to set the capacity to + * the requested size if needed. * * @param size The size in bytes. * @return a new ArrowBuf, or null if the request can't be satisfied @@ -36,9 +37,10 @@ public interface BufferAllocator extends AutoCloseable { ArrowBuf buffer(long size); /** - * Allocate a new or reused buffer of the provided size. Note that the buffer may technically be - * larger than the requested size for rounding purposes. However, the buffer's capacity will be - * set to the configured size. + * Allocate a new or reused buffer of the provided size. The buffer may be larger than the + * requested size for rounding purposes (e.g. to a power of two), and the buffer's capacity will + * reflect the actual allocated size. Use {@link ArrowBuf#capacity(long)} to set the capacity to + * the requested size if needed. * * @param size The size in bytes. * @param manager A buffer manager to manage reallocation. From 60a1a424200675710cc002c87b5a9a3afc6bada5 Mon Sep 17 00:00:00 2001 From: Aleksei Starikov Date: Wed, 11 Feb 2026 12:00:23 +0100 Subject: [PATCH 144/271] GH-1014: [Docs] Fix broken and obsolete links in the README.md (#1015) ## What's Changed Fix broken links in `README.md` file. Remove the unused reference [2]: https://github.com/apache/arrow/blob/main/cpp/README.md. Inline footnote links. Closes #1014. --- README.md | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index c46b61c49e..32a7e82811 100644 --- a/README.md +++ b/README.md @@ -23,11 +23,11 @@ The following guides explain the fundamental data structures used in the Java implementation of Apache Arrow. -- [ValueVector](https://arrow.apache.org/docs/java/vector.html) is an abstraction that is used to store a sequence of values having the same type in an individual column. -- [VectorSchemaRoot](https://arrow.apache.org/docs/java/vector_schema_root.html) is a container that can hold multiple vectors based on a schema. -- The [Reading/Writing IPC formats](https://arrow.apache.org/docs/java/ipc.html) guide explains how to stream record batches as well as serializing record batches to files. +- [ValueVector](https://arrow.apache.org/java/current/vector.html) is an abstraction that is used to store a sequence of values having the same type in an individual column. +- [VectorSchemaRoot](https://arrow.apache.org/java/current/vector_schema_root.html#vectorschemaroot) is a container that can hold multiple vectors based on a schema. +- The [Reading/Writing IPC formats](https://arrow.apache.org/java/current/ipc.html) guide explains how to stream record batches as well as serializing record batches to files. -Generated javadoc documentation is available [here](https://arrow.apache.org/docs/java/). +Generated javadoc documentation is available [here](https://arrow.apache.org/java/current/). ## Building from source @@ -93,7 +93,7 @@ conflicting or duplicate fields set this JVM flag or use the correct static cons ## Java Code Style Guide -Arrow Java follows the Google style guide [here][3] with the following +Arrow Java follows the [Google Java Style Guide](http://google.github.io/styleguide/javaguide.html) with the following differences: * Imports are grouped, from top to bottom, in this order: static imports, @@ -119,12 +119,12 @@ following command run in the project root directory: mvn -Dlogback.configurationFile=file: ``` -See [Logback Configuration][1] for more details. +See [Logback Configuration](https://logback.qos.ch/manual/configuration.html) for more details. ## Integration Tests Integration tests which require more time or more memory can be run by activating -the `integration-tests` profile. This activates the [maven failsafe][4] plugin +the `integration-tests` profile. This activates the [Maven Failsafe](https://maven.apache.org/surefire/maven-failsafe-plugin/) plugin and any class prefixed with `IT` will be run during the testing phase. The integration tests currently require a larger amount of memory (>4GB) and time to complete. To activate the profile: @@ -133,7 +133,3 @@ the profile: mvn -Pintegration-tests ``` -[1]: https://logback.qos.ch/manual/configuration.html -[2]: https://github.com/apache/arrow/blob/main/cpp/README.md -[3]: http://google.github.io/styleguide/javaguide.html -[4]: https://maven.apache.org/surefire/maven-failsafe-plugin/ From 2b74309ba4d5844593d6a3191d7b958f18468ae9 Mon Sep 17 00:00:00 2001 From: Aleksei Starikov Date: Mon, 16 Feb 2026 11:42:29 +0100 Subject: [PATCH 145/271] MINOR: [Docs] Remove extra line in README.md (fix pre-commit) (#1018) ## What's Changed Remove extra line in README.md for fixing the `end-of-file-fixer` hook in the `pre-commit` build. E.g. [here](https://github.com/apache/arrow-java/actions/runs/21902454512/job/63233962122) in the main branch build. ``` fix end of files.........................................................Failed - hook id: end-of-file-fixer - exit code: 1 - files were modified by this hook Fixing README.md ``` --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 32a7e82811..b0715aadf1 100644 --- a/README.md +++ b/README.md @@ -132,4 +132,3 @@ the profile: ```bash mvn -Pintegration-tests ``` - From bc7132b92c9c8ab693e5264cddf5145a3eed902e Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 16 Feb 2026 22:02:33 +0900 Subject: [PATCH 146/271] GH-1021: Use released apache/arrow instead of main (#1022) ## What's Changed In general, we should use released apache/arrow for apache/arrow-java release. Closes #1021. --- .github/workflows/rc.yml | 44 +++++++++++++++------------------------- 1 file changed, 16 insertions(+), 28 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index efa69533d3..37b2209966 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -107,18 +107,12 @@ jobs: - name: Extract source archive run: | tar -xf apache-arrow-java-*.tar.gz --strip-components=1 - # We always use the main branch for apache/arrow for now. - # Because we want to use - # https://github.com/apache/arrow/pull/45114 in - # apache/arrow-java. We can revert this workaround once Apache - # Arrow 20.0.0 that includes the change released. - # - # - name: Download the latest Apache Arrow C++ - # if: github.event_name != 'schedule' - # run: | - # ci/scripts/download_cpp.sh + - name: Download the latest Apache Arrow C++ + if: github.event_name != 'schedule' + run: | + ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ - # if: github.event_name == 'schedule' + if: github.event_name == 'schedule' uses: actions/checkout@v6 with: repository: apache/arrow @@ -180,12 +174,12 @@ jobs: - name: Extract source archive run: | tar -xf apache-arrow-java-*.tar.gz --strip-components=1 - # - name: Download the latest Apache Arrow C++ - # if: github.event_name != 'schedule' - # run: | - # ci/scripts/download_cpp.sh + - name: Download the latest Apache Arrow C++ + if: github.event_name != 'schedule' + run: | + ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ - # if: github.event_name == 'schedule' + if: github.event_name == 'schedule' uses: actions/checkout@v6 with: repository: apache/arrow @@ -309,19 +303,13 @@ jobs: shell: bash run: | tar -xf apache-arrow-java-*.tar.gz --strip-components=1 - # We always use the main branch for apache/arrow for now. - # Because we want to use - # https://github.com/apache/arrow/pull/47749 in - # apache/arrow-java. We can revert this workaround once Apache - # Arrow 22.0.0 that includes the change released. - # - # - name: Download the latest Apache Arrow C++ - # if: github.event_name != 'schedule' - # shell: bash - # run: | - # ci/scripts/download_cpp.sh + - name: Download the latest Apache Arrow C++ + if: github.event_name != 'schedule' + shell: bash + run: | + ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ - # if: github.event_name == 'schedule' + if: github.event_name == 'schedule' uses: actions/checkout@v6 with: repository: apache/arrow From 6dbc5d3690168e381f6b70c94638a9907c5489b3 Mon Sep 17 00:00:00 2001 From: Tamas Mate <50709850+tmater@users.noreply.github.com> Date: Tue, 17 Feb 2026 06:16:03 +0100 Subject: [PATCH 147/271] GH-946: Add Variant extension type support (#947) ### Summary This PR adds support for the Variant extension type in Arrow Java, enabling storage and manipulation of semi-structured variant data with metadata and value buffers. ### Changes A new `arrow-variant` module introduces the `Variant` class for parsing and working with variant data. This module is separated from the core vector module to isolate the `parquet-variant` dependency, so users of the Arrow vector library don't have to depend on Parquet. This also maintains a clean API boundary between Arrow's core functionality and variant-specific parsing logic. The core vector module gains `VariantType` as an extension type along with `VariantVector` for storing variant data as metadata/value buffer pairs. The implementation includes reader and writer support through `VariantReaderImpl`, `VariantWriterImpl`, and `NullableVariantHolderReaderImpl`, with corresponding holder classes for use in generated code paths. ### Testing - Unit tests for `VariantType`, `VariantVector`, and `Variant` parsing - Integration tests with `ListVector` and `MapVector` - Extension type round-trip tests Closes #946 --- arrow-variant/pom.xml | 51 ++ arrow-variant/src/main/java/module-info.java | 28 + .../org/apache/arrow/variant/Variant.java | 217 +++++ .../arrow/variant/extension/VariantType.java | 93 ++ .../variant/extension/VariantVector.java | 348 ++++++++ .../holders/NullableVariantHolder.java | 56 ++ .../arrow/variant/holders/VariantHolder.java | 56 ++ .../impl/NullableVariantHolderReaderImpl.java | 69 ++ .../arrow/variant/impl/VariantReaderImpl.java | 73 ++ .../arrow/variant/impl/VariantWriterImpl.java | 121 +++ .../org/apache/arrow/variant/TestVariant.java | 439 +++++++++ .../extension/TestVariantExtensionType.java | 249 ++++++ .../extension/TestVariantInListVector.java | 202 +++++ .../extension/TestVariantInMapVector.java | 125 +++ .../variant/extension/TestVariantType.java | 308 +++++++ .../variant/extension/TestVariantVector.java | 844 ++++++++++++++++++ bom/pom.xml | 5 + pom.xml | 2 + .../templates/AbstractFieldReader.java | 4 +- 19 files changed, 3288 insertions(+), 2 deletions(-) create mode 100644 arrow-variant/pom.xml create mode 100644 arrow-variant/src/main/java/module-info.java create mode 100644 arrow-variant/src/main/java/org/apache/arrow/variant/Variant.java create mode 100644 arrow-variant/src/main/java/org/apache/arrow/variant/extension/VariantType.java create mode 100644 arrow-variant/src/main/java/org/apache/arrow/variant/extension/VariantVector.java create mode 100644 arrow-variant/src/main/java/org/apache/arrow/variant/holders/NullableVariantHolder.java create mode 100644 arrow-variant/src/main/java/org/apache/arrow/variant/holders/VariantHolder.java create mode 100644 arrow-variant/src/main/java/org/apache/arrow/variant/impl/NullableVariantHolderReaderImpl.java create mode 100644 arrow-variant/src/main/java/org/apache/arrow/variant/impl/VariantReaderImpl.java create mode 100644 arrow-variant/src/main/java/org/apache/arrow/variant/impl/VariantWriterImpl.java create mode 100644 arrow-variant/src/test/java/org/apache/arrow/variant/TestVariant.java create mode 100644 arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantExtensionType.java create mode 100644 arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantInListVector.java create mode 100644 arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantInMapVector.java create mode 100644 arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantType.java create mode 100644 arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantVector.java diff --git a/arrow-variant/pom.xml b/arrow-variant/pom.xml new file mode 100644 index 0000000000..3a842178a4 --- /dev/null +++ b/arrow-variant/pom.xml @@ -0,0 +1,51 @@ + + + + 4.0.0 + + org.apache.arrow + arrow-java-root + 19.0.0-SNAPSHOT + + arrow-variant + Arrow Variant + Arrow Variant type support. + + + + org.apache.arrow + arrow-memory-core + + + org.apache.arrow + arrow-vector + + + org.apache.parquet + parquet-variant + ${dep.parquet.version} + + + org.apache.arrow + arrow-memory-unsafe + test + + + diff --git a/arrow-variant/src/main/java/module-info.java b/arrow-variant/src/main/java/module-info.java new file mode 100644 index 0000000000..da94173969 --- /dev/null +++ b/arrow-variant/src/main/java/module-info.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@SuppressWarnings("requires-automatic") +module org.apache.arrow.variant { + exports org.apache.arrow.variant; + exports org.apache.arrow.variant.extension; + exports org.apache.arrow.variant.impl; + exports org.apache.arrow.variant.holders; + + requires org.apache.arrow.memory.core; + requires org.apache.arrow.vector; + requires parquet.variant; +} diff --git a/arrow-variant/src/main/java/org/apache/arrow/variant/Variant.java b/arrow-variant/src/main/java/org/apache/arrow/variant/Variant.java new file mode 100644 index 0000000000..fa05cdd93f --- /dev/null +++ b/arrow-variant/src/main/java/org/apache/arrow/variant/Variant.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant; + +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.util.Objects; +import java.util.UUID; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.variant.holders.NullableVariantHolder; + +/** + * Wrapper around parquet-variant's Variant implementation. + * + *

This wrapper exists to isolate the parquet-variant dependency from Arrow's public API, + * allowing the vector module to expose variant functionality without requiring users to depend on + * parquet-variant directly. It also ensures that nested variant values (from arrays and objects) + * are consistently wrapped. + */ +public class Variant { + + private final org.apache.parquet.variant.Variant delegate; + + /** Creates a Variant from raw metadata and value byte arrays. */ + public Variant(byte[] metadata, byte[] value) { + this.delegate = new org.apache.parquet.variant.Variant(value, metadata); + } + + /** Creates a Variant by copying data from ArrowBuf instances. */ + public Variant( + ArrowBuf metadataBuffer, + int metadataStart, + int metadataEnd, + ArrowBuf valueBuffer, + int valueStart, + int valueEnd) { + byte[] metadata = new byte[metadataEnd - metadataStart]; + byte[] value = new byte[valueEnd - valueStart]; + metadataBuffer.getBytes(metadataStart, metadata); + valueBuffer.getBytes(valueStart, value); + this.delegate = new org.apache.parquet.variant.Variant(value, metadata); + } + + private Variant(org.apache.parquet.variant.Variant delegate) { + this.delegate = delegate; + } + + /** Constructs a Variant from a NullableVariantHolder. */ + public Variant(NullableVariantHolder holder) { + this( + holder.metadataBuffer, + holder.metadataStart, + holder.metadataEnd, + holder.valueBuffer, + holder.valueStart, + holder.valueEnd); + } + + public ByteBuffer getValueBuffer() { + return delegate.getValueBuffer(); + } + + public ByteBuffer getMetadataBuffer() { + return delegate.getMetadataBuffer(); + } + + public boolean getBoolean() { + return delegate.getBoolean(); + } + + public byte getByte() { + return delegate.getByte(); + } + + public short getShort() { + return delegate.getShort(); + } + + public int getInt() { + return delegate.getInt(); + } + + public long getLong() { + return delegate.getLong(); + } + + public double getDouble() { + return delegate.getDouble(); + } + + public BigDecimal getDecimal() { + return delegate.getDecimal(); + } + + public float getFloat() { + return delegate.getFloat(); + } + + public ByteBuffer getBinary() { + return delegate.getBinary(); + } + + public UUID getUUID() { + return delegate.getUUID(); + } + + public String getString() { + return delegate.getString(); + } + + public Type getType() { + return Type.fromParquet(delegate.getType()); + } + + public int numObjectElements() { + return delegate.numObjectElements(); + } + + public Variant getFieldByKey(String key) { + org.apache.parquet.variant.Variant result = delegate.getFieldByKey(key); + return result != null ? wrap(result) : null; + } + + public ObjectField getFieldAtIndex(int idx) { + org.apache.parquet.variant.Variant.ObjectField field = delegate.getFieldAtIndex(idx); + return new ObjectField(field.key, wrap(field.value)); + } + + public int numArrayElements() { + return delegate.numArrayElements(); + } + + public Variant getElementAtIndex(int index) { + org.apache.parquet.variant.Variant result = delegate.getElementAtIndex(index); + return result != null ? wrap(result) : null; + } + + private static Variant wrap(org.apache.parquet.variant.Variant parquetVariant) { + return new Variant(parquetVariant); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Variant variant = (Variant) o; + return delegate.getMetadataBuffer().equals(variant.delegate.getMetadataBuffer()) + && delegate.getValueBuffer().equals(variant.delegate.getValueBuffer()); + } + + @Override + public int hashCode() { + return Objects.hash(delegate.getMetadataBuffer(), delegate.getValueBuffer()); + } + + @Override + public String toString() { + return "Variant{type=" + getType() + '}'; + } + + public enum Type { + OBJECT, + ARRAY, + NULL, + BOOLEAN, + BYTE, + SHORT, + INT, + LONG, + STRING, + DOUBLE, + DECIMAL4, + DECIMAL8, + DECIMAL16, + DATE, + TIMESTAMP_TZ, + TIMESTAMP_NTZ, + FLOAT, + BINARY, + TIME, + TIMESTAMP_NANOS_TZ, + TIMESTAMP_NANOS_NTZ, + UUID; + + static Type fromParquet(org.apache.parquet.variant.Variant.Type parquetType) { + return Type.valueOf(parquetType.name()); + } + } + + public static final class ObjectField { + public final String key; + public final Variant value; + + public ObjectField(String key, Variant value) { + this.key = key; + this.value = value; + } + } +} diff --git a/arrow-variant/src/main/java/org/apache/arrow/variant/extension/VariantType.java b/arrow-variant/src/main/java/org/apache/arrow/variant/extension/VariantType.java new file mode 100644 index 0000000000..3deb70cdc0 --- /dev/null +++ b/arrow-variant/src/main/java/org/apache/arrow/variant/extension/VariantType.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant.extension; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.variant.impl.VariantWriterImpl; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.complex.writer.FieldWriter; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ArrowType.ExtensionType; +import org.apache.arrow.vector.types.pojo.ExtensionTypeRegistry; +import org.apache.arrow.vector.types.pojo.FieldType; + +/** + * Arrow extension type for Parquet + * Variant binary encoding. The type itself does not support shredded variant data. + */ +public final class VariantType extends ExtensionType { + + public static final VariantType INSTANCE = new VariantType(); + + public static final String EXTENSION_NAME = "parquet.variant"; + + static { + ExtensionTypeRegistry.register(INSTANCE); + } + + private VariantType() {} + + @Override + public ArrowType storageType() { + return ArrowType.Struct.INSTANCE; + } + + @Override + public String extensionName() { + return EXTENSION_NAME; + } + + @Override + public boolean extensionEquals(ExtensionType other) { + return other instanceof VariantType; + } + + @Override + public String serialize() { + return ""; + } + + @Override + public ArrowType deserialize(ArrowType storageType, String serializedData) { + if (!storageType.equals(this.storageType())) { + throw new UnsupportedOperationException( + "Cannot construct VariantType from underlying type " + storageType); + } + return INSTANCE; + } + + @Override + public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator) { + return new VariantVector(name, allocator); + } + + @Override + public boolean isComplex() { + // The type itself is not complex meaning we need separate functions to convert/extract + // different types. + // Meanwhile, the containing vector is complex in terms of containing multiple values (metadata + // and value) + return false; + } + + @Override + public FieldWriter getNewFieldWriter(ValueVector vector) { + return new VariantWriterImpl((VariantVector) vector); + } +} diff --git a/arrow-variant/src/main/java/org/apache/arrow/variant/extension/VariantVector.java b/arrow-variant/src/main/java/org/apache/arrow/variant/extension/VariantVector.java new file mode 100644 index 0000000000..1bbf1a6bdb --- /dev/null +++ b/arrow-variant/src/main/java/org/apache/arrow/variant/extension/VariantVector.java @@ -0,0 +1,348 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant.extension; + +import java.nio.ByteBuffer; +import java.util.List; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.util.hash.ArrowBufHasher; +import org.apache.arrow.variant.Variant; +import org.apache.arrow.variant.holders.NullableVariantHolder; +import org.apache.arrow.variant.holders.VariantHolder; +import org.apache.arrow.vector.BitVectorHelper; +import org.apache.arrow.vector.ExtensionTypeVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.complex.AbstractStructVector; +import org.apache.arrow.vector.complex.StructVector; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ArrowType.Binary; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.TransferPair; + +/** + * Arrow vector for storing {@link VariantType} values. + * + *

Stores semi-structured data (like JSON) as metadata + value binary pairs, allowing + * type-flexible columnar storage within Arrow's type system. + */ +public class VariantVector extends ExtensionTypeVector { + + public static final String METADATA_VECTOR_NAME = "metadata"; + public static final String VALUE_VECTOR_NAME = "value"; + + private final Field rootField; + + /** + * Constructs a new VariantVector with the given name and allocator. + * + * @param name the name of the vector + * @param allocator the buffer allocator for memory management + */ + public VariantVector(String name, BufferAllocator allocator) { + super( + name, + allocator, + new StructVector( + name, + allocator, + FieldType.nullable(ArrowType.Struct.INSTANCE), + null, + AbstractStructVector.ConflictPolicy.CONFLICT_ERROR, + false)); + rootField = createVariantField(name); + ((FieldVector) this.getUnderlyingVector()) + .initializeChildrenFromFields(rootField.getChildren()); + } + + /** + * Creates a new VariantVector with the given name. The Variant Field schema has to be the same + * everywhere, otherwise ArrowBuffer loading might fail during serialization/deserialization and + * schema mismatches can occur. This includes CompleteType's VARIANT and VARIANT_REQUIRED types. + */ + public static Field createVariantField(String name) { + return new Field( + name, new FieldType(true, VariantType.INSTANCE, null), createVariantChildFields()); + } + + /** + * Creates the child fields for the VariantVector. Metadata vector will be index 0 and value + * vector will be index 1. + */ + public static List createVariantChildFields() { + return List.of( + new Field(METADATA_VECTOR_NAME, new FieldType(false, Binary.INSTANCE, null), null), + new Field(VALUE_VECTOR_NAME, new FieldType(false, Binary.INSTANCE, null), null)); + } + + @Override + public void initializeChildrenFromFields(List children) { + // No-op, as children are initialized in the constructor + } + + @Override + public Field getField() { + return rootField; + } + + public VarBinaryVector getMetadataVector() { + return getUnderlyingVector().getChild(METADATA_VECTOR_NAME, VarBinaryVector.class); + } + + public VarBinaryVector getValueVector() { + return getUnderlyingVector().getChild(VALUE_VECTOR_NAME, VarBinaryVector.class); + } + + @Override + public TransferPair makeTransferPair(ValueVector target) { + return new VariantTransferPair(this, (VariantVector) target); + } + + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator) { + return new VariantTransferPair(this, new VariantVector(field.getName(), allocator)); + } + + @Override + public TransferPair getTransferPair(Field field, BufferAllocator allocator, CallBack callBack) { + return getTransferPair(field, allocator); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new VariantTransferPair(this, new VariantVector(ref, allocator)); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return getTransferPair(ref, allocator); + } + + @Override + public TransferPair getTransferPair(BufferAllocator allocator) { + return getTransferPair(this.getField().getName(), allocator); + } + + @Override + public void copyFrom(int fromIndex, int thisIndex, ValueVector from) { + getUnderlyingVector() + .copyFrom(fromIndex, thisIndex, ((VariantVector) from).getUnderlyingVector()); + } + + @Override + public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { + getUnderlyingVector() + .copyFromSafe(fromIndex, thisIndex, ((VariantVector) from).getUnderlyingVector()); + } + + @Override + public Object getObject(int index) { + if (isNull(index)) { + return null; + } + VarBinaryVector metadataVector = getMetadataVector(); + VarBinaryVector valueVector = getValueVector(); + + int metadataStart = metadataVector.getStartOffset(index); + int metadataEnd = metadataVector.getEndOffset(index); + int valueStart = valueVector.getStartOffset(index); + int valueEnd = valueVector.getEndOffset(index); + + return new Variant( + metadataVector.getDataBuffer(), + metadataStart, + metadataEnd, + valueVector.getDataBuffer(), + valueStart, + valueEnd); + } + + /** + * Retrieves the variant value at the specified index into the provided holder. + * + * @param index the index of the value to retrieve + * @param holder the holder to populate with the variant data + */ + public void get(int index, NullableVariantHolder holder) { + if (isNull(index)) { + holder.isSet = 0; + } else { + holder.isSet = 1; + VarBinaryVector metadataVector = getMetadataVector(); + VarBinaryVector valueVector = getValueVector(); + assert !metadataVector.isNull(index) && !valueVector.isNull(index); + + holder.metadataStart = metadataVector.getStartOffset(index); + holder.metadataEnd = metadataVector.getEndOffset(index); + holder.metadataBuffer = metadataVector.getDataBuffer(); + holder.valueStart = valueVector.getStartOffset(index); + holder.valueEnd = valueVector.getEndOffset(index); + holder.valueBuffer = valueVector.getDataBuffer(); + } + } + + /** + * Retrieves the variant value at the specified index into the provided non-nullable holder. + * + * @param index the index of the value to retrieve + * @param holder the holder to populate with the variant data + */ + public void get(int index, VariantHolder holder) { + VarBinaryVector metadataVector = getMetadataVector(); + VarBinaryVector valueVector = getValueVector(); + assert !metadataVector.isNull(index) && !valueVector.isNull(index); + + holder.metadataStart = metadataVector.getStartOffset(index); + holder.metadataEnd = metadataVector.getEndOffset(index); + holder.metadataBuffer = metadataVector.getDataBuffer(); + holder.valueStart = valueVector.getStartOffset(index); + holder.valueEnd = valueVector.getEndOffset(index); + holder.valueBuffer = valueVector.getDataBuffer(); + } + + /** + * Sets the variant value at the specified index from the provided holder. + * + * @param index the index at which to set the value + * @param holder the holder containing the variant data to set + */ + public void set(int index, VariantHolder holder) { + BitVectorHelper.setBit(getUnderlyingVector().getValidityBuffer(), index); + getMetadataVector() + .set(index, 1, holder.metadataStart, holder.metadataEnd, holder.metadataBuffer); + getValueVector().set(index, 1, holder.valueStart, holder.valueEnd, holder.valueBuffer); + } + + /** + * Sets the variant value at the specified index from the provided nullable holder. + * + * @param index the index at which to set the value + * @param holder the nullable holder containing the variant data to set + */ + public void set(int index, NullableVariantHolder holder) { + BitVectorHelper.setValidityBit(getUnderlyingVector().getValidityBuffer(), index, holder.isSet); + if (holder.isSet == 0) { + return; + } + getMetadataVector() + .set(index, 1, holder.metadataStart, holder.metadataEnd, holder.metadataBuffer); + getValueVector().set(index, 1, holder.valueStart, holder.valueEnd, holder.valueBuffer); + } + + /** + * Sets the variant value at the specified index from the provided holder, with bounds checking. + * + * @param index the index at which to set the value + * @param holder the holder containing the variant data to set + */ + public void setSafe(int index, VariantHolder holder) { + getUnderlyingVector().setIndexDefined(index); + getMetadataVector() + .setSafe(index, 1, holder.metadataStart, holder.metadataEnd, holder.metadataBuffer); + getValueVector().setSafe(index, 1, holder.valueStart, holder.valueEnd, holder.valueBuffer); + } + + /** + * Sets the variant value at the specified index from the provided nullable holder, with bounds + * checking. + * + * @param index the index at which to set the value + * @param holder the nullable holder containing the variant data to set + */ + public void setSafe(int index, NullableVariantHolder holder) { + if (holder.isSet == 0) { + getUnderlyingVector().setNull(index); + return; + } + getUnderlyingVector().setIndexDefined(index); + getMetadataVector() + .setSafe(index, 1, holder.metadataStart, holder.metadataEnd, holder.metadataBuffer); + getValueVector().setSafe(index, 1, holder.valueStart, holder.valueEnd, holder.valueBuffer); + } + + /** Sets the value at the given index from the provided Variant. */ + public void setSafe(int index, Variant variant) { + ByteBuffer metadataBuffer = variant.getMetadataBuffer(); + ByteBuffer valueBuffer = variant.getValueBuffer(); + int metadataLength = metadataBuffer.remaining(); + int valueLength = valueBuffer.remaining(); + try (ArrowBuf metaBuf = getAllocator().buffer(metadataLength); + ArrowBuf valBuf = getAllocator().buffer(valueLength)) { + metaBuf.setBytes(0, metadataBuffer.duplicate()); + valBuf.setBytes(0, valueBuffer.duplicate()); + getUnderlyingVector().setIndexDefined(index); + getMetadataVector().setSafe(index, 1, 0, metadataLength, metaBuf); + getValueVector().setSafe(index, 1, 0, valueLength, valBuf); + } + } + + @Override + protected FieldReader getReaderImpl() { + return new org.apache.arrow.variant.impl.VariantReaderImpl(this); + } + + @Override + public int hashCode(int index) { + return hashCode(index, null); + } + + @Override + public int hashCode(int index, ArrowBufHasher hasher) { + return getUnderlyingVector().hashCode(index, hasher); + } + + /** + * VariantTransferPair is a transfer pair for VariantVector. It transfers the metadata and value + * together using the underlyingVector's transfer pair. + */ + protected static class VariantTransferPair implements TransferPair { + private final TransferPair pair; + private final VariantVector from; + private final VariantVector to; + + public VariantTransferPair(VariantVector from, VariantVector to) { + this.from = from; + this.to = to; + this.pair = from.getUnderlyingVector().makeTransferPair((to).getUnderlyingVector()); + } + + @Override + public void transfer() { + pair.transfer(); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + pair.splitAndTransfer(startIndex, length); + } + + @Override + public ValueVector getTo() { + return to; + } + + @Override + public void copyValueSafe(int from, int to) { + pair.copyValueSafe(from, to); + } + } +} diff --git a/arrow-variant/src/main/java/org/apache/arrow/variant/holders/NullableVariantHolder.java b/arrow-variant/src/main/java/org/apache/arrow/variant/holders/NullableVariantHolder.java new file mode 100644 index 0000000000..b78d4a2013 --- /dev/null +++ b/arrow-variant/src/main/java/org/apache/arrow/variant/holders/NullableVariantHolder.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant.holders; + +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.variant.extension.VariantType; +import org.apache.arrow.vector.holders.ExtensionHolder; +import org.apache.arrow.vector.types.pojo.ArrowType; + +@SuppressWarnings("checkstyle:VisibilityModifier") +public final class NullableVariantHolder extends ExtensionHolder { + + public int isSet; + public int metadataStart; + public int metadataEnd; + public ArrowBuf metadataBuffer; + public int valueStart; + public int valueEnd; + public ArrowBuf valueBuffer; + + public NullableVariantHolder() {} + + @Override + public boolean equals(Object obj) { + throw new UnsupportedOperationException(); + } + + @Override + public int hashCode() { + throw new UnsupportedOperationException(); + } + + @Override + public String toString() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowType type() { + return VariantType.INSTANCE; + } +} diff --git a/arrow-variant/src/main/java/org/apache/arrow/variant/holders/VariantHolder.java b/arrow-variant/src/main/java/org/apache/arrow/variant/holders/VariantHolder.java new file mode 100644 index 0000000000..e3947ac439 --- /dev/null +++ b/arrow-variant/src/main/java/org/apache/arrow/variant/holders/VariantHolder.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant.holders; + +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.variant.extension.VariantType; +import org.apache.arrow.vector.holders.ExtensionHolder; +import org.apache.arrow.vector.types.pojo.ArrowType; + +@SuppressWarnings("checkstyle:VisibilityModifier") +public final class VariantHolder extends ExtensionHolder { + + public final int isSet = 1; + public int metadataStart; + public int metadataEnd; + public ArrowBuf metadataBuffer; + public int valueStart; + public int valueEnd; + public ArrowBuf valueBuffer; + + public VariantHolder() {} + + @Override + public boolean equals(Object obj) { + throw new UnsupportedOperationException(); + } + + @Override + public int hashCode() { + throw new UnsupportedOperationException(); + } + + @Override + public String toString() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowType type() { + return VariantType.INSTANCE; + } +} diff --git a/arrow-variant/src/main/java/org/apache/arrow/variant/impl/NullableVariantHolderReaderImpl.java b/arrow-variant/src/main/java/org/apache/arrow/variant/impl/NullableVariantHolderReaderImpl.java new file mode 100644 index 0000000000..1645529c0c --- /dev/null +++ b/arrow-variant/src/main/java/org/apache/arrow/variant/impl/NullableVariantHolderReaderImpl.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant.impl; + +import org.apache.arrow.variant.holders.NullableVariantHolder; +import org.apache.arrow.vector.complex.impl.AbstractFieldReader; +import org.apache.arrow.vector.types.Types; + +public class NullableVariantHolderReaderImpl extends AbstractFieldReader { + private final NullableVariantHolder holder; + + public NullableVariantHolderReaderImpl(NullableVariantHolder holder) { + this.holder = holder; + } + + @Override + public int size() { + throw new UnsupportedOperationException("You can't call size on a Holder value reader."); + } + + @Override + public boolean next() { + throw new UnsupportedOperationException("You can't call next on a single value reader."); + } + + @Override + public void setPosition(int index) { + throw new UnsupportedOperationException("You can't call setPosition on a single value reader."); + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.EXTENSIONTYPE; + } + + @Override + public boolean isSet() { + return holder.isSet == 1; + } + + /** + * Reads the variant holder data into the provided holder. + * + * @param h the holder to read into + */ + public void read(NullableVariantHolder h) { + h.metadataStart = this.holder.metadataStart; + h.metadataEnd = this.holder.metadataEnd; + h.metadataBuffer = this.holder.metadataBuffer; + h.valueStart = this.holder.valueStart; + h.valueEnd = this.holder.valueEnd; + h.valueBuffer = this.holder.valueBuffer; + h.isSet = this.isSet() ? 1 : 0; + } +} diff --git a/arrow-variant/src/main/java/org/apache/arrow/variant/impl/VariantReaderImpl.java b/arrow-variant/src/main/java/org/apache/arrow/variant/impl/VariantReaderImpl.java new file mode 100644 index 0000000000..670104b7d1 --- /dev/null +++ b/arrow-variant/src/main/java/org/apache/arrow/variant/impl/VariantReaderImpl.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant.impl; + +import org.apache.arrow.variant.extension.VariantVector; +import org.apache.arrow.variant.holders.NullableVariantHolder; +import org.apache.arrow.variant.holders.VariantHolder; +import org.apache.arrow.vector.complex.impl.AbstractFieldReader; +import org.apache.arrow.vector.holders.ExtensionHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.Field; + +public class VariantReaderImpl extends AbstractFieldReader { + private final VariantVector vector; + + public VariantReaderImpl(VariantVector vector) { + this.vector = vector; + } + + @Override + public Types.MinorType getMinorType() { + return this.vector.getMinorType(); + } + + @Override + public Field getField() { + return this.vector.getField(); + } + + @Override + public boolean isSet() { + return !this.vector.isNull(this.idx()); + } + + @Override + public void read(ExtensionHolder holder) { + if (holder instanceof VariantHolder) { + vector.get(idx(), (VariantHolder) holder); + } else if (holder instanceof NullableVariantHolder) { + vector.get(idx(), (NullableVariantHolder) holder); + } else { + throw new IllegalArgumentException( + "Unsupported holder type for VariantReader: " + holder.getClass()); + } + } + + public void read(VariantHolder h) { + this.vector.get(this.idx(), h); + } + + public void read(NullableVariantHolder h) { + this.vector.get(this.idx(), h); + } + + @Override + public Object readObject() { + return this.vector.getObject(this.idx()); + } +} diff --git a/arrow-variant/src/main/java/org/apache/arrow/variant/impl/VariantWriterImpl.java b/arrow-variant/src/main/java/org/apache/arrow/variant/impl/VariantWriterImpl.java new file mode 100644 index 0000000000..266ddb75d2 --- /dev/null +++ b/arrow-variant/src/main/java/org/apache/arrow/variant/impl/VariantWriterImpl.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant.impl; + +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.variant.Variant; +import org.apache.arrow.variant.extension.VariantVector; +import org.apache.arrow.variant.holders.NullableVariantHolder; +import org.apache.arrow.variant.holders.VariantHolder; +import org.apache.arrow.vector.complex.impl.AbstractExtensionTypeWriter; +import org.apache.arrow.vector.holders.ExtensionHolder; +import org.apache.arrow.vector.types.pojo.ArrowType; + +/** + * Writer implementation for VARIANT extension type vectors. + * + *

This writer handles writing variant data to a {@link VariantVector}. It accepts both {@link + * VariantHolder} and {@link NullableVariantHolder} objects containing metadata and value buffers + * and writes them to the appropriate position in the vector. + */ +public class VariantWriterImpl extends AbstractExtensionTypeWriter { + + private static final String UNSUPPORTED_TYPE_TEMPLATE = "Unsupported type for Variant: %s"; + + /** + * Constructs a new VariantWriterImpl for the given vector. + * + * @param vector the variant vector to write to + */ + public VariantWriterImpl(VariantVector vector) { + super(vector); + } + + /** + * Writes an extension type or variant value to the vector. + * + *

This method handles {@link ExtensionHolder} by delegating to {@link #write(ExtensionHolder)} + * and {@link Variant} by delegating to {@link #writeVariant(Variant)}. + * + * @param object the object to write, must be an {@link ExtensionHolder} or {@link Variant} + * @throws IllegalArgumentException if the object is not an {@link ExtensionHolder} or {@link + * Variant} + */ + @Override + public void writeExtension(Object object) { + if (object instanceof ExtensionHolder) { + write((ExtensionHolder) object); + } else if (object instanceof Variant) { + writeVariant((Variant) object); + } else { + throw new IllegalArgumentException( + String.format(UNSUPPORTED_TYPE_TEMPLATE, object.getClass().getName())); + } + } + + private void writeVariant(Variant variant) { + java.nio.ByteBuffer metadataBuffer = variant.getMetadataBuffer(); + java.nio.ByteBuffer valueBuffer = variant.getValueBuffer(); + int metadataLength = metadataBuffer.remaining(); + int valueLength = valueBuffer.remaining(); + try (ArrowBuf metadataBuf = vector.getAllocator().buffer(metadataLength); + ArrowBuf valueBuf = vector.getAllocator().buffer(valueLength)) { + metadataBuf.setBytes(0, metadataBuffer.duplicate()); + valueBuf.setBytes(0, valueBuffer.duplicate()); + NullableVariantHolder holder = new NullableVariantHolder(); + holder.isSet = 1; + holder.metadataBuffer = metadataBuf; + holder.metadataStart = 0; + holder.metadataEnd = metadataLength; + holder.valueBuffer = valueBuf; + holder.valueStart = 0; + holder.valueEnd = valueLength; + vector.setSafe(getPosition(), holder); + vector.setValueCount(getPosition() + 1); + } + } + + @Override + public void writeExtension(Object value, ArrowType type) { + writeExtension(value); + } + + /** + * Writes a variant holder to the vector at the current position. + * + *

The holder can be either a {@link VariantHolder} (non-nullable, always set) or a {@link + * NullableVariantHolder} (nullable, may be null). The data is written using {@link + * VariantVector#setSafe(int, NullableVariantHolder)} which handles buffer allocation and copying. + * + * @param extensionHolder the variant holder to write, must be a {@link VariantHolder} or {@link + * NullableVariantHolder} + * @throws IllegalArgumentException if the holder is neither a {@link VariantHolder} nor a {@link + * NullableVariantHolder} + */ + @Override + public void write(ExtensionHolder extensionHolder) { + if (extensionHolder instanceof VariantHolder) { + vector.setSafe(getPosition(), (VariantHolder) extensionHolder); + } else if (extensionHolder instanceof NullableVariantHolder) { + vector.setSafe(getPosition(), (NullableVariantHolder) extensionHolder); + } else { + throw new IllegalArgumentException( + String.format(UNSUPPORTED_TYPE_TEMPLATE, extensionHolder.getClass().getName())); + } + vector.setValueCount(getPosition() + 1); + } +} diff --git a/arrow-variant/src/test/java/org/apache/arrow/variant/TestVariant.java b/arrow-variant/src/test/java/org/apache/arrow/variant/TestVariant.java new file mode 100644 index 0000000000..bc46a68616 --- /dev/null +++ b/arrow-variant/src/test/java/org/apache/arrow/variant/TestVariant.java @@ -0,0 +1,439 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.util.UUID; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.parquet.variant.VariantBuilder; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class TestVariant { + + private BufferAllocator allocator; + + @BeforeEach + void beforeEach() { + allocator = new RootAllocator(); + } + + @AfterEach + void afterEach() { + allocator.close(); + } + + static Variant buildVariant(VariantBuilder builder) { + org.apache.parquet.variant.Variant parquetVariant = builder.build(); + ByteBuffer valueBuf = parquetVariant.getValueBuffer(); + ByteBuffer metaBuf = parquetVariant.getMetadataBuffer(); + byte[] valueBytes = new byte[valueBuf.remaining()]; + byte[] metaBytes = new byte[metaBuf.remaining()]; + valueBuf.get(valueBytes); + metaBuf.get(metaBytes); + return new Variant(metaBytes, valueBytes); + } + + public static Variant variantString(String value) { + VariantBuilder builder = new VariantBuilder(); + builder.appendString(value); + return buildVariant(builder); + } + + @Test + void testConstructionWithArrowBuf() { + VariantBuilder builder = new VariantBuilder(); + builder.appendInt(42); + Variant source = buildVariant(builder); + int metaLen = source.getMetadataBuffer().remaining(); + int valueLen = source.getValueBuffer().remaining(); + + try (ArrowBuf metadataArrowBuf = allocator.buffer(metaLen + 2); + ArrowBuf valueArrowBuf = allocator.buffer(valueLen + 3)) { + metadataArrowBuf.setBytes(2, source.getMetadataBuffer()); + valueArrowBuf.setBytes(3, source.getValueBuffer()); + + Variant variant = + new Variant(metadataArrowBuf, 2, 2 + metaLen, valueArrowBuf, 3, 3 + valueLen); + + assertEquals(Variant.Type.INT, variant.getType()); + assertEquals(42, variant.getInt()); + } + } + + @Test + void testNullType() { + VariantBuilder builder = new VariantBuilder(); + builder.appendNull(); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.NULL, variant.getType()); + } + + @Test + void testBooleanType() { + VariantBuilder builder = new VariantBuilder(); + builder.appendBoolean(true); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.BOOLEAN, variant.getType()); + assertTrue(variant.getBoolean()); + + builder = new VariantBuilder(); + builder.appendBoolean(false); + variant = buildVariant(builder); + + assertEquals(Variant.Type.BOOLEAN, variant.getType()); + assertFalse(variant.getBoolean()); + } + + @Test + void testByteType() { + VariantBuilder builder = new VariantBuilder(); + builder.appendByte((byte) 42); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.BYTE, variant.getType()); + assertEquals((byte) 42, variant.getByte()); + } + + @Test + void testShortType() { + VariantBuilder builder = new VariantBuilder(); + builder.appendShort((short) 1234); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.SHORT, variant.getType()); + assertEquals((short) 1234, variant.getShort()); + } + + @Test + void testIntType() { + VariantBuilder builder = new VariantBuilder(); + builder.appendInt(123456); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.INT, variant.getType()); + assertEquals(123456, variant.getInt()); + } + + @Test + void testLongType() { + VariantBuilder builder = new VariantBuilder(); + builder.appendLong(9876543210L); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.LONG, variant.getType()); + assertEquals(9876543210L, variant.getLong()); + } + + @Test + void testFloatType() { + VariantBuilder builder = new VariantBuilder(); + builder.appendFloat(3.14f); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.FLOAT, variant.getType()); + assertEquals(3.14f, variant.getFloat(), 0.001f); + } + + @Test + void testDoubleType() { + VariantBuilder builder = new VariantBuilder(); + builder.appendDouble(3.14159265359); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.DOUBLE, variant.getType()); + assertEquals(3.14159265359, variant.getDouble(), 0.0000001); + } + + @Test + void testStringType() { + VariantBuilder builder = new VariantBuilder(); + builder.appendString("hello world"); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.STRING, variant.getType()); + assertEquals("hello world", variant.getString()); + } + + @Test + void testDecimalType() { + VariantBuilder builder = new VariantBuilder(); + builder.appendDecimal(new BigDecimal("123.456")); + Variant variant = buildVariant(builder); + + assertTrue( + variant.getType() == Variant.Type.DECIMAL4 + || variant.getType() == Variant.Type.DECIMAL8 + || variant.getType() == Variant.Type.DECIMAL16); + assertEquals(new BigDecimal("123.456"), variant.getDecimal()); + } + + @Test + void testBinaryType() { + VariantBuilder builder = new VariantBuilder(); + byte[] data = new byte[] {1, 2, 3, 4, 5}; + builder.appendBinary(ByteBuffer.wrap(data)); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.BINARY, variant.getType()); + ByteBuffer result = variant.getBinary(); + byte[] resultBytes = new byte[result.remaining()]; + result.get(resultBytes); + assertArrayEquals(data, resultBytes); + } + + @Test + void testUuidType() { + VariantBuilder builder = new VariantBuilder(); + UUID uuid = UUID.randomUUID(); + builder.appendUUID(uuid); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.UUID, variant.getType()); + assertEquals(uuid, variant.getUUID()); + } + + @Test + void testDateType() { + VariantBuilder builder = new VariantBuilder(); + int daysSinceEpoch = 19000; + builder.appendDate(daysSinceEpoch); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.DATE, variant.getType()); + } + + @Test + void testTimestampTzType() { + VariantBuilder builder = new VariantBuilder(); + long micros = System.currentTimeMillis() * 1000; + builder.appendTimestampTz(micros); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.TIMESTAMP_TZ, variant.getType()); + } + + @Test + void testTimestampNtzType() { + VariantBuilder builder = new VariantBuilder(); + long micros = System.currentTimeMillis() * 1000; + builder.appendTimestampNtz(micros); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.TIMESTAMP_NTZ, variant.getType()); + } + + @Test + void testTimeType() { + VariantBuilder builder = new VariantBuilder(); + long micros = 12345678L; + builder.appendTime(micros); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.TIME, variant.getType()); + } + + @Test + void testObjectType() { + VariantBuilder builder = new VariantBuilder(); + var objBuilder = builder.startObject(); + objBuilder.appendKey("name"); + objBuilder.appendString("test"); + objBuilder.appendKey("value"); + objBuilder.appendInt(42); + builder.endObject(); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.OBJECT, variant.getType()); + assertEquals(2, variant.numObjectElements()); + + Variant nameField = variant.getFieldByKey("name"); + assertNotNull(nameField); + assertEquals(Variant.Type.STRING, nameField.getType()); + assertEquals("test", nameField.getString()); + + Variant valueField = variant.getFieldByKey("value"); + assertNotNull(valueField); + assertEquals(Variant.Type.INT, valueField.getType()); + assertEquals(42, valueField.getInt()); + + assertNull(variant.getFieldByKey("nonexistent")); + + // Empty object + builder = new VariantBuilder(); + builder.startObject(); + builder.endObject(); + Variant emptyObj = buildVariant(builder); + assertEquals(Variant.Type.OBJECT, emptyObj.getType()); + assertEquals(0, emptyObj.numObjectElements()); + } + + @Test + void testObjectFieldAtIndex() { + VariantBuilder builder = new VariantBuilder(); + var objBuilder = builder.startObject(); + objBuilder.appendKey("alpha"); + objBuilder.appendInt(1); + objBuilder.appendKey("beta"); + objBuilder.appendInt(2); + builder.endObject(); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.OBJECT, variant.getType()); + assertEquals(2, variant.numObjectElements()); + + Variant.ObjectField field0 = variant.getFieldAtIndex(0); + assertNotNull(field0); + assertNotNull(field0.key); + assertNotNull(field0.value); + + Variant.ObjectField field1 = variant.getFieldAtIndex(1); + assertNotNull(field1); + assertNotNull(field1.key); + assertNotNull(field1.value); + } + + @Test + void testArrayType() { + VariantBuilder builder = new VariantBuilder(); + var arrayBuilder = builder.startArray(); + arrayBuilder.appendInt(1); + arrayBuilder.appendInt(2); + arrayBuilder.appendInt(3); + builder.endArray(); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.ARRAY, variant.getType()); + assertEquals(3, variant.numArrayElements()); + + Variant elem0 = variant.getElementAtIndex(0); + assertNotNull(elem0); + assertEquals(Variant.Type.INT, elem0.getType()); + assertEquals(1, elem0.getInt()); + + Variant elem1 = variant.getElementAtIndex(1); + assertEquals(2, elem1.getInt()); + + Variant elem2 = variant.getElementAtIndex(2); + assertEquals(3, elem2.getInt()); + + assertNull(variant.getElementAtIndex(-1)); + assertNull(variant.getElementAtIndex(3)); + + // Empty array + builder = new VariantBuilder(); + builder.startArray(); + builder.endArray(); + Variant emptyArr = buildVariant(builder); + assertEquals(Variant.Type.ARRAY, emptyArr.getType()); + assertEquals(0, emptyArr.numArrayElements()); + } + + @Test + void testNestedStructure() { + VariantBuilder builder = new VariantBuilder(); + var objBuilder = builder.startObject(); + objBuilder.appendKey("items"); + var arrayBuilder = objBuilder.startArray(); + arrayBuilder.appendString("a"); + arrayBuilder.appendString("b"); + objBuilder.endArray(); + builder.endObject(); + Variant variant = buildVariant(builder); + + assertEquals(Variant.Type.OBJECT, variant.getType()); + Variant items = variant.getFieldByKey("items"); + assertNotNull(items); + assertEquals(Variant.Type.ARRAY, items.getType()); + assertEquals(2, items.numArrayElements()); + assertEquals("a", items.getElementAtIndex(0).getString()); + assertEquals("b", items.getElementAtIndex(1).getString()); + } + + @Test + void testEquals() { + VariantBuilder builder1 = new VariantBuilder(); + builder1.appendString("test"); + Variant variant1 = buildVariant(builder1); + + VariantBuilder builder2 = new VariantBuilder(); + builder2.appendString("test"); + Variant variant2 = buildVariant(builder2); + + VariantBuilder builder3 = new VariantBuilder(); + builder3.appendString("different"); + Variant variant3 = buildVariant(builder3); + + assertEquals(variant1, variant1); + assertEquals(variant1, variant2); + assertNotEquals(variant1, variant3); + assertNotEquals(variant1, null); + assertNotEquals(variant1, "not a variant"); + } + + @Test + void testHashCode() { + VariantBuilder builder1 = new VariantBuilder(); + builder1.appendInt(42); + Variant variant1 = buildVariant(builder1); + + VariantBuilder builder2 = new VariantBuilder(); + builder2.appendInt(42); + Variant variant2 = buildVariant(builder2); + + assertEquals(variant1.hashCode(), variant2.hashCode()); + } + + @Test + void testToString() { + VariantBuilder builder = new VariantBuilder(); + builder.appendString("test"); + Variant variant = buildVariant(builder); + + String str = variant.toString(); + assertNotNull(str); + assertTrue(str.contains("type=")); + } + + @Test + void testTypeEnumsMatch() { + for (Variant.Type arrowType : Variant.Type.values()) { + org.apache.parquet.variant.Variant.Type parquetType = + org.apache.parquet.variant.Variant.Type.valueOf(arrowType.name()); + assertEquals(arrowType, Variant.Type.fromParquet(parquetType)); + } + for (org.apache.parquet.variant.Variant.Type parquetType : + org.apache.parquet.variant.Variant.Type.values()) { + Variant.Type arrowType = Variant.Type.valueOf(parquetType.name()); + assertEquals(parquetType.name(), arrowType.name()); + } + } +} diff --git a/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantExtensionType.java b/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantExtensionType.java new file mode 100644 index 0000000000..f3213d523a --- /dev/null +++ b/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantExtensionType.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant.extension; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.channels.SeekableByteChannel; +import java.nio.channels.WritableByteChannel; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.Collections; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.variant.TestVariant; +import org.apache.arrow.variant.Variant; +import org.apache.arrow.vector.ExtensionTypeVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.compare.Range; +import org.apache.arrow.vector.compare.RangeEqualsVisitor; +import org.apache.arrow.vector.complex.StructVector; +import org.apache.arrow.vector.complex.writer.BaseWriter; +import org.apache.arrow.vector.ipc.ArrowFileReader; +import org.apache.arrow.vector.ipc.ArrowFileWriter; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ArrowType.ExtensionType; +import org.apache.arrow.vector.types.pojo.ExtensionTypeRegistry; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.arrow.vector.util.VectorBatchAppender; +import org.apache.arrow.vector.validate.ValidateVectorVisitor; +import org.junit.jupiter.api.Test; + +public class TestVariantExtensionType { + + private static void ensureRegistered(ArrowType.ExtensionType type) { + if (ExtensionTypeRegistry.lookup(type.extensionName()) == null) { + ExtensionTypeRegistry.register(type); + } + } + + @Test + public void roundtripVariant() throws IOException { + ensureRegistered(VariantType.INSTANCE); + final Schema schema = + new Schema(Collections.singletonList(Field.nullable("a", VariantType.INSTANCE))); + try (final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); + final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { + VariantVector vector = (VariantVector) root.getVector("a"); + vector.allocateNew(); + + vector.setSafe(0, TestVariant.variantString("hello")); + vector.setSafe(1, TestVariant.variantString("world")); + vector.setValueCount(2); + root.setRowCount(2); + + final File file = File.createTempFile("varianttest", ".arrow"); + try (final WritableByteChannel channel = + FileChannel.open(Paths.get(file.getAbsolutePath()), StandardOpenOption.WRITE); + final ArrowFileWriter writer = new ArrowFileWriter(root, null, channel)) { + writer.start(); + writer.writeBatch(); + writer.end(); + } + + try (final SeekableByteChannel channel = + Files.newByteChannel(Paths.get(file.getAbsolutePath())); + final ArrowFileReader reader = new ArrowFileReader(channel, allocator)) { + reader.loadNextBatch(); + final VectorSchemaRoot readerRoot = reader.getVectorSchemaRoot(); + assertEquals(root.getSchema(), readerRoot.getSchema()); + + final Field field = readerRoot.getSchema().getFields().get(0); + final VariantType expectedType = VariantType.INSTANCE; + assertEquals( + field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_NAME), + expectedType.extensionName()); + assertEquals( + field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_METADATA), + expectedType.serialize()); + + final ExtensionTypeVector deserialized = + (ExtensionTypeVector) readerRoot.getFieldVectors().get(0); + assertEquals(vector.getValueCount(), deserialized.getValueCount()); + for (int i = 0; i < vector.getValueCount(); i++) { + assertEquals(vector.isNull(i), deserialized.isNull(i)); + if (!vector.isNull(i)) { + assertEquals(vector.getObject(i), deserialized.getObject(i)); + } + } + } + } + } + + @Test + public void readVariantAsUnderlyingType() throws IOException { + ensureRegistered(VariantType.INSTANCE); + final Schema schema = + new Schema(Collections.singletonList(VariantVector.createVariantField("a"))); + try (final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); + final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { + VariantVector vector = (VariantVector) root.getVector("a"); + vector.allocateNew(); + + vector.setSafe(0, TestVariant.variantString("hello")); + vector.setValueCount(1); + root.setRowCount(1); + + final File file = File.createTempFile("varianttest", ".arrow"); + try (final WritableByteChannel channel = + FileChannel.open(Paths.get(file.getAbsolutePath()), StandardOpenOption.WRITE); + final ArrowFileWriter writer = new ArrowFileWriter(root, null, channel)) { + writer.start(); + writer.writeBatch(); + writer.end(); + } + + ExtensionTypeRegistry.unregister(VariantType.INSTANCE); + + try (final SeekableByteChannel channel = + Files.newByteChannel(Paths.get(file.getAbsolutePath())); + final ArrowFileReader reader = new ArrowFileReader(channel, allocator)) { + reader.loadNextBatch(); + VectorSchemaRoot readRoot = reader.getVectorSchemaRoot(); + + // Verify schema properties + assertEquals(1, readRoot.getSchema().getFields().size()); + assertEquals("a", readRoot.getSchema().getFields().get(0).getName()); + assertTrue(readRoot.getSchema().getFields().get(0).getType() instanceof ArrowType.Struct); + + // Verify extension metadata is preserved + final Field field = readRoot.getSchema().getFields().get(0); + assertEquals( + VariantType.EXTENSION_NAME, + field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_NAME)); + assertEquals("", field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_METADATA)); + + // Verify vector type and row count + assertEquals(1, readRoot.getRowCount()); + FieldVector readVector = readRoot.getVector("a"); + assertEquals(StructVector.class, readVector.getClass()); + + // Verify value count matches + StructVector structVector = (StructVector) readVector; + assertEquals(vector.getValueCount(), structVector.getValueCount()); + + // Verify the underlying data can be accessed from child vectors + VarBinaryVector metadataVector = + structVector.getChild(VariantVector.METADATA_VECTOR_NAME, VarBinaryVector.class); + VarBinaryVector valueVector = + structVector.getChild(VariantVector.VALUE_VECTOR_NAME, VarBinaryVector.class); + assertNotNull(metadataVector); + assertNotNull(valueVector); + assertEquals(1, metadataVector.getValueCount()); + assertEquals(1, valueVector.getValueCount()); + } + } + } + + @Test + public void testVariantVectorCompare() { + VariantType variantType = VariantType.INSTANCE; + ExtensionTypeRegistry.register(variantType); + Variant hello = TestVariant.variantString("hello"); + Variant world = TestVariant.variantString("world"); + try (final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); + VariantVector a1 = + (VariantVector) + variantType.getNewVector("a", FieldType.nullable(variantType), allocator); + VariantVector a2 = + (VariantVector) + variantType.getNewVector("a", FieldType.nullable(variantType), allocator); + VariantVector bb = + (VariantVector) + variantType.getNewVector("a", FieldType.nullable(variantType), allocator)) { + + ValidateVectorVisitor validateVisitor = new ValidateVectorVisitor(); + validateVisitor.visit(a1, null); + + a1.allocateNew(); + a2.allocateNew(); + bb.allocateNew(); + + a1.setSafe(0, hello); + a1.setSafe(1, world); + a1.setValueCount(2); + + a2.setSafe(0, hello); + a2.setSafe(1, world); + a2.setValueCount(2); + + bb.setSafe(0, world); + bb.setSafe(1, hello); + bb.setValueCount(2); + + Range range = new Range(0, 0, a1.getValueCount()); + RangeEqualsVisitor visitor = new RangeEqualsVisitor(a1, a2); + assertTrue(visitor.rangeEquals(range)); + + visitor = new RangeEqualsVisitor(a1, bb); + assertFalse(visitor.rangeEquals(range)); + + VectorBatchAppender.batchAppend(a1, a2, bb); + assertEquals(6, a1.getValueCount()); + validateVisitor.visit(a1, null); + } + } + + @Test + public void testVariantCopyAsValueThrowsException() { + ensureRegistered(VariantType.INSTANCE); + try (BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); + VariantVector vector = new VariantVector("variant", allocator)) { + vector.allocateNew(); + vector.setSafe(0, TestVariant.variantString("hello")); + vector.setValueCount(1); + + var reader = vector.getReader(); + reader.setPosition(0); + + assertThrows( + IllegalArgumentException.class, () -> reader.copyAsValue((BaseWriter.StructWriter) null)); + } + } +} diff --git a/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantInListVector.java b/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantInListVector.java new file mode 100644 index 0000000000..8b6000bc46 --- /dev/null +++ b/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantInListVector.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant.extension; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.variant.TestVariant; +import org.apache.arrow.variant.Variant; +import org.apache.arrow.variant.holders.NullableVariantHolder; +import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.impl.UnionListReader; +import org.apache.arrow.vector.complex.impl.UnionListWriter; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.complex.writer.BaseWriter.ExtensionWriter; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class TestVariantInListVector { + + private BufferAllocator allocator; + + @BeforeEach + public void init() { + allocator = new RootAllocator(Long.MAX_VALUE); + } + + @AfterEach + public void terminate() throws Exception { + allocator.close(); + } + + @Test + public void testListVectorWithVariantExtensionType() { + final FieldType type = FieldType.nullable(VariantType.INSTANCE); + try (ListVector inVector = new ListVector("input", allocator, type, null)) { + Variant variant1 = TestVariant.variantString("hello"); + Variant variant2 = TestVariant.variantString("bye"); + + UnionListWriter writer = inVector.getWriter(); + writer.allocate(); + + writer.setPosition(0); + writer.startList(); + ExtensionWriter extensionWriter = writer.extension(VariantType.INSTANCE); + extensionWriter.writeExtension(variant1); + extensionWriter.writeExtension(variant2); + writer.endList(); + inVector.setValueCount(1); + + ArrayList resultSet = (ArrayList) inVector.getObject(0); + assertEquals(2, resultSet.size()); + assertEquals(variant1, resultSet.get(0)); + assertEquals(variant2, resultSet.get(1)); + } + } + + @Test + public void testListVectorReaderForVariantExtensionType() { + try (ListVector inVector = ListVector.empty("input", allocator)) { + Variant variant1 = TestVariant.variantString("hello"); + Variant variant2 = TestVariant.variantString("bye"); + + UnionListWriter writer = inVector.getWriter(); + writer.allocate(); + + writer.setPosition(0); + writer.startList(); + ExtensionWriter extensionWriter = writer.extension(VariantType.INSTANCE); + extensionWriter.writeExtension(variant1); + writer.endList(); + + writer.setPosition(1); + writer.startList(); + extensionWriter.writeExtension(variant2); + extensionWriter.writeExtension(variant2); + writer.endList(); + + inVector.setValueCount(2); + + UnionListReader reader = inVector.getReader(); + reader.setPosition(0); + assertTrue(reader.next()); + FieldReader variantReader = reader.reader(); + NullableVariantHolder resultHolder = new NullableVariantHolder(); + variantReader.read(resultHolder); + assertEquals(variant1, new Variant(resultHolder)); + + reader.setPosition(1); + assertTrue(reader.next()); + variantReader = reader.reader(); + variantReader.read(resultHolder); + assertEquals(variant2, new Variant(resultHolder)); + + assertTrue(reader.next()); + variantReader = reader.reader(); + variantReader.read(resultHolder); + assertEquals(variant2, new Variant(resultHolder)); + } + } + + @Test + public void testCopyFromForVariantExtensionType() { + try (ListVector inVector = ListVector.empty("input", allocator); + ListVector outVector = ListVector.empty("output", allocator)) { + Variant variant1 = TestVariant.variantString("hello"); + Variant variant2 = TestVariant.variantString("bye"); + + UnionListWriter writer = inVector.getWriter(); + writer.allocate(); + + writer.setPosition(0); + writer.startList(); + ExtensionWriter extensionWriter = writer.extension(VariantType.INSTANCE); + extensionWriter.writeExtension(variant1); + writer.endList(); + + writer.setPosition(1); + writer.startList(); + extensionWriter.writeExtension(variant2); + extensionWriter.writeExtension(variant2); + writer.endList(); + + inVector.setValueCount(2); + + outVector.allocateNew(); + outVector.copyFrom(0, 0, inVector); + outVector.copyFrom(1, 1, inVector); + outVector.setValueCount(2); + + ArrayList resultSet0 = (ArrayList) outVector.getObject(0); + assertEquals(1, resultSet0.size()); + assertEquals(variant1, resultSet0.get(0)); + + ArrayList resultSet1 = (ArrayList) outVector.getObject(1); + assertEquals(2, resultSet1.size()); + assertEquals(variant2, resultSet1.get(0)); + assertEquals(variant2, resultSet1.get(1)); + } + } + + @Test + public void testCopyValueSafeForVariantExtensionType() { + try (ListVector inVector = ListVector.empty("input", allocator)) { + Variant variant1 = TestVariant.variantString("hello"); + Variant variant2 = TestVariant.variantString("bye"); + + UnionListWriter writer = inVector.getWriter(); + writer.allocate(); + + writer.setPosition(0); + writer.startList(); + ExtensionWriter extensionWriter = writer.extension(VariantType.INSTANCE); + extensionWriter.writeExtension(variant1); + writer.endList(); + + writer.setPosition(1); + writer.startList(); + extensionWriter.writeExtension(variant2); + extensionWriter.writeExtension(variant2); + writer.endList(); + + inVector.setValueCount(2); + + try (ListVector outVector = (ListVector) inVector.getTransferPair(allocator).getTo()) { + TransferPair tp = inVector.makeTransferPair(outVector); + tp.copyValueSafe(0, 0); + tp.copyValueSafe(1, 1); + outVector.setValueCount(2); + + ArrayList resultSet0 = (ArrayList) outVector.getObject(0); + assertEquals(1, resultSet0.size()); + assertEquals(variant1, resultSet0.get(0)); + + ArrayList resultSet1 = (ArrayList) outVector.getObject(1); + assertEquals(2, resultSet1.size()); + assertEquals(variant2, resultSet1.get(0)); + assertEquals(variant2, resultSet1.get(1)); + } + } + } +} diff --git a/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantInMapVector.java b/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantInMapVector.java new file mode 100644 index 0000000000..dd925810de --- /dev/null +++ b/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantInMapVector.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant.extension; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.variant.TestVariant; +import org.apache.arrow.variant.Variant; +import org.apache.arrow.variant.holders.NullableVariantHolder; +import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.impl.UnionMapReader; +import org.apache.arrow.vector.complex.impl.UnionMapWriter; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class TestVariantInMapVector { + + private BufferAllocator allocator; + + @BeforeEach + public void init() { + allocator = new RootAllocator(Long.MAX_VALUE); + } + + @AfterEach + public void terminate() { + allocator.close(); + } + + @Test + public void testMapVectorWithVariantExtensionType() { + Variant variant1 = TestVariant.variantString("hello"); + Variant variant2 = TestVariant.variantString("world"); + try (final MapVector inVector = MapVector.empty("map", allocator, false)) { + inVector.allocateNew(); + UnionMapWriter writer = inVector.getWriter(); + writer.setPosition(0); + + writer.startMap(); + writer.startEntry(); + writer.key().bigInt().writeBigInt(0); + writer.value().extension(VariantType.INSTANCE).writeExtension(variant1, VariantType.INSTANCE); + writer.endEntry(); + writer.startEntry(); + writer.key().bigInt().writeBigInt(1); + writer.value().extension(VariantType.INSTANCE).writeExtension(variant2, VariantType.INSTANCE); + writer.endEntry(); + writer.endMap(); + + writer.setValueCount(1); + + UnionMapReader mapReader = inVector.getReader(); + mapReader.setPosition(0); + mapReader.next(); + FieldReader variantReader = mapReader.value(); + NullableVariantHolder holder = new NullableVariantHolder(); + variantReader.read(holder); + assertEquals(variant1, new Variant(holder)); + + mapReader.next(); + variantReader = mapReader.value(); + variantReader.read(holder); + assertEquals(variant2, new Variant(holder)); + } + } + + @Test + public void testCopyFromForVariantExtensionType() { + Variant variant1 = TestVariant.variantString("hello"); + Variant variant2 = TestVariant.variantString("world"); + try (final MapVector inVector = MapVector.empty("in", allocator, false); + final MapVector outVector = MapVector.empty("out", allocator, false)) { + inVector.allocateNew(); + UnionMapWriter writer = inVector.getWriter(); + writer.setPosition(0); + + writer.startMap(); + writer.startEntry(); + writer.key().bigInt().writeBigInt(0); + writer.value().extension(VariantType.INSTANCE).writeExtension(variant1, VariantType.INSTANCE); + writer.endEntry(); + writer.startEntry(); + writer.key().bigInt().writeBigInt(1); + writer.value().extension(VariantType.INSTANCE).writeExtension(variant2, VariantType.INSTANCE); + writer.endEntry(); + writer.endMap(); + + writer.setValueCount(1); + outVector.allocateNew(); + outVector.copyFrom(0, 0, inVector); + outVector.setValueCount(1); + + UnionMapReader mapReader = outVector.getReader(); + mapReader.setPosition(0); + mapReader.next(); + FieldReader variantReader = mapReader.value(); + NullableVariantHolder holder = new NullableVariantHolder(); + variantReader.read(holder); + assertEquals(variant1, new Variant(holder)); + + mapReader.next(); + variantReader = mapReader.value(); + variantReader.read(holder); + assertEquals(variant2, new Variant(holder)); + } + } +} diff --git a/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantType.java b/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantType.java new file mode 100644 index 0000000000..017e71224b --- /dev/null +++ b/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantType.java @@ -0,0 +1,308 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant.extension; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.variant.holders.NullableVariantHolder; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.dictionary.DictionaryProvider; +import org.apache.arrow.vector.ipc.ArrowStreamReader; +import org.apache.arrow.vector.ipc.ArrowStreamWriter; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ExtensionTypeRegistry; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class TestVariantType { + BufferAllocator allocator; + + @BeforeEach + void beforeEach() { + allocator = new RootAllocator(); + } + + @AfterEach + void afterEach() { + allocator.close(); + } + + @Test + void testConstants() { + assertNotNull(VariantType.INSTANCE); + } + + @Test + void testStorageType() { + VariantType type = VariantType.INSTANCE; + assertEquals(ArrowType.Struct.INSTANCE, type.storageType()); + assertInstanceOf(ArrowType.Struct.class, type.storageType()); + } + + @Test + void testExtensionName() { + VariantType type = VariantType.INSTANCE; + assertEquals("parquet.variant", type.extensionName()); + } + + @Test + void testExtensionEquals() { + VariantType type1 = VariantType.INSTANCE; + VariantType type2 = VariantType.INSTANCE; + + assertTrue(type1.extensionEquals(type2)); + } + + @Test + void testIsComplex() { + VariantType type = VariantType.INSTANCE; + assertFalse(type.isComplex()); + } + + @Test + void testSerialize() { + VariantType type = VariantType.INSTANCE; + String serialized = type.serialize(); + assertEquals("", serialized); + } + + @Test + void testDeserializeValid() { + VariantType type = VariantType.INSTANCE; + ArrowType storageType = ArrowType.Struct.INSTANCE; + + ArrowType deserialized = assertDoesNotThrow(() -> type.deserialize(storageType, "")); + assertInstanceOf(VariantType.class, deserialized); + assertEquals(VariantType.INSTANCE, deserialized); + } + + @Test + void testDeserializeInvalidStorageType() { + VariantType type = VariantType.INSTANCE; + ArrowType wrongStorageType = ArrowType.Utf8.INSTANCE; + + assertThrows(UnsupportedOperationException.class, () -> type.deserialize(wrongStorageType, "")); + } + + @Test + void testGetNewVector() { + VariantType type = VariantType.INSTANCE; + try (FieldVector vector = + type.getNewVector("variant_field", FieldType.nullable(type), allocator)) { + assertInstanceOf(VariantVector.class, vector); + assertEquals("variant_field", vector.getField().getName()); + assertEquals(type, vector.getField().getType()); + } + } + + @Test + void testGetNewVectorWithNullableFieldType() { + VariantType type = VariantType.INSTANCE; + FieldType nullableFieldType = FieldType.nullable(type); + + try (FieldVector vector = type.getNewVector("nullable_variant", nullableFieldType, allocator)) { + assertInstanceOf(VariantVector.class, vector); + assertEquals("nullable_variant", vector.getField().getName()); + assertTrue(vector.getField().isNullable()); + } + } + + @Test + void testGetNewVectorWithNonNullableFieldType() { + VariantType type = VariantType.INSTANCE; + FieldType nonNullableFieldType = FieldType.notNullable(type); + + try (FieldVector vector = + type.getNewVector("non_nullable_variant", nonNullableFieldType, allocator)) { + assertInstanceOf(VariantVector.class, vector); + assertEquals("non_nullable_variant", vector.getField().getName()); + } + } + + @Test + void testIpcRoundTrip() { + VariantType type = VariantType.INSTANCE; + + Schema schema = new Schema(Collections.singletonList(Field.nullable("variant", type))); + byte[] serialized = schema.serializeAsMessage(); + Schema deserialized = Schema.deserializeMessage(ByteBuffer.wrap(serialized)); + assertEquals(schema, deserialized); + } + + @Test + void testVectorIpcRoundTrip() throws IOException { + VariantType type = VariantType.INSTANCE; + + try (FieldVector vector = type.getNewVector("field", FieldType.nullable(type), allocator); + ArrowBuf metadataBuf1 = allocator.buffer(10); + ArrowBuf valueBuf1 = allocator.buffer(10); + ArrowBuf metadataBuf2 = allocator.buffer(10); + ArrowBuf valueBuf2 = allocator.buffer(10)) { + VariantVector variantVector = (VariantVector) vector; + + byte[] metadata1 = new byte[] {1, 2, 3}; + byte[] value1 = new byte[] {4, 5, 6, 7}; + metadataBuf1.setBytes(0, metadata1); + valueBuf1.setBytes(0, value1); + + byte[] metadata2 = new byte[] {8, 9}; + byte[] value2 = new byte[] {10, 11, 12}; + metadataBuf2.setBytes(0, metadata2); + valueBuf2.setBytes(0, value2); + + NullableVariantHolder holder1 = new NullableVariantHolder(); + holder1.isSet = 1; + holder1.metadataStart = 0; + holder1.metadataEnd = metadata1.length; + holder1.metadataBuffer = metadataBuf1; + holder1.valueStart = 0; + holder1.valueEnd = value1.length; + holder1.valueBuffer = valueBuf1; + + NullableVariantHolder holder2 = new NullableVariantHolder(); + holder2.isSet = 1; + holder2.metadataStart = 0; + holder2.metadataEnd = metadata2.length; + holder2.metadataBuffer = metadataBuf2; + holder2.valueStart = 0; + holder2.valueEnd = value2.length; + holder2.valueBuffer = valueBuf2; + + variantVector.setSafe(0, holder1); + variantVector.setNull(1); + variantVector.setSafe(2, holder2); + variantVector.setValueCount(3); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (VectorSchemaRoot root = new VectorSchemaRoot(Collections.singletonList(variantVector)); + ArrowStreamWriter writer = + new ArrowStreamWriter(root, new DictionaryProvider.MapDictionaryProvider(), baos)) { + writer.start(); + writer.writeBatch(); + } + + try (ArrowStreamReader reader = + new ArrowStreamReader(new ByteArrayInputStream(baos.toByteArray()), allocator)) { + assertTrue(reader.loadNextBatch()); + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + assertEquals(3, root.getRowCount()); + assertEquals( + new Schema(Collections.singletonList(variantVector.getField())), root.getSchema()); + + VariantVector actual = assertInstanceOf(VariantVector.class, root.getVector("field")); + assertFalse(actual.isNull(0)); + assertTrue(actual.isNull(1)); + assertFalse(actual.isNull(2)); + + NullableVariantHolder result1 = new NullableVariantHolder(); + actual.get(0, result1); + assertEquals(1, result1.isSet); + assertEquals(metadata1.length, result1.metadataEnd - result1.metadataStart); + assertEquals(value1.length, result1.valueEnd - result1.valueStart); + + assertNull(actual.getObject(1)); + + NullableVariantHolder result2 = new NullableVariantHolder(); + actual.get(2, result2); + assertEquals(1, result2.isSet); + assertEquals(metadata2.length, result2.metadataEnd - result2.metadataStart); + assertEquals(value2.length, result2.valueEnd - result2.valueStart); + } + } + } + + @Test + void testSingleton() { + VariantType type1 = VariantType.INSTANCE; + VariantType type2 = VariantType.INSTANCE; + + // Same instance + assertSame(type1, type2); + assertTrue(type1.extensionEquals(type2)); + } + + @Test + void testExtensionTypeRegistry() { + // VariantType should be automatically registered via static initializer + ArrowType.ExtensionType registeredType = + ExtensionTypeRegistry.lookup(VariantType.EXTENSION_NAME); + assertNotNull(registeredType); + assertInstanceOf(VariantType.class, registeredType); + assertEquals(VariantType.INSTANCE, registeredType); + } + + @Test + void testFieldMetadata() { + Map metadata = new HashMap<>(); + metadata.put("key1", "value1"); + metadata.put("key2", "value2"); + + FieldType fieldType = new FieldType(true, VariantType.INSTANCE, null, metadata); + try (VariantVector vector = new VariantVector("test", allocator)) { + Field field = new Field("test", fieldType, VariantVector.createVariantChildFields()); + + // Field metadata includes both custom metadata and extension type metadata + Map fieldMetadata = field.getMetadata(); + assertEquals("value1", fieldMetadata.get("key1")); + assertEquals("value2", fieldMetadata.get("key2")); + // Extension type metadata is also present + assertTrue(fieldMetadata.containsKey("ARROW:extension:name")); + assertTrue(fieldMetadata.containsKey("ARROW:extension:metadata")); + } + } + + @Test + void testFieldChildren() { + try (VariantVector vector = new VariantVector("test", allocator)) { + Field field = vector.getField(); + + assertNotNull(field.getChildren()); + assertEquals(2, field.getChildren().size()); + + Field metadataField = field.getChildren().get(0); + assertEquals(VariantVector.METADATA_VECTOR_NAME, metadataField.getName()); + assertEquals(ArrowType.Binary.INSTANCE, metadataField.getType()); + + Field valueField = field.getChildren().get(1); + assertEquals(VariantVector.VALUE_VECTOR_NAME, valueField.getName()); + assertEquals(ArrowType.Binary.INSTANCE, valueField.getType()); + } + } +} diff --git a/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantVector.java b/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantVector.java new file mode 100644 index 0000000000..1c172e304f --- /dev/null +++ b/arrow-variant/src/test/java/org/apache/arrow/variant/extension/TestVariantVector.java @@ -0,0 +1,844 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.variant.extension; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.variant.Variant; +import org.apache.arrow.variant.holders.NullableVariantHolder; +import org.apache.arrow.variant.holders.VariantHolder; +import org.apache.arrow.variant.impl.VariantReaderImpl; +import org.apache.arrow.variant.impl.VariantWriterImpl; +import org.apache.arrow.vector.holders.ExtensionHolder; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** Tests for VariantVector, VariantWriterImpl, and VariantReaderImpl. */ +class TestVariantVector { + + private BufferAllocator allocator; + + @BeforeEach + void beforeEach() { + allocator = new RootAllocator(); + } + + @AfterEach + void afterEach() { + allocator.close(); + } + + private VariantHolder createHolder( + ArrowBuf metadataBuf, byte[] metadata, ArrowBuf valueBuf, byte[] value) { + VariantHolder holder = new VariantHolder(); + holder.metadataStart = 0; + holder.metadataEnd = metadata.length; + holder.metadataBuffer = metadataBuf; + holder.valueStart = 0; + holder.valueEnd = value.length; + holder.valueBuffer = valueBuf; + return holder; + } + + private NullableVariantHolder createNullableHolder( + ArrowBuf metadataBuf, byte[] metadata, ArrowBuf valueBuf, byte[] value) { + NullableVariantHolder holder = new NullableVariantHolder(); + holder.isSet = 1; + holder.metadataStart = 0; + holder.metadataEnd = metadata.length; + holder.metadataBuffer = metadataBuf; + holder.valueStart = 0; + holder.valueEnd = value.length; + holder.valueBuffer = valueBuf; + return holder; + } + + private NullableVariantHolder createNullHolder() { + NullableVariantHolder holder = new NullableVariantHolder(); + holder.isSet = 0; + return holder; + } + + // ========== Basic Vector Tests ========== + + @Test + void testVectorCreation() { + try (VariantVector vector = new VariantVector("test", allocator)) { + assertNotNull(vector); + assertEquals("test", vector.getField().getName()); + assertNotNull(vector.getMetadataVector()); + assertNotNull(vector.getValueVector()); + } + } + + @Test + void testSetAndGet() { + try (VariantVector vector = new VariantVector("test", allocator); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1, 2, 3}; + byte[] value = new byte[] {4, 5, 6, 7}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + NullableVariantHolder holder = createNullableHolder(metadataBuf, metadata, valueBuf, value); + + vector.setSafe(0, holder); + vector.setValueCount(1); + + // Retrieve and verify + NullableVariantHolder result = new NullableVariantHolder(); + vector.get(0, result); + + assertEquals(1, result.isSet); + assertEquals(metadata.length, result.metadataEnd - result.metadataStart); + assertEquals(value.length, result.valueEnd - result.valueStart); + + byte[] actualMetadata = new byte[metadata.length]; + byte[] actualValue = new byte[value.length]; + result.metadataBuffer.getBytes(result.metadataStart, actualMetadata); + result.valueBuffer.getBytes(result.valueStart, actualValue); + + assertArrayEquals(metadata, actualMetadata); + assertArrayEquals(value, actualValue); + } + } + + @Test + void testSetNull() { + try (VariantVector vector = new VariantVector("test", allocator)) { + NullableVariantHolder holder = createNullHolder(); + + vector.setSafe(0, holder); + vector.setValueCount(1); + + assertTrue(vector.isNull(0)); + + NullableVariantHolder result = new NullableVariantHolder(); + vector.get(0, result); + assertEquals(0, result.isSet); + } + } + + @Test + void testMultipleValues() { + try (VariantVector vector = new VariantVector("test", allocator); + ArrowBuf metadataBuf1 = allocator.buffer(10); + ArrowBuf valueBuf1 = allocator.buffer(10); + ArrowBuf metadataBuf2 = allocator.buffer(10); + ArrowBuf valueBuf2 = allocator.buffer(10)) { + + byte[] metadata1 = new byte[] {1, 2}; + byte[] value1 = new byte[] {3, 4, 5}; + metadataBuf1.setBytes(0, metadata1); + valueBuf1.setBytes(0, value1); + + NullableVariantHolder holder1 = + createNullableHolder(metadataBuf1, metadata1, valueBuf1, value1); + + byte[] metadata2 = new byte[] {6, 7, 8}; + byte[] value2 = new byte[] {9, 10}; + metadataBuf2.setBytes(0, metadata2); + valueBuf2.setBytes(0, value2); + + NullableVariantHolder holder2 = + createNullableHolder(metadataBuf2, metadata2, valueBuf2, value2); + + vector.setSafe(0, holder1); + vector.setSafe(1, holder2); + vector.setValueCount(2); + + // Verify first value + NullableVariantHolder result1 = new NullableVariantHolder(); + vector.get(0, result1); + assertEquals(1, result1.isSet); + + byte[] actualMetadata1 = new byte[metadata1.length]; + byte[] actualValue1 = new byte[value1.length]; + result1.metadataBuffer.getBytes(result1.metadataStart, actualMetadata1); + result1.valueBuffer.getBytes(result1.valueStart, actualValue1); + assertArrayEquals(metadata1, actualMetadata1); + assertArrayEquals(value1, actualValue1); + + // Verify second value + NullableVariantHolder result2 = new NullableVariantHolder(); + vector.get(1, result2); + assertEquals(1, result2.isSet); + + byte[] actualMetadata2 = new byte[metadata2.length]; + byte[] actualValue2 = new byte[value2.length]; + result2.metadataBuffer.getBytes(result2.metadataStart, actualMetadata2); + result2.valueBuffer.getBytes(result2.valueStart, actualValue2); + assertArrayEquals(metadata2, actualMetadata2); + assertArrayEquals(value2, actualValue2); + } + } + + @Test + void testNonNullableHolder() { + try (VariantVector vector = new VariantVector("test", allocator); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1, 2, 3}; + byte[] value = new byte[] {4, 5, 6}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + VariantHolder holder = createHolder(metadataBuf, metadata, valueBuf, value); + + vector.setSafe(0, holder); + vector.setValueCount(1); + + assertFalse(vector.isNull(0)); + + NullableVariantHolder result = new NullableVariantHolder(); + vector.get(0, result); + assertEquals(1, result.isSet); + } + } + + // ========== Writer Tests ========== + + @Test + void testWriteWithVariantHolder() { + try (VariantVector vector = new VariantVector("test", allocator); + VariantWriterImpl writer = new VariantWriterImpl(vector); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1, 2}; + byte[] value = new byte[] {3, 4, 5}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + VariantHolder holder = createHolder(metadataBuf, metadata, valueBuf, value); + + writer.setPosition(0); + writer.write(holder); + + assertEquals(1, vector.getValueCount()); + assertFalse(vector.isNull(0)); + } + } + + @Test + void testWriteWithNullableVariantHolder() { + try (VariantVector vector = new VariantVector("test", allocator); + VariantWriterImpl writer = new VariantWriterImpl(vector); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1, 2}; + byte[] value = new byte[] {3, 4, 5}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + NullableVariantHolder holder = createNullableHolder(metadataBuf, metadata, valueBuf, value); + + writer.setPosition(0); + writer.write(holder); + + assertEquals(1, vector.getValueCount()); + assertFalse(vector.isNull(0)); + } + } + + @Test + void testWriteWithNullableVariantHolderNull() { + try (VariantVector vector = new VariantVector("test", allocator); + VariantWriterImpl writer = new VariantWriterImpl(vector)) { + + NullableVariantHolder holder = createNullHolder(); + + writer.setPosition(0); + writer.write(holder); + + assertEquals(1, vector.getValueCount()); + assertTrue(vector.isNull(0)); + } + } + + @Test + void testWriteExtensionWithUnsupportedType() { + try (VariantVector vector = new VariantVector("test", allocator); + VariantWriterImpl writer = new VariantWriterImpl(vector)) { + + writer.setPosition(0); + + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> writer.writeExtension("invalid-type")); + + assertTrue(exception.getMessage().contains("Unsupported type for Variant")); + } + } + + @Test + void testWriteWithUnsupportedHolder() { + try (VariantVector vector = new VariantVector("test", allocator); + VariantWriterImpl writer = new VariantWriterImpl(vector)) { + + ExtensionHolder unsupportedHolder = + new ExtensionHolder() { + @Override + public ArrowType type() { + return VariantType.INSTANCE; + } + }; + + writer.setPosition(0); + + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> writer.write(unsupportedHolder)); + + assertTrue(exception.getMessage().contains("Unsupported type for Variant")); + } + } + + // ========== Reader Tests ========== + + @Test + void testReaderReadWithNullableVariantHolder() { + try (VariantVector vector = new VariantVector("test", allocator); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1, 2, 3}; + byte[] value = new byte[] {4, 5, 6}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + NullableVariantHolder holder = createNullableHolder(metadataBuf, metadata, valueBuf, value); + + vector.setSafe(0, holder); + vector.setValueCount(1); + + VariantReaderImpl reader = (VariantReaderImpl) vector.getReader(); + reader.setPosition(0); + + NullableVariantHolder result = new NullableVariantHolder(); + reader.read(result); + + assertEquals(1, result.isSet); + assertEquals(metadata.length, result.metadataEnd - result.metadataStart); + assertEquals(value.length, result.valueEnd - result.valueStart); + } + } + + @Test + void testReaderReadWithNullableVariantHolderNull() { + try (VariantVector vector = new VariantVector("test", allocator)) { + vector.setNull(0); + vector.setValueCount(1); + + VariantReaderImpl reader = (VariantReaderImpl) vector.getReader(); + reader.setPosition(0); + + NullableVariantHolder holder = new NullableVariantHolder(); + reader.read(holder); + + assertEquals(0, holder.isSet); + } + } + + @Test + void testReaderIsSet() { + try (VariantVector vector = new VariantVector("test", allocator); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1}; + byte[] value = new byte[] {2}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + NullableVariantHolder holder = createNullableHolder(metadataBuf, metadata, valueBuf, value); + + vector.setSafe(0, holder); + vector.setNull(1); + vector.setValueCount(2); + + VariantReaderImpl reader = (VariantReaderImpl) vector.getReader(); + + reader.setPosition(0); + assertTrue(reader.isSet()); + + reader.setPosition(1); + assertFalse(reader.isSet()); + } + } + + @Test + void testReaderGetMinorType() { + try (VariantVector vector = new VariantVector("test", allocator)) { + VariantReaderImpl reader = (VariantReaderImpl) vector.getReader(); + assertEquals(vector.getMinorType(), reader.getMinorType()); + } + } + + @Test + void testReaderGetField() { + try (VariantVector vector = new VariantVector("test", allocator)) { + VariantReaderImpl reader = (VariantReaderImpl) vector.getReader(); + assertEquals(vector.getField(), reader.getField()); + assertEquals("test", reader.getField().getName()); + } + } + + @Test + void testReaderReadWithNonNullableVariantHolder() { + try (VariantVector vector = new VariantVector("test", allocator); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1, 2, 3}; + byte[] value = new byte[] {4, 5, 6}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + NullableVariantHolder holder = createNullableHolder(metadataBuf, metadata, valueBuf, value); + + vector.setSafe(0, holder); + vector.setValueCount(1); + + VariantReaderImpl reader = (VariantReaderImpl) vector.getReader(); + reader.setPosition(0); + + VariantHolder result = new VariantHolder(); + reader.read(result); + + // Verify the data was read correctly + byte[] actualMetadata = new byte[metadata.length]; + byte[] actualValue = new byte[value.length]; + result.metadataBuffer.getBytes(result.metadataStart, actualMetadata); + result.valueBuffer.getBytes(result.valueStart, actualValue); + + assertArrayEquals(metadata, actualMetadata); + assertArrayEquals(value, actualValue); + assertEquals(1, result.isSet); + } + } + + // ========== Transfer Pair Tests ========== + + @Test + void testTransferPair() { + try (VariantVector fromVector = new VariantVector("from", allocator); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1, 2, 3}; + byte[] value = new byte[] {4, 5, 6, 7}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + NullableVariantHolder holder = createNullableHolder(metadataBuf, metadata, valueBuf, value); + + fromVector.setSafe(0, holder); + fromVector.setValueCount(1); + + org.apache.arrow.vector.util.TransferPair transferPair = + fromVector.getTransferPair(allocator); + VariantVector toVector = (VariantVector) transferPair.getTo(); + + transferPair.transfer(); + + assertEquals(0, fromVector.getValueCount()); + assertEquals(1, toVector.getValueCount()); + + NullableVariantHolder result = new NullableVariantHolder(); + toVector.get(0, result); + assertEquals(1, result.isSet); + + byte[] actualMetadata = new byte[metadata.length]; + byte[] actualValue = new byte[value.length]; + result.metadataBuffer.getBytes(result.metadataStart, actualMetadata); + result.valueBuffer.getBytes(result.valueStart, actualValue); + + assertArrayEquals(metadata, actualMetadata); + assertArrayEquals(value, actualValue); + + toVector.close(); + } + } + + @Test + void testSplitAndTransfer() { + try (VariantVector fromVector = new VariantVector("from", allocator); + ArrowBuf metadataBuf1 = allocator.buffer(10); + ArrowBuf valueBuf1 = allocator.buffer(10); + ArrowBuf metadataBuf2 = allocator.buffer(10); + ArrowBuf valueBuf2 = allocator.buffer(10); + ArrowBuf metadataBuf3 = allocator.buffer(10); + ArrowBuf valueBuf3 = allocator.buffer(10)) { + + byte[] metadata1 = new byte[] {1}; + byte[] value1 = new byte[] {2, 3}; + metadataBuf1.setBytes(0, metadata1); + valueBuf1.setBytes(0, value1); + + byte[] metadata2 = new byte[] {4, 5}; + byte[] value2 = new byte[] {6}; + metadataBuf2.setBytes(0, metadata2); + valueBuf2.setBytes(0, value2); + + byte[] metadata3 = new byte[] {7, 8, 9}; + byte[] value3 = new byte[] {10, 11, 12}; + metadataBuf3.setBytes(0, metadata3); + valueBuf3.setBytes(0, value3); + + NullableVariantHolder holder1 = + createNullableHolder(metadataBuf1, metadata1, valueBuf1, value1); + NullableVariantHolder holder2 = + createNullableHolder(metadataBuf2, metadata2, valueBuf2, value2); + NullableVariantHolder holder3 = + createNullableHolder(metadataBuf3, metadata3, valueBuf3, value3); + + fromVector.setSafe(0, holder1); + fromVector.setSafe(1, holder2); + fromVector.setSafe(2, holder3); + fromVector.setValueCount(3); + + org.apache.arrow.vector.util.TransferPair transferPair = + fromVector.getTransferPair(allocator); + VariantVector toVector = (VariantVector) transferPair.getTo(); + + // Split and transfer indices 1-2 (middle and last) + transferPair.splitAndTransfer(1, 2); + + assertEquals(2, toVector.getValueCount()); + + // Verify transferred values + NullableVariantHolder result1 = new NullableVariantHolder(); + toVector.get(0, result1); + assertEquals(1, result1.isSet); + + byte[] actualMetadata1 = new byte[metadata2.length]; + byte[] actualValue1 = new byte[value2.length]; + result1.metadataBuffer.getBytes(result1.metadataStart, actualMetadata1); + result1.valueBuffer.getBytes(result1.valueStart, actualValue1); + assertArrayEquals(metadata2, actualMetadata1); + assertArrayEquals(value2, actualValue1); + + NullableVariantHolder result2 = new NullableVariantHolder(); + toVector.get(1, result2); + assertEquals(1, result2.isSet); + + byte[] actualMetadata2 = new byte[metadata3.length]; + byte[] actualValue2 = new byte[value3.length]; + result2.metadataBuffer.getBytes(result2.metadataStart, actualMetadata2); + result2.valueBuffer.getBytes(result2.valueStart, actualValue2); + assertArrayEquals(metadata3, actualMetadata2); + assertArrayEquals(value3, actualValue2); + + toVector.close(); + } + } + + @Test + void testCopyValueSafe() { + try (VariantVector fromVector = new VariantVector("from", allocator); + VariantVector toVector = new VariantVector("to", allocator); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1, 2}; + byte[] value = new byte[] {3, 4, 5}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + NullableVariantHolder holder = createNullableHolder(metadataBuf, metadata, valueBuf, value); + + fromVector.setSafe(0, holder); + fromVector.setValueCount(1); + + org.apache.arrow.vector.util.TransferPair transferPair = + fromVector.makeTransferPair(toVector); + + transferPair.copyValueSafe(0, 0); + toVector.setValueCount(1); + + // Verify the value was copied + NullableVariantHolder result = new NullableVariantHolder(); + toVector.get(0, result); + assertEquals(1, result.isSet); + + byte[] actualMetadata = new byte[metadata.length]; + byte[] actualValue = new byte[value.length]; + result.metadataBuffer.getBytes(result.metadataStart, actualMetadata); + result.valueBuffer.getBytes(result.valueStart, actualValue); + + assertArrayEquals(metadata, actualMetadata); + assertArrayEquals(value, actualValue); + + // Original vector should still have the value + NullableVariantHolder originalResult = new NullableVariantHolder(); + fromVector.get(0, originalResult); + assertEquals(1, originalResult.isSet); + } + } + + @Test + void testGetTransferPairWithField() { + try (VariantVector fromVector = new VariantVector("from", allocator); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1}; + byte[] value = new byte[] {2}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + NullableVariantHolder holder = createNullableHolder(metadataBuf, metadata, valueBuf, value); + + fromVector.setSafe(0, holder); + fromVector.setValueCount(1); + + org.apache.arrow.vector.util.TransferPair transferPair = + fromVector.getTransferPair(fromVector.getField(), allocator); + VariantVector toVector = (VariantVector) transferPair.getTo(); + + transferPair.transfer(); + + assertEquals(1, toVector.getValueCount()); + assertEquals(fromVector.getField().getName(), toVector.getField().getName()); + + toVector.close(); + } + } + + // ========== Copy Operations Tests ========== + + @Test + void testCopyFrom() { + try (VariantVector fromVector = new VariantVector("from", allocator); + VariantVector toVector = new VariantVector("to", allocator); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1, 2, 3}; + byte[] value = new byte[] {4, 5}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + NullableVariantHolder holder = createNullableHolder(metadataBuf, metadata, valueBuf, value); + + fromVector.setSafe(0, holder); + fromVector.setValueCount(1); + + toVector.allocateNew(); + toVector.copyFrom(0, 0, fromVector); + toVector.setValueCount(1); + + NullableVariantHolder result = new NullableVariantHolder(); + toVector.get(0, result); + assertEquals(1, result.isSet); + + byte[] actualMetadata = new byte[metadata.length]; + byte[] actualValue = new byte[value.length]; + result.metadataBuffer.getBytes(result.metadataStart, actualMetadata); + result.valueBuffer.getBytes(result.valueStart, actualValue); + + assertArrayEquals(metadata, actualMetadata); + assertArrayEquals(value, actualValue); + } + } + + @Test + void testCopyFromSafe() { + try (VariantVector fromVector = new VariantVector("from", allocator); + VariantVector toVector = new VariantVector("to", allocator); + ArrowBuf metadataBuf1 = allocator.buffer(10); + ArrowBuf valueBuf1 = allocator.buffer(10); + ArrowBuf metadataBuf2 = allocator.buffer(10); + ArrowBuf valueBuf2 = allocator.buffer(10)) { + + byte[] metadata1 = new byte[] {1}; + byte[] value1 = new byte[] {2, 3}; + metadataBuf1.setBytes(0, metadata1); + valueBuf1.setBytes(0, value1); + + NullableVariantHolder holder1 = + createNullableHolder(metadataBuf1, metadata1, valueBuf1, value1); + + byte[] metadata2 = new byte[] {4, 5}; + byte[] value2 = new byte[] {6}; + metadataBuf2.setBytes(0, metadata2); + valueBuf2.setBytes(0, value2); + + NullableVariantHolder holder2 = + createNullableHolder(metadataBuf2, metadata2, valueBuf2, value2); + + fromVector.setSafe(0, holder1); + fromVector.setSafe(1, holder2); + fromVector.setValueCount(2); + + // Copy without pre-allocating toVector + for (int i = 0; i < 2; i++) { + toVector.copyFromSafe(i, i, fromVector); + } + toVector.setValueCount(2); + + // Verify both values + NullableVariantHolder result1 = new NullableVariantHolder(); + toVector.get(0, result1); + assertEquals(1, result1.isSet); + + byte[] actualMetadata1 = new byte[metadata1.length]; + byte[] actualValue1 = new byte[value1.length]; + result1.metadataBuffer.getBytes(result1.metadataStart, actualMetadata1); + result1.valueBuffer.getBytes(result1.valueStart, actualValue1); + assertArrayEquals(metadata1, actualMetadata1); + assertArrayEquals(value1, actualValue1); + + NullableVariantHolder result2 = new NullableVariantHolder(); + toVector.get(1, result2); + assertEquals(1, result2.isSet); + + byte[] actualMetadata2 = new byte[metadata2.length]; + byte[] actualValue2 = new byte[value2.length]; + result2.metadataBuffer.getBytes(result2.metadataStart, actualMetadata2); + result2.valueBuffer.getBytes(result2.valueStart, actualValue2); + assertArrayEquals(metadata2, actualMetadata2); + assertArrayEquals(value2, actualValue2); + } + } + + @Test + void testCopyFromWithNulls() { + try (VariantVector fromVector = new VariantVector("from", allocator); + VariantVector toVector = new VariantVector("to", allocator); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1}; + byte[] value = new byte[] {2}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + NullableVariantHolder holder = createNullableHolder(metadataBuf, metadata, valueBuf, value); + + fromVector.setSafe(0, holder); + fromVector.setNull(1); + fromVector.setSafe(2, holder); + fromVector.setValueCount(3); + + toVector.allocateNew(); + for (int i = 0; i < 3; i++) { + toVector.copyFromSafe(i, i, fromVector); + } + toVector.setValueCount(3); + + assertFalse(toVector.isNull(0)); + assertTrue(toVector.isNull(1)); + assertFalse(toVector.isNull(2)); + } + } + + // ========== GetObject Tests ========== + + @Test + void testGetObject() { + try (VariantVector vector = new VariantVector("test", allocator); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1, 2}; + byte[] value = new byte[] {3, 4, 5}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + NullableVariantHolder holder = createNullableHolder(metadataBuf, metadata, valueBuf, value); + + vector.setSafe(0, holder); + vector.setValueCount(1); + + Object obj = vector.getObject(0); + assertNotNull(obj); + assertTrue(obj instanceof Variant); + assertEquals(new Variant(metadata, value), obj); + } + } + + @Test + void testGetObjectNull() { + try (VariantVector vector = new VariantVector("test", allocator)) { + vector.setNull(0); + vector.setValueCount(1); + + Object obj = vector.getObject(0); + assertNull(obj); + } + } + + // ========== Allocate and Capacity Tests ========== + + @Test + void testAllocateNew() { + try (VariantVector vector = new VariantVector("test", allocator)) { + vector.allocateNew(); + assertTrue(vector.getValueCapacity() > 0); + } + } + + @Test + void testSetInitialCapacity() { + try (VariantVector vector = new VariantVector("test", allocator)) { + vector.setInitialCapacity(100); + vector.allocateNew(); + assertTrue(vector.getValueCapacity() >= 100); + } + } + + @Test + void testClearAndReuse() { + try (VariantVector vector = new VariantVector("test", allocator); + ArrowBuf metadataBuf = allocator.buffer(10); + ArrowBuf valueBuf = allocator.buffer(10)) { + + byte[] metadata = new byte[] {1}; + byte[] value = new byte[] {2}; + metadataBuf.setBytes(0, metadata); + valueBuf.setBytes(0, value); + + NullableVariantHolder holder = createNullableHolder(metadataBuf, metadata, valueBuf, value); + + vector.setSafe(0, holder); + vector.setValueCount(1); + + assertFalse(vector.isNull(0)); + + vector.clear(); + vector.allocateNew(); + + // After clear, vector should be empty + assertEquals(0, vector.getValueCount()); + } + } +} diff --git a/bom/pom.xml b/bom/pom.xml index 9efde53243..b631f9366d 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -194,6 +194,11 @@ under the License. arrow-tools ${project.version} + + org.apache.arrow + arrow-variant + ${project.version} + diff --git a/pom.xml b/pom.xml index d64df1ade3..9ffaf6b60d 100644 --- a/pom.xml +++ b/pom.xml @@ -68,6 +68,7 @@ under the License. bom format memory + arrow-variant vector tools adapter/jdbc @@ -104,6 +105,7 @@ under the License. 3.4.2 25.2.10 1.12.1 + 1.17.0 5.17.0 2 diff --git a/vector/src/main/codegen/templates/AbstractFieldReader.java b/vector/src/main/codegen/templates/AbstractFieldReader.java index 556fb576ce..789295e959 100644 --- a/vector/src/main/codegen/templates/AbstractFieldReader.java +++ b/vector/src/main/codegen/templates/AbstractFieldReader.java @@ -29,9 +29,9 @@ * Source code generated using FreeMarker template ${.template_name} */ @SuppressWarnings("unused") -abstract class AbstractFieldReader extends AbstractBaseReader implements FieldReader{ +public abstract class AbstractFieldReader extends AbstractBaseReader implements FieldReader{ - AbstractFieldReader(){ + protected AbstractFieldReader(){ super(); } From 9d6237ed4761b0ae923499a24d1545bae6218add Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Feb 2026 07:49:34 +0100 Subject: [PATCH 148/271] MINOR: Bump org.mockito:mockito-bom from 5.17.0 to 5.21.0 (#1000) Bumps [org.mockito:mockito-bom](https://github.com/mockito/mockito) from 5.17.0 to 5.21.0.

Release notes

Sourced from org.mockito:mockito-bom's releases.

v5.21.0

Changelog generated by Shipkit Changelog Gradle Plugin

5.21.0

v5.20.0

Changelog generated by Shipkit Changelog Gradle Plugin

5.20.0

v5.19.0

Changelog generated by Shipkit Changelog Gradle Plugin

5.19.0

... (truncated)

Commits
  • 09d2230 Bump graalvm/setup-graalvm from 1.4.3 to 1.4.4 (#3768)
  • df3e0cc Bump graalvm/setup-graalvm from 1.4.2 to 1.4.3 (#3767)
  • 04a6e9f Bump actions/checkout from 5 to 6 (#3765)
  • 756a3cf Add description of matchers to potential mismatch (#3760)
  • 58ba445 Forbid mocking WeakReference with inline mock maker (#3759)
  • 966d600 Bump actions/upload-artifact from 4 to 5 (#3756)
  • 632bf7b Bump graalvm/setup-graalvm from 1.4.1 to 1.4.2 (#3755)
  • 8564b43 Fix primitives support in GenericArrayReturnType for Android (#3753)
  • bf3a809 Bump graalvm/setup-graalvm from 1.4.0 to 1.4.1 (#3744)
  • cffddd4 Bump gradle/actions from 4 to 5 (#3743)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.mockito:mockito-bom&package-manager=maven&previous-version=5.17.0&new-version=5.21.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9ffaf6b60d..1f57039015 100644 --- a/pom.xml +++ b/pom.xml @@ -106,7 +106,7 @@ under the License. 25.2.10 1.12.1 1.17.0 - 5.17.0 + 5.21.0 2 10.23.0 From 32984435ed1ffbe6ce1578e075da6eeb012d6bbb Mon Sep 17 00:00:00 2001 From: Aleksei Starikov Date: Tue, 17 Feb 2026 13:59:36 +0100 Subject: [PATCH 149/271] GH-130: Fix AutoCloseables to work with @Nullable structures (#1017) ## What's Changed `AutoCloseables` supposes to work with nullable `Iterables`, `varargs`, and `collection of nulls`. The PR introduces: - `@Nullable` annotation for all public methods in `AutoCloseables` (only private `flatten` method doesn't support null `Iterable`) - `null` checks to prevent NPEs --- The change is backward compatible. Only possible NPEs are prevented. --- Closes #130 . --- .../org/apache/arrow/util/AutoCloseables.java | 61 ++-- .../apache/arrow/util/TestAutoCloseables.java | 268 ++++++++++++++++++ 2 files changed, 311 insertions(+), 18 deletions(-) create mode 100644 memory/memory-core/src/test/java/org/apache/arrow/util/TestAutoCloseables.java diff --git a/memory/memory-core/src/main/java/org/apache/arrow/util/AutoCloseables.java b/memory/memory-core/src/main/java/org/apache/arrow/util/AutoCloseables.java index a39004a9d0..ba5a539a87 100644 --- a/memory/memory-core/src/main/java/org/apache/arrow/util/AutoCloseables.java +++ b/memory/memory-core/src/main/java/org/apache/arrow/util/AutoCloseables.java @@ -22,7 +22,9 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.stream.Stream; import java.util.stream.StreamSupport; +import org.checkerframework.checker.nullness.qual.Nullable; /** Utilities for AutoCloseable classes. */ public final class AutoCloseables { @@ -33,7 +35,8 @@ private AutoCloseables() {} * Returns a new {@link AutoCloseable} that calls {@link #close(Iterable)} on autoCloseables * when close is called. */ - public static AutoCloseable all(final Collection autoCloseables) { + public static AutoCloseable all( + final @Nullable Collection autoCloseables) { return new AutoCloseable() { @Override public void close() throws Exception { @@ -48,7 +51,10 @@ public void close() throws Exception { * @param t the throwable to add suppressed exception to * @param autoCloseables the closeables to close */ - public static void close(Throwable t, AutoCloseable... autoCloseables) { + public static void close(Throwable t, @Nullable AutoCloseable... autoCloseables) { + if (autoCloseables == null) { + return; + } close(t, Arrays.asList(autoCloseables)); } @@ -58,7 +64,8 @@ public static void close(Throwable t, AutoCloseable... autoCloseables) { * @param t the throwable to add suppressed exception to * @param autoCloseables the closeables to close */ - public static void close(Throwable t, Iterable autoCloseables) { + public static void close( + Throwable t, @Nullable Iterable autoCloseables) { try { close(autoCloseables); } catch (Exception e) { @@ -71,7 +78,10 @@ public static void close(Throwable t, Iterable autoClos * * @param autoCloseables the closeables to close */ - public static void close(AutoCloseable... autoCloseables) throws Exception { + public static void close(@Nullable AutoCloseable... autoCloseables) throws Exception { + if (autoCloseables == null) { + return; + } close(Arrays.asList(autoCloseables)); } @@ -80,7 +90,8 @@ public static void close(AutoCloseable... autoCloseables) throws Exception { * * @param ac the closeables to close */ - public static void close(Iterable ac) throws Exception { + public static void close(@Nullable Iterable ac) + throws Exception { // this method can be called on a single object if it implements Iterable // like for example VectorContainer make sure we handle that properly if (ac == null) { @@ -111,12 +122,17 @@ public static void close(Iterable ac) throws Exception /** Calls {@link #close(Iterable)} on the flattened list of closeables. */ @SafeVarargs - public static void close(Iterable... closeables) throws Exception { + public static void close(@Nullable Iterable... closeables) + throws Exception { + if (closeables == null) { + return; + } close(flatten(closeables)); } @SafeVarargs - private static Iterable flatten(Iterable... closeables) { + private static Iterable flatten( + Iterable... closeables) { return new Iterable() { // Cast from Iterable to Iterable is safe in this // context @@ -127,16 +143,18 @@ public Iterator iterator() { return Arrays.stream(closeables) .flatMap( (Iterable i) -> - StreamSupport.stream( - ((Iterable) i).spliterator(), /* parallel= */ false)) + i == null + ? Stream.empty() + : StreamSupport.stream( + ((Iterable) i).spliterator(), /* parallel= */ false)) .iterator(); } }; } /** Converts ac to a {@link Iterable} filtering out any null values. */ - public static Iterable iter(AutoCloseable... ac) { - if (ac.length == 0) { + public static Iterable iter(@Nullable AutoCloseable... ac) { + if (ac == null || ac.length == 0) { return Collections.emptyList(); } else { final List nonNullAc = new ArrayList<>(); @@ -153,10 +171,11 @@ public static Iterable iter(AutoCloseable... ac) { public static class RollbackCloseable implements AutoCloseable { private boolean commit = false; - private List closeables; + private final List closeables; - public RollbackCloseable(AutoCloseable... closeables) { - this.closeables = new ArrayList<>(Arrays.asList(closeables)); + public RollbackCloseable(@Nullable AutoCloseable... closeables) { + this.closeables = + closeables == null ? new ArrayList<>() : new ArrayList<>(Arrays.asList(closeables)); } public T add(T t) { @@ -165,12 +184,18 @@ public T add(T t) { } /** Add all of list to the rollback list. */ - public void addAll(AutoCloseable... list) { + public void addAll(@Nullable AutoCloseable... list) { + if (list == null) { + return; + } closeables.addAll(Arrays.asList(list)); } /** Add all of list to the rollback list. */ - public void addAll(Iterable list) { + public void addAll(@Nullable Iterable list) { + if (list == null) { + return; + } for (AutoCloseable ac : list) { closeables.add(ac); } @@ -189,7 +214,7 @@ public void close() throws Exception { } /** Creates an {@link RollbackCloseable} from the given closeables. */ - public static RollbackCloseable rollbackable(AutoCloseable... closeables) { + public static RollbackCloseable rollbackable(@Nullable AutoCloseable... closeables) { return new RollbackCloseable(closeables); } @@ -203,7 +228,7 @@ public static RollbackCloseable rollbackable(AutoCloseable... closeables) { * @throws RuntimeException if an Exception occurs; the Exception is wrapped by the * RuntimeException */ - public static void closeNoChecked(final AutoCloseable autoCloseable) { + public static void closeNoChecked(final @Nullable AutoCloseable autoCloseable) { if (autoCloseable != null) { try { autoCloseable.close(); diff --git a/memory/memory-core/src/test/java/org/apache/arrow/util/TestAutoCloseables.java b/memory/memory-core/src/test/java/org/apache/arrow/util/TestAutoCloseables.java new file mode 100644 index 0000000000..ba5b78178a --- /dev/null +++ b/memory/memory-core/src/test/java/org/apache/arrow/util/TestAutoCloseables.java @@ -0,0 +1,268 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import org.junit.jupiter.api.Test; + +public class TestAutoCloseables { + + /** Closeable that records that it was closed and can optionally throw. */ + private static final class TrackCloseable implements AutoCloseable { + private boolean closed; + private final Exception toThrow; + + TrackCloseable() { + this.toThrow = null; + } + + TrackCloseable(Exception toThrow) { + this.toThrow = toThrow; + } + + @Override + public void close() throws Exception { + closed = true; + if (toThrow != null) { + throw toThrow; + } + } + + boolean isClosed() { + return closed; + } + } + + @Test + public void testCloseVarargsIgnoresNulls() throws Exception { + TrackCloseable a = new TrackCloseable(); + TrackCloseable b = new TrackCloseable(); + AutoCloseables.close(a, null, b); + assertTrue(a.isClosed()); + assertTrue(b.isClosed()); + } + + @Test + public void testCloseVarargsThrowsFirstExceptionAndSuppressesRest() throws Exception { + Exception e1 = new Exception("first"); + Exception e2 = new Exception("second"); + TrackCloseable c1 = new TrackCloseable(e1); + TrackCloseable c2 = new TrackCloseable(e2); + Exception thrown = assertThrows(Exception.class, () -> AutoCloseables.close(c1, c2)); + assertEquals("first", thrown.getMessage()); + assertTrue(Arrays.asList(thrown.getSuppressed()).contains(e2)); + } + + @Test + public void testCloseIterableNullIterableReturns() throws Exception { + AutoCloseables.close((List) null); // no exception + } + + @Test + public void testCloseIterableIgnoresNullElements() throws Exception { + TrackCloseable a = new TrackCloseable(); + TrackCloseable b = new TrackCloseable(); + List list = Arrays.asList(a, null, b); + AutoCloseables.close(list); + assertTrue(a.isClosed()); + assertTrue(b.isClosed()); + } + + @Test + public void testCloseIterableWhenIterableIsAlsoAutoCloseable() throws Exception { + TrackCloseable iter = new TrackCloseable(); + TrackCloseable inner = new TrackCloseable(); + // When the Iterable itself implements AutoCloseable (e.g. VectorContainer), + // close(Iterable) calls close() on it and does not iterate over elements + class IterableCloseable implements Iterable, AutoCloseable { + @Override + @SuppressWarnings("unchecked") + public Iterator iterator() { + return (Iterator) Collections.singletonList(inner); + } + + @Override + public void close() throws Exception { + iter.close(); + } + } + AutoCloseables.close(new IterableCloseable()); + assertTrue(iter.isClosed()); + assertFalse(inner.isClosed()); + } + + @Test + public void testCloseIterableVarargsWithNullIterables() throws Exception { + TrackCloseable a = new TrackCloseable(); + TrackCloseable b = new TrackCloseable(); + TrackCloseable c = new TrackCloseable(); + List list1 = Arrays.asList(null, a, b); + List list2 = Collections.singletonList(c); + AutoCloseables.close(list1, null, list2); + assertTrue(a.isClosed()); + assertTrue(b.isClosed()); + assertTrue(c.isClosed()); + } + + @Test + public void testCloseThrowableSuppressesException() { + Exception e = new Exception("from close"); + TrackCloseable c = new TrackCloseable(e); + Exception main = new Exception("main"); + AutoCloseables.close(main, c); + assertTrue(c.isClosed()); + assertEquals(1, main.getSuppressed().length); + assertEquals(e, main.getSuppressed()[0]); + } + + @Test + public void testCloseThrowableWithNullCloseables() { + Exception main = new Exception("main"); + AutoCloseables.close(main, (AutoCloseable) null); + assertEquals(0, main.getSuppressed().length); + + AutoCloseables.close(main, (AutoCloseable[]) null); // no exception + } + + @Test + public void testIterFiltersNulls() { + TrackCloseable a = new TrackCloseable(); + TrackCloseable b = new TrackCloseable(); + Iterable it = AutoCloseables.iter(a, null, b); + List list = new ArrayList<>(); + it.forEach(list::add); + assertEquals(2, list.size()); + assertTrue(list.contains(a)); + assertTrue(list.contains(b)); + } + + @Test + public void testIterEmptyVarargs() { + Iterable it = AutoCloseables.iter(); + List list = new ArrayList<>(); + it.forEach(list::add); + assertTrue(list.isEmpty()); + } + + @Test + public void testIterWithNull() { + AutoCloseables.iter((AutoCloseable) null); // no exception + } + + @Test + public void testCloseNoCheckedWithNull() { + AutoCloseables.closeNoChecked(null); // no exception + } + + @Test + public void testCloseNoCheckedWrapsException() { + Exception e = new Exception("close failed"); + TrackCloseable c = new TrackCloseable(e); + RuntimeException re = + assertThrows(RuntimeException.class, () -> AutoCloseables.closeNoChecked(c)); + assertSame(re.getCause(), e); + assertTrue(re.getMessage().contains("close failed")); + } + + @Test + public void testNoop() throws Exception { + AutoCloseable noop = AutoCloseables.noop(); + assertSame(noop, AutoCloseables.noop()); + noop.close(); // no exception + } + + @Test + public void testAllClosesCollectionOnClose() throws Exception { + TrackCloseable a = new TrackCloseable(); + TrackCloseable b = new TrackCloseable(); + List list = Arrays.asList(a, b); + AutoCloseable all = AutoCloseables.all(list); + assertFalse(a.isClosed()); + assertFalse(b.isClosed()); + all.close(); + assertTrue(a.isClosed()); + assertTrue(b.isClosed()); + } + + @Test + public void testAllWithNullCollection() throws Exception { + AutoCloseable all = AutoCloseables.all(null); + all.close(); // no exception + } + + @Test + public void testRollbackCloseableClosesWhenNotCommitted() throws Exception { + TrackCloseable a = new TrackCloseable(); + TrackCloseable b = new TrackCloseable(); + AutoCloseables.RollbackCloseable rb = AutoCloseables.rollbackable(a, b); + rb.close(); + assertTrue(a.isClosed()); + assertTrue(b.isClosed()); + } + + @Test + public void testRollbackCloseableDoesNotCloseWhenCommitted() throws Exception { + TrackCloseable a = new TrackCloseable(); + TrackCloseable b = new TrackCloseable(); + AutoCloseables.RollbackCloseable rb = AutoCloseables.rollbackable(a, b); + rb.commit(); + rb.close(); + assertFalse(a.isClosed()); + assertFalse(b.isClosed()); + } + + @Test + public void testRollbackCloseableAddAndAddAll() throws Exception { + TrackCloseable a = new TrackCloseable(); + TrackCloseable b = new TrackCloseable(); + TrackCloseable c = new TrackCloseable(); + TrackCloseable d = new TrackCloseable(); + AutoCloseables.RollbackCloseable rb = AutoCloseables.rollbackable(a); + rb.add(b); + rb.addAll(c, d); + rb.addAll((AutoCloseable[]) null); // null varargs shouldn't fail + rb.addAll((List) null); // null Iterable shouldn't fail + rb.close(); + assertTrue(a.isClosed()); + assertTrue(b.isClosed()); + assertTrue(c.isClosed()); + assertTrue(d.isClosed()); + } + + @Test + public void testRollbackCloseableWithNull() throws Exception { + AutoCloseables.rollbackable((AutoCloseable) null); // no exception + } + + @Test + public void testRollbackCloseableWithNulls() throws Exception { + TrackCloseable a = new TrackCloseable(); + AutoCloseables.RollbackCloseable rb = AutoCloseables.rollbackable(a, null); + rb.close(); + assertTrue(a.isClosed()); + } +} From 46211fccf642d6dcfcebc31081817da1529d6a35 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Feb 2026 14:06:40 +0100 Subject: [PATCH 150/271] MINOR: Bump com.gradle:develocity-maven-extension from 2.3.1 to 2.3.3 (#1001) Bumps com.gradle:develocity-maven-extension from 2.3.1 to 2.3.3. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.gradle:develocity-maven-extension&package-manager=maven&previous-version=2.3.1&new-version=2.3.3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .mvn/extensions.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index b136e95f43..0e25cc84f8 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -23,7 +23,7 @@ com.gradle develocity-maven-extension - 2.3.1 + 2.3.4 com.gradle From 9fd4973d12abbd9e9d14098c2acd41d97a6c4407 Mon Sep 17 00:00:00 2001 From: Aleksei Starikov Date: Tue, 17 Feb 2026 14:15:00 +0100 Subject: [PATCH 151/271] GH-470: [Vector] Fix ListViewVector.getElementEndIndex(index) method (#1019) ## What's Changed [First commit](https://github.com/apache/arrow-java/commit/a758cadb17c3d50c08a139a3e2ddced71215ccf5) changes logic: - The PR fixes a bug in the `ListViewVector.getElementEndIndex(index)` method . Before: ``` public int getElementEndIndex(int index) { return sizeBuffer.getInt(index * OFFSET_WIDTH); } ``` After: ``` public int getElementEndIndex(int index) { return offsetBuffer.getInt(index * OFFSET_WIDTH) + sizeBuffer.getInt(index * SIZE_WIDTH); } ``` [Second commit](https://github.com/apache/arrow-java/commit/aec29750202ff1aac73e7fdc9c2020b3dcf72696) doesn't change logic: - Fixes a bug of usage `sizeBuffer` with `OFFSET_WIDTH` (`hashCode` method) and `offsetBuffer` with `SIZE_WIDTH` (`setSize` method). It doesn't introduce real changes in the logic as `OFFSET_WIDTH` == `SIZE_WIDTH` == 4 - Plus small refactoring of ListViewVector to avoid code duplication and similar issues in the future. --- It's a bug fix. --- Closes #470. --- .../arrow/vector/complex/ListViewVector.java | 68 +++--- .../arrow/vector/TestListViewVector.java | 215 ++++++++++++++---- 2 files changed, 204 insertions(+), 79 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java index 8711db5e0f..d41f61e291 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java @@ -226,8 +226,8 @@ private void setReaderAndWriterIndex() { sizeBuffer.writerIndex(0); } else { validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); - offsetBuffer.writerIndex(valueCount * OFFSET_WIDTH); - sizeBuffer.writerIndex(valueCount * SIZE_WIDTH); + offsetBuffer.writerIndex((long) valueCount * OFFSET_WIDTH); + sizeBuffer.writerIndex((long) valueCount * SIZE_WIDTH); } } @@ -445,14 +445,22 @@ public int hashCode(int index, ArrowBufHasher hasher) { return ArrowBufPointer.NULL_HASH_CODE; } int hash = 0; - final int start = offsetBuffer.getInt(index * OFFSET_WIDTH); - final int end = sizeBuffer.getInt(index * OFFSET_WIDTH); + final int start = getElementStartIndex(index); + final int end = getElementEndIndex(index); for (int i = start; i < end; i++) { hash = ByteFunctionHelpers.combineHash(hash, vector.hashCode(i, hasher)); } return hash; } + private void setElementOffsetBuffer(int index, int value) { + offsetBuffer.setInt((long) index * OFFSET_WIDTH, value); + } + + private void setElementSizeBuffer(int index, int value) { + sizeBuffer.setInt((long) index * SIZE_WIDTH, value); + } + private class TransferImpl implements TransferPair { ListViewVector to; @@ -498,7 +506,6 @@ public void splitAndTransfer(int startIndex, int length) { valueCount); to.clear(); if (length > 0) { - final int startPoint = offsetBuffer.getInt((long) startIndex * OFFSET_WIDTH); // we have to scan by index since there are out-of-order offsets to.offsetBuffer = to.allocateBuffers((long) length * OFFSET_WIDTH); to.sizeBuffer = to.allocateBuffers((long) length * SIZE_WIDTH); @@ -507,9 +514,9 @@ public void splitAndTransfer(int startIndex, int length) { int maxOffsetAndSizeSum = -1; int minOffsetValue = -1; for (int i = 0; i < length; i++) { - final int offsetValue = offsetBuffer.getInt((long) (startIndex + i) * OFFSET_WIDTH); - final int sizeValue = sizeBuffer.getInt((long) (startIndex + i) * SIZE_WIDTH); - to.sizeBuffer.setInt((long) i * SIZE_WIDTH, sizeValue); + final int offsetValue = getElementStartIndex(startIndex + i); + final int sizeValue = getElementSize(startIndex + i); + to.setElementSizeBuffer(i, sizeValue); if (maxOffsetAndSizeSum < offsetValue + sizeValue) { maxOffsetAndSizeSum = offsetValue + sizeValue; } @@ -520,9 +527,9 @@ public void splitAndTransfer(int startIndex, int length) { /* splitAndTransfer the offset buffer */ for (int i = 0; i < length; i++) { - final int offsetValue = offsetBuffer.getInt((long) (startIndex + i) * OFFSET_WIDTH); + final int offsetValue = getElementStartIndex(startIndex + i); final int relativeOffset = offsetValue - minOffsetValue; - to.offsetBuffer.setInt((long) i * OFFSET_WIDTH, relativeOffset); + to.setElementOffsetBuffer(i, relativeOffset); } /* splitAndTransfer the validity buffer */ @@ -678,8 +685,8 @@ public List getObject(int index) { if (isSet(index) == 0) { return null; } - final int start = offsetBuffer.getInt(index * OFFSET_WIDTH); - final int end = start + sizeBuffer.getInt((index) * SIZE_WIDTH); + final int start = getElementStartIndex(index); + final int end = getElementEndIndex(index); final ValueVector vv = getDataVector(); final List vals = new JsonStringArrayList<>(end - start); for (int i = start; i < end; i++) { @@ -711,7 +718,7 @@ public boolean isEmpty(int index) { if (isNull(index)) { return true; } else { - return sizeBuffer.getInt(index * SIZE_WIDTH) == 0; + return getElementSize(index) == 0; } } @@ -722,10 +729,7 @@ public boolean isEmpty(int index) { * @return 1 if element at given index is not null, 0 otherwise */ public int isSet(int index) { - final int byteIndex = index >> 3; - final byte b = validityBuffer.getByte(byteIndex); - final int bitIndex = index & 7; - return (b >> bitIndex) & 0x01; + return BitVectorHelper.get(validityBuffer, index); } /** @@ -775,8 +779,8 @@ public void setNull(int index) { reallocValidityAndSizeAndOffsetBuffers(); } - offsetBuffer.setInt(index * OFFSET_WIDTH, 0); - sizeBuffer.setInt(index * SIZE_WIDTH, 0); + setElementOffsetBuffer(index, 0); + setElementSizeBuffer(index, 0); BitVectorHelper.unsetBit(validityBuffer, index); } @@ -794,11 +798,11 @@ public int startNewValue(int index) { if (index > 0) { final int prevOffset = getMaxViewEndChildVectorByIndex(index); - offsetBuffer.setInt(index * OFFSET_WIDTH, prevOffset); + setElementOffsetBuffer(index, prevOffset); } BitVectorHelper.setBit(validityBuffer, index); - return offsetBuffer.getInt(index * OFFSET_WIDTH); + return getElementStartIndex(index); } /** @@ -836,9 +840,9 @@ private void validateInvariants(int offset, int size) { * @param value value to set */ public void setOffset(int index, int value) { - validateInvariants(value, sizeBuffer.getInt(index * SIZE_WIDTH)); + validateInvariants(value, getElementSize(index)); - offsetBuffer.setInt(index * OFFSET_WIDTH, value); + setElementOffsetBuffer(index, value); } /** @@ -848,9 +852,9 @@ public void setOffset(int index, int value) { * @param value value to set */ public void setSize(int index, int value) { - validateInvariants(offsetBuffer.getInt(index * SIZE_WIDTH), value); + validateInvariants(getElementStartIndex(index), value); - sizeBuffer.setInt(index * SIZE_WIDTH, value); + setElementSizeBuffer(index, value); } /** @@ -886,12 +890,16 @@ public void setValueCount(int valueCount) { @Override public int getElementStartIndex(int index) { - return offsetBuffer.getInt(index * OFFSET_WIDTH); + return offsetBuffer.getInt((long) index * OFFSET_WIDTH); + } + + private int getElementSize(int index) { + return sizeBuffer.getInt((long) index * SIZE_WIDTH); } @Override public int getElementEndIndex(int index) { - return sizeBuffer.getInt(index * OFFSET_WIDTH); + return getElementStartIndex(index) + getElementSize(index); } @Override @@ -948,8 +956,8 @@ public double getDensity() { @Override public void validate() { for (int i = 0; i < valueCount; i++) { - final int offset = offsetBuffer.getInt(i * OFFSET_WIDTH); - final int size = sizeBuffer.getInt(i * SIZE_WIDTH); + final int offset = getElementStartIndex(i); + final int size = getElementSize(i); validateInvariants(offset, size); } } @@ -961,6 +969,6 @@ public void validate() { * @param size number of elements in the list that was written */ public void endValue(int index, int size) { - sizeBuffer.setInt(index * SIZE_WIDTH, size); + setElementSizeBuffer(index, size); } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestListViewVector.java b/vector/src/test/java/org/apache/arrow/vector/TestListViewVector.java index 2f282e1988..8ab0edb145 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestListViewVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestListViewVector.java @@ -1550,55 +1550,7 @@ public void testOverwriteWithNull() { public void testOutOfOrderOffset1() { // [[12, -7, 25], null, [0, -127, 127, 50], [], [50, 12]] try (ListViewVector listViewVector = ListViewVector.empty("listview", allocator)) { - // Allocate buffers in listViewVector by calling `allocateNew` method. - listViewVector.allocateNew(); - - // Initialize the child vector using `initializeChildrenFromFields` method. - - FieldType fieldType = new FieldType(true, new ArrowType.Int(16, true), null, null); - Field field = new Field("child-vector", fieldType, null); - listViewVector.initializeChildrenFromFields(Collections.singletonList(field)); - - // Set values in the child vector. - FieldVector fieldVector = listViewVector.getDataVector(); - fieldVector.clear(); - - SmallIntVector childVector = (SmallIntVector) fieldVector; - - childVector.allocateNew(7); - - childVector.set(0, 0); - childVector.set(1, -127); - childVector.set(2, 127); - childVector.set(3, 50); - childVector.set(4, 12); - childVector.set(5, -7); - childVector.set(6, 25); - - childVector.setValueCount(7); - - // Set validity, offset and size buffers using `setValidity`, - // `setOffset` and `setSize` methods. - listViewVector.setValidity(0, 1); - listViewVector.setValidity(1, 0); - listViewVector.setValidity(2, 1); - listViewVector.setValidity(3, 1); - listViewVector.setValidity(4, 1); - - listViewVector.setOffset(0, 4); - listViewVector.setOffset(1, 7); - listViewVector.setOffset(2, 0); - listViewVector.setOffset(3, 0); - listViewVector.setOffset(4, 3); - - listViewVector.setSize(0, 3); - listViewVector.setSize(1, 0); - listViewVector.setSize(2, 4); - listViewVector.setSize(3, 0); - listViewVector.setSize(4, 2); - - // Set value count using `setValueCount` method. - listViewVector.setValueCount(5); + initializeListViewVectorAsInSpecification(listViewVector); final ArrowBuf offSetBuffer = listViewVector.getOffsetBuffer(); final ArrowBuf sizeBuffer = listViewVector.getSizeBuffer(); @@ -2217,6 +2169,105 @@ public void testRangeChildVector2() { } } + @Test + public void testGetElementStartIndexAndEndIndexOrderedOffsetsNoIntersection() { + /* + values = [10, 20, 30, 40, 50] + offsets = [0, 3] + sizes = [3, 2] + vector: [[10, 20, 30], [40, 50]] + */ + try (ListViewVector listViewVector = ListViewVector.empty("sourceVector", allocator)) { + initializeListViewVector( + listViewVector, List.of(10, 20, 30, 40, 50), List.of(1, 1), List.of(0, 3), List.of(3, 2)); + + assertEquals(0, listViewVector.getElementStartIndex(0)); + assertEquals(3, listViewVector.getElementEndIndex(0)); + assertEquals(3, listViewVector.getElementStartIndex(1)); + assertEquals(5, listViewVector.getElementEndIndex(1)); + + final FieldVector dataVec = listViewVector.getDataVector(); + int elemIndex = 0; + int start = listViewVector.getElementStartIndex(elemIndex); + int end = listViewVector.getElementEndIndex(elemIndex); + List list = listViewVector.getObject(elemIndex); + assertEquals(end - start, list.size()); + for (int j = 0; j < list.size(); j++) { + assertEquals(((SmallIntVector) dataVec).get(start + j), list.get(j)); + } + } + } + + @Test + public void testGetElementStartIndexAndEndIndexNotOrderedOffsetsNoIntersection() { + /* + values = [1, 2, 3, 4, 5, 6] + validity = [1, 1, 1] + offsets = [4, 2, 0] + sizes = [2, 2, 2] + vector: [[5, 6], [3, 4], [1, 2]] + */ + try (ListViewVector listViewVector = ListViewVector.empty("sourceVector", allocator)) { + initializeListViewVector( + listViewVector, + List.of(1, 2, 3, 4, 5, 6), + List.of(1, 1, 1), + List.of(4, 2, 0), + List.of(2, 2, 2)); + + assertEquals(4, listViewVector.getElementStartIndex(0)); + assertEquals(6, listViewVector.getElementEndIndex(0)); + assertEquals(2, listViewVector.getElementStartIndex(1)); + assertEquals(4, listViewVector.getElementEndIndex(1)); + assertEquals(0, listViewVector.getElementStartIndex(2)); + assertEquals(2, listViewVector.getElementEndIndex(2)); + } + } + + @Test + public void testGetElementStartIndexAndEndIndexOrderedOffsetsWithIntersection() { + /* + values = [1, 2, 3, 4, 5] + validity = [1, 1, 1] + offsets = [0, 1, 4] + sizes = [2, 3, 1] + vector: [[1, 2], [2, 3, 4], [5]] + */ + try (ListViewVector listViewVector = ListViewVector.empty("sourceVector", allocator)) { + initializeListViewVector( + listViewVector, + List.of(1, 2, 3, 4, 5), + List.of(1, 1, 1), + List.of(0, 1, 4), + List.of(2, 3, 1)); + + assertEquals(0, listViewVector.getElementStartIndex(0)); + assertEquals(2, listViewVector.getElementEndIndex(0)); + assertEquals(1, listViewVector.getElementStartIndex(1)); + assertEquals(4, listViewVector.getElementEndIndex(1)); + assertEquals(4, listViewVector.getElementStartIndex(2)); + assertEquals(5, listViewVector.getElementEndIndex(2)); + } + } + + @Test + public void testGetElementStartIndexAndEndIndexOrderedOffsetsAsInSpecification() { + try (ListViewVector listViewVector = ListViewVector.empty("sourceVector", allocator)) { + initializeListViewVectorAsInSpecification(listViewVector); + + assertEquals(4, listViewVector.getElementStartIndex(0)); + assertEquals(7, listViewVector.getElementEndIndex(0)); + assertEquals(7, listViewVector.getElementStartIndex(1)); + assertEquals(7, listViewVector.getElementEndIndex(1)); + assertEquals(0, listViewVector.getElementStartIndex(2)); + assertEquals(4, listViewVector.getElementEndIndex(2)); + assertEquals(0, listViewVector.getElementStartIndex(3)); + assertEquals(0, listViewVector.getElementEndIndex(3)); + assertEquals(3, listViewVector.getElementStartIndex(4)); + assertEquals(5, listViewVector.getElementEndIndex(4)); + } + } + private void writeIntValues(UnionListViewWriter writer, int[] values) { writer.startListView(); for (int v : values) { @@ -2224,4 +2275,70 @@ private void writeIntValues(UnionListViewWriter writer, int[] values) { } writer.endListView(); } + + /** + * ListViewVector from the specification. + */ + private void initializeListViewVectorAsInSpecification(ListViewVector listViewVector) { + /* + values = [0, -127, 127, 50, 12, -7, 25] + validity = [1, 1, 1, 0, 1] (reversed) + offsets = [4, 7, 0, 0, 3] + sizes = [3, 0, 4, 0, 2] + vector: [[12, -7, 25], null, [0, -127, 127, 50], [], [50, 12]] + */ + initializeListViewVector( + listViewVector, + List.of(0, -127, 127, 50, 12, -7, 25), + List.of(1, 1, 1, 0, 1), + List.of(4, 7, 0, 0, 3), + List.of(3, 0, 4, 0, 2)); + } + + private void initializeListViewVector( + ListViewVector listViewVector, + List values, + List validity, + List offsets, + List sizes) { + // Allocate buffers in listViewVector by calling `allocateNew` method. + assert offsets.size() == sizes.size(); + listViewVector.allocateNew(); + + // Initialize the child vector using `initializeChildrenFromFields` method. + FieldType fieldType = new FieldType(true, new ArrowType.Int(16, true), null, null); + Field field = new Field("child-vector", fieldType, null); + listViewVector.initializeChildrenFromFields(Collections.singletonList(field)); + + // Set values in the child vector. + FieldVector fieldVector = listViewVector.getDataVector(); + fieldVector.clear(); + + SmallIntVector childVector = (SmallIntVector) fieldVector; + childVector.allocateNew(values.size()); + for (int i = 0; i < values.size(); i++) { + childVector.set(i, values.get(i)); + } + childVector.setValueCount(values.size()); + + // Set validity, offset and size buffers using `setValidity`, + // `setOffset` and `setSize` methods. + List reversedValidity = new ArrayList<>(validity); + Collections.reverse(reversedValidity); + for (int i = 0; i < reversedValidity.size(); i++) { + listViewVector.setValidity(i, reversedValidity.get(i)); + } + + for (int i = 0; i < offsets.size(); i++) { + listViewVector.setOffset(i, offsets.get(i)); + } + + for (int i = 0; i < sizes.size(); i++) { + listViewVector.setSize(i, sizes.get(i)); + } + + // Set value count using `setValueCount` method. + listViewVector.setValueCount(offsets.size()); + } } From de97138845abbd12ad253170b5738c4ec3d45473 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Feb 2026 21:29:08 +0100 Subject: [PATCH 152/271] MINOR: Bump logback.version from 1.5.26 to 1.5.27 (#999) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps `logback.version` from 1.5.26 to 1.5.27. Updates `ch.qos.logback:logback-classic` from 1.5.26 to 1.5.27
Release notes

Sourced from ch.qos.logback:logback-classic's releases.

Logback 1.5.27

2026-01-30 Release of logback version 1.5.27

• Updated license to Eclipse Public License version 2.0 from version 1.0, retaining the GPL 2.1 dual-license.

• Fixed missing MDC data transmitted by SocketAppender reported in issues/1010 by Lars Vogel.

• Removed all Receiver classes and components which were already disabled for several years.

• Refactored file scanning code for improved clarity.

• In SizeAndTimeBasedRollingPolicy modified totalSizeCap and maxFileSize comparison to taking into account file compression. This fixes issues/1007.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit 3618eb01aad6672f9cd250dccf7546a69cbe982f associated with the tag v_1.5.27. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits
  • 3618eb0 increase timeout delay to 2000 millis
  • db150c3 prepare release 1.5.27
  • 0370b13 fix missing MDC transmission in SocketAppender. Fixes issues/1010
  • 8100acd remove RemoteAppender*
  • 2b67210 remove Receiver related classes
  • d84b586 remove ReceiverModelHandler - project still builds indicating no active usage
  • 44049ed remove support for receivers in SerializedModelConfigurator and JoranConfigur...
  • 56085d8 fix test
  • e7764f4 refactor file change scanning for clarity
  • e56a12f bump assertj version
  • Additional commits viewable in compare view

Updates `ch.qos.logback:logback-core` from 1.5.26 to 1.5.27
Release notes

Sourced from ch.qos.logback:logback-core's releases.

Logback 1.5.27

2026-01-30 Release of logback version 1.5.27

• Updated license to Eclipse Public License version 2.0 from version 1.0, retaining the GPL 2.1 dual-license.

• Fixed missing MDC data transmitted by SocketAppender reported in issues/1010 by Lars Vogel.

• Removed all Receiver classes and components which were already disabled for several years.

• Refactored file scanning code for improved clarity.

• In SizeAndTimeBasedRollingPolicy modified totalSizeCap and maxFileSize comparison to taking into account file compression. This fixes issues/1007.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit 3618eb01aad6672f9cd250dccf7546a69cbe982f associated with the tag v_1.5.27. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits
  • 3618eb0 increase timeout delay to 2000 millis
  • db150c3 prepare release 1.5.27
  • 0370b13 fix missing MDC transmission in SocketAppender. Fixes issues/1010
  • 8100acd remove RemoteAppender*
  • 2b67210 remove Receiver related classes
  • d84b586 remove ReceiverModelHandler - project still builds indicating no active usage
  • 44049ed remove support for receivers in SerializedModelConfigurator and JoranConfigur...
  • 56085d8 fix test
  • e7764f4 refactor file change scanning for clarity
  • e56a12f bump assertj version
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1f57039015..c118f6aaa4 100644 --- a/pom.xml +++ b/pom.xml @@ -113,7 +113,7 @@ under the License. true 2.42.0 3.53.0 - 1.5.26 + 1.5.32 none -Xdoclint:none From b209fa2ee8ca9840530b404e1448f90d5ae0889d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Feb 2026 21:29:33 +0100 Subject: [PATCH 153/271] MINOR: [CI] Bump docker/login-action from 3.6.0 to 3.7.0 (#996) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [docker/login-action](https://github.com/docker/login-action) from 3.6.0 to 3.7.0.
Release notes

Sourced from docker/login-action's releases.

v3.7.0

Full Changelog: https://github.com/docker/login-action/compare/v3.6.0...v3.7.0

Commits
  • c94ce9f Merge pull request #915 from docker/dependabot/npm_and_yarn/lodash-4.17.23
  • 8339c95 Merge pull request #912 from docker/scope
  • c83e932 build(deps): bump lodash from 4.17.21 to 4.17.23
  • b268aa5 chore: update generated content
  • a603229 documentation for scope input
  • 7567f92 Add scope input to set scopes for the authentication token
  • 0567fa5 Merge pull request #914 from dphi/add-support-for-amazonaws.eu
  • f6ef577 feat: add support for AWS European Sovereign Cloud ECR registries
  • 916386b Merge pull request #911 from crazy-max/ensure-redact
  • 5b3f94a chore: update generated content
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=docker/login-action&package-manager=github_actions&previous-version=3.6.0&new-version=3.7.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 37b2209966..41ea193809 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -127,7 +127,7 @@ jobs: with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing - - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ghcr.io username: ${{ github.actor }} From 5adfb7e32922de6bc7d929a248911904aab80abd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Feb 2026 07:11:51 +0100 Subject: [PATCH 154/271] MINOR: Bump commons-codec:commons-codec from 1.20.0 to 1.21.0 (#997) Bumps [commons-codec:commons-codec](https://github.com/apache/commons-codec) from 1.20.0 to 1.21.0.
Changelog

Sourced from commons-codec:commons-codec's changelog.

Apache Commons Codec 1.21.0 Release Notes

The Apache Commons Codec team is pleased to announce the release of Apache Commons Codec 1.21.0.

The Apache Commons Codec component contains encoders and decoders for formats such as Base16, Base32, Base64, digest, and Hexadecimal. In addition to these widely used encoders and decoders, the codec package also maintains a collection of phonetic encoding utilities.

This is a feature and maintenance release. Java 8 or later is required.

New features

  • CODEC-333: Add distinct Base64 decoding for standard and URL-safe formats. Thanks to Aleksandr Beliakov, Gary Gregory.

Fixed Bugs

  •  Fix oak leaf icon references in overview.html when running
`mvn clean javadoc:javadoc`. Thanks to Gary Gregory.

  •  Fix Apache RAT plugin console warnings. Thanks to Gary
Gregory.

  •  Fix malformed Javadoc comments. Thanks to Gary Gregory.

Changes

  •  Bump org.apache.commons:commons-parent from 91 to 96
[#415](https://github.com/apache/commons-codec/issues/415),
[#418](https://github.com/apache/commons-codec/issues/418). Thanks to
Gary Gregory, Dependabot.

  •  Bump commons-io:commons-io from 2.20.0 to 2.21.0. Thanks to
Gary Gregory.

  •  Bump org.apache.commons:commons-lang3 from 3.19.0 to 3.20.0.
Thanks to Gary Gregory, Dependabot.

For complete information on Apache Commons Codec, including instructions on how to submit bug reports, patches, or suggestions for improvement, see the Apache Commons Codec website:

https://commons.apache.org/proper/commons-codec/

Download page: https://commons.apache.org/proper/commons-codec/download_codec.cgi

Commits
  • 91c4404 Prepare for the release candidate 1.21.0 RC1
  • 21fe1d7 Prepare for the next release candidate
  • d4ea4d0 Bump actions/checkout from 6.0.1 to 6.0.2
  • e30b1f6 Bump actions/setup-java from 5.1.0 to 5.2.0
  • 2e4891c Bump org.apache.commons:commons-parent from 95 to 96
  • d02c003 Use a URL to a prettier page: https://www.ietf.org/rfc/rfc2045
  • 3c961b8 Checkstyle
  • 99cf6b7 Javadoc and exception messages: "base 32" -> "Base32".
  • 2df7b9a Javadoc and exception messages: "base 64" -> "Base64".
  • 0643fdd Javadoc 8 doesn't know how to find this link
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=commons-codec:commons-codec&package-manager=maven&previous-version=1.20.0&new-version=1.21.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- vector/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vector/pom.xml b/vector/pom.xml index 89e9779008..b24f37d5f9 100644 --- a/vector/pom.xml +++ b/vector/pom.xml @@ -60,7 +60,7 @@ under the License. commons-codec commons-codec - 1.20.0 + 1.21.0 org.apache.arrow From 6b6d16a42fdcb46dae728ef0795559e1606c2372 Mon Sep 17 00:00:00 2001 From: Aleksei Starikov Date: Sun, 22 Feb 2026 14:41:02 +0100 Subject: [PATCH 155/271] GH-139: [Flight] Stop return null from MetadataAdapter.getAll(String) and getAllByte(String) (#1016) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What's Changed `CallHeaders` has 3 implementations: - FlightCallHeaders - ErrorFlightMetadata - MetadataAdapter **Before this change:** `MetadataAdapter` could return `null` from `getAll(String)` and `getAllByte(String)` when there were no values for the key, because gRPC’s `Metadata.getAll()` returns `null` in that case. This was undocumented and forced callers to null-check. **After this change:** All 3 implementations consistently return an `empty iterable` (never `null`) when the key is absent or has no values. The contract is documented on the interface and covered by tests for each implementation. --- **This contains breaking changes.** `MetadataAdapter.getAll(String)` and `getAllByte(String)` return empty iterator instead of null. --- Closes #139. --- .../org/apache/arrow/flight/CallHeaders.java | 14 ++++++-- .../arrow/flight/ServerSessionMiddleware.java | 26 +++++++------- .../flight/client/ClientCookieMiddleware.java | 5 +-- .../arrow/flight/grpc/MetadataAdapter.java | 9 +++-- .../apache/arrow/flight/TestCallOptions.java | 11 ++++++ .../arrow/flight/TestErrorMetadata.java | 11 ++++++ .../flight/grpc/TestMetadataAdapter.java | 36 +++++++++++++++++++ 7 files changed, 90 insertions(+), 22 deletions(-) create mode 100644 flight/flight-core/src/test/java/org/apache/arrow/flight/grpc/TestMetadataAdapter.java diff --git a/flight/flight-core/src/main/java/org/apache/arrow/flight/CallHeaders.java b/flight/flight-core/src/main/java/org/apache/arrow/flight/CallHeaders.java index f4f6486a3c..0939d232cf 100644 --- a/flight/flight-core/src/main/java/org/apache/arrow/flight/CallHeaders.java +++ b/flight/flight-core/src/main/java/org/apache/arrow/flight/CallHeaders.java @@ -26,10 +26,20 @@ public interface CallHeaders { /** Get the value of a metadata key. If multiple values are present, then get the last one. */ byte[] getByte(String key); - /** Get all values present for the given metadata key. */ + /** + * Get all values present for the given metadata key. + * + * @param key the metadata key + * @return an iterable of all values for the key. Returns an empty iterable if no value to return. + */ Iterable getAll(String key); - /** Get all values present for the given metadata key. */ + /** + * Get all values present for the given metadata key. + * + * @param key the metadata key + * @return an iterable of all values for the key. Returns an empty iterable if no value to return. + */ Iterable getAllByte(String key); /** diff --git a/flight/flight-core/src/main/java/org/apache/arrow/flight/ServerSessionMiddleware.java b/flight/flight-core/src/main/java/org/apache/arrow/flight/ServerSessionMiddleware.java index 47fd6f1366..5ec01b9c83 100644 --- a/flight/flight-core/src/main/java/org/apache/arrow/flight/ServerSessionMiddleware.java +++ b/flight/flight-core/src/main/java/org/apache/arrow/flight/ServerSessionMiddleware.java @@ -80,20 +80,18 @@ public ServerSessionMiddleware onCallStarted( String sessionId = null; final Iterable it = incomingHeaders.getAll("cookie"); - if (it != null) { - findIdCookie: - for (final String headerValue : it) { - for (final String cookie : headerValue.split(" ;")) { - final String[] cookiePair = cookie.split("="); - if (cookiePair.length != 2) { - // Soft failure: Ignore invalid cookie list field - break; - } - - if (sessionCookieName.equals(cookiePair[0]) && cookiePair[1].length() > 0) { - sessionId = cookiePair[1]; - break findIdCookie; - } + findIdCookie: + for (final String headerValue : it) { + for (final String cookie : headerValue.split(" ;")) { + final String[] cookiePair = cookie.split("="); + if (cookiePair.length != 2) { + // Soft failure: Ignore invalid cookie list field + break; + } + + if (sessionCookieName.equals(cookiePair[0]) && cookiePair[1].length() > 0) { + sessionId = cookiePair[1]; + break findIdCookie; } } } diff --git a/flight/flight-core/src/main/java/org/apache/arrow/flight/client/ClientCookieMiddleware.java b/flight/flight-core/src/main/java/org/apache/arrow/flight/client/ClientCookieMiddleware.java index e5eb934001..b33e6b7ecc 100644 --- a/flight/flight-core/src/main/java/org/apache/arrow/flight/client/ClientCookieMiddleware.java +++ b/flight/flight-core/src/main/java/org/apache/arrow/flight/client/ClientCookieMiddleware.java @@ -100,10 +100,7 @@ public void onBeforeSendingHeaders(CallHeaders outgoingHeaders) { @Override public void onHeadersReceived(CallHeaders incomingHeaders) { - final Iterable setCookieHeaders = incomingHeaders.getAll(SET_COOKIE_HEADER); - if (setCookieHeaders != null) { - factory.updateCookies(setCookieHeaders); - } + factory.updateCookies(incomingHeaders.getAll(SET_COOKIE_HEADER)); } @Override diff --git a/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/MetadataAdapter.java b/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/MetadataAdapter.java index a1de16ede6..64a0769d63 100644 --- a/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/MetadataAdapter.java +++ b/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/MetadataAdapter.java @@ -18,6 +18,7 @@ import io.grpc.Metadata; import java.nio.charset.StandardCharsets; +import java.util.Collections; import java.util.HashSet; import java.util.Set; import java.util.stream.Collectors; @@ -53,13 +54,17 @@ public byte[] getByte(String key) { @Override public Iterable getAll(String key) { - return this.metadata.getAll(Metadata.Key.of(key, Metadata.ASCII_STRING_MARSHALLER)); + final Iterable all = + this.metadata.getAll(Metadata.Key.of(key, Metadata.ASCII_STRING_MARSHALLER)); + return all != null ? all : Collections.emptyList(); } @Override public Iterable getAllByte(String key) { if (key.endsWith(Metadata.BINARY_HEADER_SUFFIX)) { - return this.metadata.getAll(Metadata.Key.of(key, Metadata.BINARY_BYTE_MARSHALLER)); + final Iterable all = + this.metadata.getAll(Metadata.Key.of(key, Metadata.BINARY_BYTE_MARSHALLER)); + return all != null ? all : Collections.emptyList(); } return StreamSupport.stream(getAll(key).spliterator(), false) .map(String::getBytes) diff --git a/flight/flight-core/src/test/java/org/apache/arrow/flight/TestCallOptions.java b/flight/flight-core/src/test/java/org/apache/arrow/flight/TestCallOptions.java index a54ce69812..8aef9c69a1 100644 --- a/flight/flight-core/src/test/java/org/apache/arrow/flight/TestCallOptions.java +++ b/flight/flight-core/src/test/java/org/apache/arrow/flight/TestCallOptions.java @@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -110,6 +111,16 @@ public void mixedProperties() { testHeaders(headers); } + @Test + public void getAllReturnsEmptyIterableForMissingKey() { + FlightCallHeaders headers = new FlightCallHeaders(); + + assertNotNull(headers.getAll("missing")); + assertFalse(headers.getAll("missing").iterator().hasNext()); + assertNotNull(headers.getAllByte("missing-bin")); + assertFalse(headers.getAllByte("missing-bin").iterator().hasNext()); + } + private void testHeaders(CallHeaders headers) { try (BufferAllocator a = new RootAllocator(Long.MAX_VALUE); HeaderProducer producer = new HeaderProducer(); diff --git a/flight/flight-core/src/test/java/org/apache/arrow/flight/TestErrorMetadata.java b/flight/flight-core/src/test/java/org/apache/arrow/flight/TestErrorMetadata.java index a9a3e355bc..214614defd 100644 --- a/flight/flight-core/src/test/java/org/apache/arrow/flight/TestErrorMetadata.java +++ b/flight/flight-core/src/test/java/org/apache/arrow/flight/TestErrorMetadata.java @@ -20,6 +20,7 @@ import static org.apache.arrow.flight.Location.forGrpcInsecure; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -119,6 +120,16 @@ public void testFlightMetadata() throws Exception { } } + @Test + public void getAllReturnsEmptyIterableForMissingKey() { + ErrorFlightMetadata metadata = new ErrorFlightMetadata(); + + assertNotNull(metadata.getAll("missing")); + assertFalse(metadata.getAll("missing").iterator().hasNext()); + assertNotNull(metadata.getAllByte("missing-bin")); + assertFalse(metadata.getAllByte("missing-bin").iterator().hasNext()); + } + private static class StatusRuntimeExceptionProducer extends NoOpFlightProducer { private final PerfOuterClass.Perf perf; diff --git a/flight/flight-core/src/test/java/org/apache/arrow/flight/grpc/TestMetadataAdapter.java b/flight/flight-core/src/test/java/org/apache/arrow/flight/grpc/TestMetadataAdapter.java new file mode 100644 index 0000000000..b0f5dcfcfc --- /dev/null +++ b/flight/flight-core/src/test/java/org/apache/arrow/flight/grpc/TestMetadataAdapter.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.flight.grpc; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import io.grpc.Metadata; +import org.junit.jupiter.api.Test; + +public class TestMetadataAdapter { + + @Test + public void getAllReturnsEmptyIterableForMissingKey() { + MetadataAdapter headers = new MetadataAdapter(new Metadata()); + + assertNotNull(headers.getAll("missing")); + assertFalse(headers.getAll("missing").iterator().hasNext()); + assertNotNull(headers.getAllByte("missing-bin")); + assertFalse(headers.getAllByte("missing-bin").iterator().hasNext()); + } +} From 6e73f7f563b28eec4218d8afabdfaa0ef41e714a Mon Sep 17 00:00:00 2001 From: Ashish Date: Sun, 22 Feb 2026 09:29:47 -0800 Subject: [PATCH 156/271] MINOR: Fix minor issue with README (#1026) ## What's Changed Please fill in a description of the changes here. The PR fixes minor documentation issue, where commands needed to be adjusted to new repo. These were found while setting up the environment. AI was **NOT** used to generate the PR Closes #NNN. --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b0715aadf1..0196536514 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ a version of your choosing. $ flatc --version flatc version 25.1.24 -$ grep "dep.fbs.version" java/pom.xml +$ grep "dep.fbs.version" pom.xml 25.1.24 ``` @@ -60,10 +60,10 @@ $ grep "dep.fbs.version" java/pom.xml cd $ARROW_HOME # remove the existing files -rm -rf java/format/src +rm -rf format/src # regenerate from the .fbs files -flatc --java -o java/format/src/main/java format/*.fbs +flatc --java -o format/src/main/java arrow-format/*.fbs # prepend license header mvn spotless:apply -pl :arrow-format From 0e54b379ff871084139679f1bc501faf05996e18 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Feb 2026 09:45:46 +0100 Subject: [PATCH 157/271] MINOR: Bump commons-cli:commons-cli from 1.9.0 to 1.11.0 (#1028) Bumps [commons-cli:commons-cli](https://github.com/apache/commons-cli) from 1.9.0 to 1.11.0.
Changelog

Sourced from commons-cli:commons-cli's changelog.

Apache Commons CLI 1.11.0 Release Notes

The Apache Commons CLI team is pleased to announce the release of Apache Commons CLI 1.11.0.

Apache Commons CLI provides a simple API for presenting, processing, and validating a Command Line Interface.

This is a feature and maintenance release. Java 8 or later is required.

New Features

  •  Add CommandLine.getOptionCount() to measure option
repetition [#396](https://github.com/apache/commons-cli/issues/396).
Thanks to David Larochette, Gary Gregory.

Fixed Bugs

  • CLI-351: Multiple trailing BREAK_CHAR_SET characters cause infinite loop in HelpFormatter. Thanks to Damien Carbonne, Claude Warren, Gary Gregory.
  • CLI-351: Fix issue with groups not being reported in help output. #411. Thanks to Damien Carbonne, Claude Warren, Gary Gregory.

Updates

  •  Bump org.apache.commons:commons-parent from 85 to 91
[#393](https://github.com/apache/commons-cli/issues/393). Thanks to Gary
Gregory, Dependabot.

  •  Bump commons-io:commons-io from 2.20.0 to 2.21.0. Thanks to
Gary Gregory.

Historical list of changes: https://commons.apache.org/proper/commons-cli/changes.html

For complete information on Apache Commons CLI, including instructions on how to submit bug reports, patches, or suggestions for improvement, see the Apache Commons CLI website:

https://commons.apache.org/proper/commons-cli/

Download page: https://commons.apache.org/proper/commons-cli/download_cli.cgi

Have fun! The Apache Commons Team

Apache Commons CLI 1.11.0 Release Notes

The Apache Commons CLI team is pleased to announce the release of Apache Commons CLI 1.11.0.

Apache Commons CLI provides a simple API for presenting, processing, and validating a Command Line Interface.

This is a feature and maintenance release. Java 8 or later is required.

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=commons-cli:commons-cli&package-manager=maven&previous-version=1.9.0&new-version=1.11.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-integration-tests/pom.xml | 2 +- flight/flight-sql/pom.xml | 2 +- tools/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flight/flight-integration-tests/pom.xml b/flight/flight-integration-tests/pom.xml index 78a2d08ee1..f0f10ada43 100644 --- a/flight/flight-integration-tests/pom.xml +++ b/flight/flight-integration-tests/pom.xml @@ -58,7 +58,7 @@ under the License. commons-cli commons-cli - 1.9.0 + 1.11.0 org.slf4j diff --git a/flight/flight-sql/pom.xml b/flight/flight-sql/pom.xml index 56c47f64dd..a5954819c3 100644 --- a/flight/flight-sql/pom.xml +++ b/flight/flight-sql/pom.xml @@ -119,7 +119,7 @@ under the License. commons-cli commons-cli - 1.9.0 + 1.11.0 true diff --git a/tools/pom.xml b/tools/pom.xml index cb9a161308..d43adb1fdf 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -54,7 +54,7 @@ under the License. commons-cli commons-cli - 1.9.0 + 1.11.0 ch.qos.logback From 2e76de1d2542811843b7dc262cdae0e20102b034 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Feb 2026 09:46:15 +0100 Subject: [PATCH 158/271] MINOR: Bump org.codehaus.mojo:versions-maven-plugin from 2.20.0 to 2.21.0 (#1029) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.codehaus.mojo:versions-maven-plugin](https://github.com/mojohaus/versions) from 2.20.0 to 2.21.0.
Release notes

Sourced from org.codehaus.mojo:versions-maven-plugin's releases.

2.21.0

🚀 New features and improvements

🐛 Bug Fixes

  • #1331: Fix NPE in restrictionForUnchangedSegment if actual version is null (#1332) @​andrzejj0
  • #1310: Corrected UseDepVersionMojo + handling a similar case in SetMojo, SetScmTagMojo, UpdateChildModulesMojo (#1322) @​andrzejj0
  • UseDepVersionMoto should process all projects on the project list (#1320) @​andrzejj0
  • Fixed #1317: Regression coming from ArtifactVersions::filter when currentVersion is null and ignoredVersions is not null (#1319) @​andrzejj0

📝 Documentation updates

📦 Dependency updates

2.20.1

🐛 Bug Fixes

Commits
  • 1cdedea [maven-release-plugin] prepare release 2.21.0
  • b947957 Fix README typos in Contributing section
  • b85c0a8 Bump project version to 2.21.0-SNAPSHOT
  • 7ae3767 Bump byteBuddyVersion from 1.18.3 to 1.18.4 (#1335)
  • 38afa9f Bump org.apache.maven.plugin-testing:maven-plugin-testing-harness
  • 39af6a2 Bump org.codehaus.plexus:plexus-archiver from 4.10.4 to 4.11.0
  • f51b9d5 #1331: Fix NPE in restrictionForUnchangedSegment if actual version is null (#...
  • 8d209b3 Bump org.codehaus.mojo:mojo-parent from 94 to 95 (#1330)
  • 4929d48 Bump byteBuddyVersion from 1.18.2 to 1.18.3 (#1329)
  • cb84d01 Add versions.skip parameter to skip plugin execution (#1328)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.codehaus.mojo:versions-maven-plugin&package-manager=maven&previous-version=2.20.0&new-version=2.21.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- bom/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index b631f9366d..0de43a1217 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -213,7 +213,7 @@ under the License. org.codehaus.mojo versions-maven-plugin - 2.20.0 + 2.21.0 diff --git a/pom.xml b/pom.xml index c118f6aaa4..7bc88675aa 100644 --- a/pom.xml +++ b/pom.xml @@ -512,7 +512,7 @@ under the License. org.codehaus.mojo versions-maven-plugin - 2.20.0 + 2.21.0 pl.project13.maven From 39b0593ac53c1444d0eb05f698fc3c0aa300f30b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Feb 2026 09:47:30 +0100 Subject: [PATCH 159/271] MINOR: Bump com.google.api.grpc:proto-google-common-protos from 2.63.2 to 2.66.0 (#1034) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [com.google.api.grpc:proto-google-common-protos](https://github.com/googleapis/sdk-platform-java) from 2.63.2 to 2.66.0.
Release notes

Sourced from com.google.api.grpc:proto-google-common-protos's releases.

v2.66.0

2.66.0 (2026-01-23)

Features

Dependencies

v2.65.1

2.65.1 (2026-01-13)

Documentation

  • Update docs for GoogleCredentialsProvider#setScopesToApply (#4057) (0a9962f)

v2.65.0

2.65.0 (2026-01-12)

Features

Bug Fixes

  • add api_version breadcrumb to client docs (#4018) (a2b2179)
  • Create a single S2AChannelCredentials per application (#3989) (3758b43)
  • provide API to share the same background executor for channel po… (#4030) (178182c)

Dependencies

Documentation

... (truncated)

Changelog

Sourced from com.google.api.grpc:proto-google-common-protos's changelog.

2.66.0 (2026-01-23)

Features

Dependencies

2.65.1 (2026-01-13)

Documentation

  • Update docs for GoogleCredentialsProvider#setScopesToApply (#4057) (0a9962f)

2.65.0 (2026-01-12)

Features

Bug Fixes

  • add api_version breadcrumb to client docs (#4018) (a2b2179)
  • Create a single S2AChannelCredentials per application (#3989) (3758b43)
  • provide API to share the same background executor for channel po… (#4030) (178182c)

Dependencies

Documentation

2.64.2 (2025-12-10)

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.api.grpc:proto-google-common-protos&package-manager=maven&previous-version=2.63.2&new-version=2.66.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml index b1f755844e..f1d58a0cad 100644 --- a/flight/flight-core/pom.xml +++ b/flight/flight-core/pom.xml @@ -134,7 +134,7 @@ under the License. com.google.api.grpc proto-google-common-protos - 2.63.2 + 2.66.0 test From 394755bc612f9457446396e4374f2871bbc99d9d Mon Sep 17 00:00:00 2001 From: Issac Garcia Date: Thu, 26 Feb 2026 13:20:01 +0100 Subject: [PATCH 160/271] GH-1007: fix: does not break class loading if direct buffer allocator is not available (#1008) ## What's Changed The Direct Buffer is not always needed to use Arrow memory, however, we cannot load MemoryUtil class if we don't set: ``` --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED ``` Which is not always needed/possible. This fix proposes to catch the `InaccessibleObjectException` to not avoiding the load of the class. The directBuffer is, in any case not available and a `UnsupportedOperationException` will be throw as it is in the existing code Closes #1007 . --- .../apache/arrow/memory/util/MemoryUtil.java | 23 ++++++++++++++++--- .../org/apache/arrow/memory/TestOpens.java | 17 +++++--------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java b/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java index 91bd7cd905..be0749a215 100644 --- a/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java +++ b/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java @@ -18,6 +18,7 @@ import java.lang.reflect.Constructor; import java.lang.reflect.Field; +import java.lang.reflect.InaccessibleObjectException; import java.lang.reflect.InvocationTargetException; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -81,9 +82,18 @@ public Object run() { BYTE_ARRAY_BASE_OFFSET = UNSAFE.arrayBaseOffset(byte[].class); // get the offset of the address field in a java.nio.Buffer object + long maybeOffset; Field addressField = java.nio.Buffer.class.getDeclaredField("address"); - addressField.setAccessible(true); - BYTE_BUFFER_ADDRESS_OFFSET = UNSAFE.objectFieldOffset(addressField); + try { + addressField.setAccessible(true); + maybeOffset = UNSAFE.objectFieldOffset(addressField); + } catch (InaccessibleObjectException e) { + maybeOffset = -1; + logger.debug( + "Cannot access the address field of java.nio.Buffer. DirectBuffer operations wont be available", + e); + } + BYTE_BUFFER_ADDRESS_OFFSET = maybeOffset; Constructor directBufferConstructor; long address = -1; @@ -109,6 +119,9 @@ public Object run() { } catch (SecurityException e) { logger.debug("Cannot get constructor for direct buffer allocation", e); return e; + } catch (InaccessibleObjectException e) { + logger.debug("Cannot get constructor for direct buffer allocation", e); + return e; } } }); @@ -156,7 +169,11 @@ public Object run() { * @return address of the underlying memory. */ public static long getByteBufferAddress(ByteBuffer buf) { - return UNSAFE.getLong(buf, BYTE_BUFFER_ADDRESS_OFFSET); + if (BYTE_BUFFER_ADDRESS_OFFSET != -1) { + return UNSAFE.getLong(buf, BYTE_BUFFER_ADDRESS_OFFSET); + } + throw new UnsupportedOperationException( + "Byte buffer address cannot be obtained because sun.misc.Unsafe or java.nio.DirectByteBuffer.(long, int) is not available"); } private MemoryUtil() {} diff --git a/memory/memory-core/src/test/java/org/apache/arrow/memory/TestOpens.java b/memory/memory-core/src/test/java/org/apache/arrow/memory/TestOpens.java index b5e0a71e7e..f74bf63f82 100644 --- a/memory/memory-core/src/test/java/org/apache/arrow/memory/TestOpens.java +++ b/memory/memory-core/src/test/java/org/apache/arrow/memory/TestOpens.java @@ -20,32 +20,27 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.condition.JRE.JAVA_16; +import org.apache.arrow.memory.util.MemoryUtil; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledForJreRange; public class TestOpens { - /** Instantiating the RootAllocator should poke MemoryUtil and fail. */ + /** Accessing MemoryUtil.directBuffer should fail as add-opens is not configured. */ @Test @EnabledForJreRange(min = JAVA_16) public void testMemoryUtilFailsLoudly() { // This test is configured by Maven to run WITHOUT add-opens. So this should fail on JDK16+ // (where JEP396 means that add-opens is required to access JDK internals). // The test will likely fail in your IDE if it doesn't correctly pick this up. - Throwable e = - assertThrows( - Throwable.class, - () -> { - BufferAllocator allocator = new RootAllocator(); - allocator.close(); - }); + Throwable e = assertThrows(Throwable.class, () -> MemoryUtil.directBuffer(0, 10)); boolean found = false; while (e != null) { - e = e.getCause(); - if (e instanceof RuntimeException - && e.getMessage().contains("Failed to initialize MemoryUtil")) { + if (e instanceof UnsupportedOperationException + && e.getMessage().contains("java.nio.DirectByteBuffer.(long, int) not available")) { found = true; break; } + e = e.getCause(); } assertTrue(found, "Expected exception was not thrown"); } From 1a99518180b6a89fe17a9af5ce12f6956b5c6c27 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Mar 2026 13:15:36 +0100 Subject: [PATCH 161/271] MINOR: [CI] Bump actions/upload-artifact from 6.0.0 to 7.0.0 (#1045) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 6.0.0 to 7.0.0.
Release notes

Sourced from actions/upload-artifact's releases.

v7.0.0

v7 What's new

Direct Uploads

Adds support for uploading single files directly (unzipped). Callers can set the new archive parameter to false to skip zipping the file during upload. Right now, we only support single files. The action will fail if the glob passed resolves to multiple files. The name parameter is also ignored with this setting. Instead, the name of the artifact will be the name of the uploaded file.

ESM

To support new versions of the @actions/* packages, we've upgraded the package to ESM.

What's Changed

New Contributors

Full Changelog: https://github.com/actions/upload-artifact/compare/v6...v7.0.0

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/upload-artifact&package-manager=github_actions&previous-version=6.0.0&new-version=7.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 41ea193809..4ec89c2204 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -71,7 +71,7 @@ jobs: run: | dev/release/run_rat.sh "${TAR_GZ}" - name: Upload source archive - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: release-source path: | @@ -148,7 +148,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf jni-linux-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: jni-linux-${{ matrix.platform.arch }} path: jni-linux-${{ matrix.platform.arch }}.tar.gz @@ -278,7 +278,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf jni-macos-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: jni-macos-${{ matrix.platform.arch }} path: jni-macos-${{ matrix.platform.arch }}.tar.gz @@ -356,7 +356,7 @@ jobs: shell: bash run: tar -cvzf jni-windows-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: jni-windows-${{ matrix.platform.arch }} path: jni-windows-${{ matrix.platform.arch }}.tar.gz @@ -428,12 +428,12 @@ jobs: cp -a target/site/apidocs reference tar -cvzf reference.tar.gz reference - name: Upload binaries - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: release-binaries path: binaries/* - name: Upload docs - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: reference path: reference.tar.gz @@ -471,7 +471,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf html.tar.gz -C docs/build html - name: Upload artifacts - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: release-html path: html.tar.gz From 9c497848f9e7a531aae1095009257217d993c24b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Mar 2026 13:19:43 +0100 Subject: [PATCH 162/271] MINOR: Bump checker.framework.version from 3.53.0 to 3.53.1 (#1046) Bumps `checker.framework.version` from 3.53.0 to 3.53.1. Updates `org.checkerframework:checker-qual` from 3.53.0 to 3.53.1
Release notes

Sourced from org.checkerframework:checker-qual's releases.

Checker Framework 3.53.1

Version 3.53.1 (2026-02-02)

Closed issues

#4858, #6141, #6620, #7360, #7388.

Changelog

Sourced from org.checkerframework:checker-qual's changelog.

Version 3.53.1 (2026-02-02)

Closed issues

#4858, #6141, #6620, #7360, #7388.

Commits

Updates `org.checkerframework:checker` from 3.53.0 to 3.53.1
Release notes

Sourced from org.checkerframework:checker's releases.

Checker Framework 3.53.1

Version 3.53.1 (2026-02-02)

Closed issues

#4858, #6141, #6620, #7360, #7388.

Changelog

Sourced from org.checkerframework:checker's changelog.

Version 3.53.1 (2026-02-02)

Closed issues

#4858, #6141, #6620, #7360, #7388.

Commits

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7bc88675aa..2917d6cb9b 100644 --- a/pom.xml +++ b/pom.xml @@ -112,7 +112,7 @@ under the License. 10.23.0 true 2.42.0 - 3.53.0 + 3.53.1 1.5.32 none -Xdoclint:none From 41acbdc681c4792b01655066bf7230f4a07c2ef2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Mar 2026 13:26:12 +0100 Subject: [PATCH 163/271] MINOR: [CI] Bump actions/download-artifact from 7.0.0 to 8.0.0 (#1047) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 7.0.0 to 8.0.0.
Release notes

Sourced from actions/download-artifact's releases.

v8.0.0

v8 - What's new

Direct downloads

To support direct uploads in actions/upload-artifact, the action will no longer attempt to unzip all downloaded files. Instead, the action checks the Content-Type header ahead of unzipping and skips non-zipped files. Callers wishing to download a zipped file as-is can also set the new skip-decompress parameter to false.

Enforced checks (breaking)

A previous release introduced digest checks on the download. If a download hash didn't match the expected hash from the server, the action would log a warning. Callers can now configure the behavior on mismatch with the digest-mismatch parameter. To be secure by default, we are now defaulting the behavior to error which will fail the workflow run.

ESM

To support new versions of the @actions/* packages, we've upgraded the package to ESM.

What's Changed

Full Changelog: https://github.com/actions/download-artifact/compare/v7...v8.0.0

Commits
  • 70fc10c Merge pull request #461 from actions/danwkennedy/digest-mismatch-behavior
  • f258da9 Add change docs
  • ccc058e Fix linting issues
  • bd7976b Add a setting to specify what to do on hash mismatch and default it to error
  • ac21fcf Merge pull request #460 from actions/danwkennedy/download-no-unzip
  • 15999bf Add note about package bumps
  • 974686e Bump the version to v8 and add release notes
  • fbe48b1 Update test names to make it clearer what they do
  • 96bf374 One more test fix
  • b8c4819 Fix skip decompress test
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/download-artifact&package-manager=github_actions&previous-version=7.0.0&new-version=8.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 4ec89c2204..8039f4c598 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -101,7 +101,7 @@ jobs: packages: write steps: - name: Download source archive - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: release-source - name: Extract source archive @@ -168,7 +168,7 @@ jobs: MACOSX_DEPLOYMENT_TARGET: "14.0" steps: - name: Download source archive - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: release-source - name: Extract source archive @@ -296,7 +296,7 @@ jobs: arch: "x86_64" steps: - name: Download source archive - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: release-source - name: Extract source archive @@ -369,7 +369,7 @@ jobs: - jni-windows steps: - name: Download artifacts - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: path: artifacts - name: Decompress artifacts @@ -450,11 +450,11 @@ jobs: with: cache: 'pip' - name: Download source archive - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: release-source - name: Download Javadocs - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: reference - name: Extract source archive @@ -519,7 +519,7 @@ jobs: cp ../.asf.yaml ./ git add .nojekyll .asf.yaml - name: Download - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: release-html - name: Extract @@ -555,7 +555,7 @@ jobs: - ubuntu-latest steps: - name: Download release artifacts - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: pattern: release-* - name: Verify @@ -589,7 +589,7 @@ jobs: contents: write steps: - name: Download release artifacts - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: pattern: release-* path: artifacts From 5dfd2595080a15d3d6ff3e40d8de57af4bdd7858 Mon Sep 17 00:00:00 2001 From: Logan Riggs Date: Wed, 4 Mar 2026 12:03:18 -0800 Subject: [PATCH 164/271] GH-1038: Trim object memory for ArrowBuf (#1044) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What's Changed A significant number of ArrowBuf and BufferLedger objects are created during certain workloads. Saving several bytes per instance could add up to significant memory savings and reduced memory allocation expense and garbage collection. The id field, which was a sequential value used when logging object information, is replaced with an identity hash code. This should still allow enough information for debugging without the memory overhead. There may be possible duplicate values but it shouldn't matter for logging purposes. Atomic fields can be replaced by a primitive and a static updater which saves several bytes per instance. ### ArrowBuf | Component | Before | After | Savings | |-----------|--------|-------|---------| | `idGenerator` (static) | `AtomicLong` | Removed | 24 bytes globally | | `id` field (per instance) | `long` (8 bytes) | Removed | **8 bytes per instance** | | `getId()` | Returns `id` field | Returns `System.identityHashCode(this)` | — | ### BufferLedger | Component | Before | After | Savings | |-----------|--------|-------|---------| | `LEDGER_ID_GENERATOR` (static) | `AtomicLong` | Removed | 24 bytes globally | | `ledgerId` (per instance) | `long` (8 bytes) | Removed | **8 bytes per instance** | | `bufRefCnt` | `AtomicInteger` (24 bytes) | `volatile int` + static updater | **20 bytes per instance** | ### Total Savings | Scale | ArrowBuf | BufferLedger | Combined | |-------|----------|--------------|----------| | 100K | 800 KB | 2.8 MB | **~3.6 MB** | | 1M | 8 MB | 28 MB | **~36 MB** | | 10M | 80 MB | 280 MB | **~360 MB** | ### Benchmarking I ran the added benchmark before and after the metadata trimming. **Metadata Trimmed** | Benchmark | Mode | Score | Error |Units| |-------|----------|--------------|----------|----------| |MemoryFootprintBenchmarks.measureAllocationPerformance | avgt | 456.831 |± 36.059 | us/op| |MemoryFootprintBenchmarks.measureArrowBufMemoryFootprint | ss | 161.085 |± 35.596| ms/op| |Created 100000 ArrowBuf instances. Heap memory used | sum | 35631520 bytes (33.98 MB) |0 |bytes| |Average memory per ArrowBuf| sum | 356.32 bytes |0 |bytes| **Previous Object Layout** | Benchmark | Mode | Score | Error |Units| |-------|----------|--------------|----------|----------| |MemoryFootprintBenchmarks.measureAllocationPerformance | avgt | 466.171 |± 16.233 | us/op| |MemoryFootprintBenchmarks.measureArrowBufMemoryFootprint | ss | 176.790 |± 17.943 |ms/op| |Created 100000 ArrowBuf instances. Heap memory used | sum | 38817480 bytes (37.02 MB) |0 |bytes| |Average memory per ArrowBuf| sum | 388.17 bytes |0 |bytes| Closes #1038. --- .../org/apache/arrow/memory/Accountant.java | 42 ++-- .../org/apache/arrow/memory/ArrowBuf.java | 19 +- .../org/apache/arrow/memory/BufferLedger.java | 28 +-- .../memory/MemoryFootprintBenchmarks.java | 213 ++++++++++++++++++ 4 files changed, 263 insertions(+), 39 deletions(-) create mode 100644 performance/src/main/java/org/apache/arrow/memory/MemoryFootprintBenchmarks.java diff --git a/memory/memory-core/src/main/java/org/apache/arrow/memory/Accountant.java b/memory/memory-core/src/main/java/org/apache/arrow/memory/Accountant.java index 5d052c2cde..d4d76f57f4 100644 --- a/memory/memory-core/src/main/java/org/apache/arrow/memory/Accountant.java +++ b/memory/memory-core/src/main/java/org/apache/arrow/memory/Accountant.java @@ -16,7 +16,7 @@ */ package org.apache.arrow.memory; -import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicLongFieldUpdater; import org.apache.arrow.util.Preconditions; import org.checkerframework.checker.nullness.qual.Nullable; @@ -37,16 +37,24 @@ class Accountant implements AutoCloseable { */ protected final long reservation; - private final AtomicLong peakAllocation = new AtomicLong(); + // AtomicLongFieldUpdaters for memory accounting fields to reduce memory overhead + private static final AtomicLongFieldUpdater PEAK_ALLOCATION_UPDATER = + AtomicLongFieldUpdater.newUpdater(Accountant.class, "peakAllocation"); + private static final AtomicLongFieldUpdater ALLOCATION_LIMIT_UPDATER = + AtomicLongFieldUpdater.newUpdater(Accountant.class, "allocationLimit"); + private static final AtomicLongFieldUpdater LOCALLY_HELD_MEMORY_UPDATER = + AtomicLongFieldUpdater.newUpdater(Accountant.class, "locallyHeldMemory"); + + private volatile long peakAllocation = 0; /** * Maximum local memory that can be held. This can be externally updated. Changing it won't cause * past memory to change but will change responses to future allocation efforts */ - private final AtomicLong allocationLimit = new AtomicLong(); + private volatile long allocationLimit = 0; /** Currently allocated amount of memory. */ - private final AtomicLong locallyHeldMemory = new AtomicLong(); + private volatile long locallyHeldMemory = 0; public Accountant( @Nullable Accountant parent, String name, long reservation, long maxAllocation) { @@ -64,7 +72,7 @@ public Accountant( this.parent = parent; this.name = name; this.reservation = reservation; - this.allocationLimit.set(maxAllocation); + ALLOCATION_LIMIT_UPDATER.set(this, maxAllocation); if (reservation != 0) { Preconditions.checkArgument(parent != null, "parent must not be null"); @@ -117,12 +125,12 @@ private AllocationOutcome.Status allocateBytesInternal(long size) { } private void updatePeak() { - final long currentMemory = locallyHeldMemory.get(); + final long currentMemory = locallyHeldMemory; while (true) { - final long previousPeak = peakAllocation.get(); + final long previousPeak = peakAllocation; if (currentMemory > previousPeak) { - if (!peakAllocation.compareAndSet(previousPeak, currentMemory)) { + if (!PEAK_ALLOCATION_UPDATER.compareAndSet(this, previousPeak, currentMemory)) { // peak allocation changed underneath us. try again. continue; } @@ -166,7 +174,7 @@ private AllocationOutcome.Status allocate( final boolean incomingUpdatePeak, final boolean forceAllocation, @Nullable AllocationOutcomeDetails details) { - final long oldLocal = locallyHeldMemory.getAndAdd(size); + final long oldLocal = LOCALLY_HELD_MEMORY_UPDATER.getAndAdd(this, size); final long newLocal = oldLocal + size; // Borrowed from Math.addExact (but avoid exception here) // Overflow if result has opposite sign of both arguments @@ -174,7 +182,7 @@ private AllocationOutcome.Status allocate( // failure final boolean overflow = ((oldLocal ^ newLocal) & (size ^ newLocal)) < 0; final long beyondReservation = newLocal - reservation; - final boolean beyondLimit = overflow || newLocal > allocationLimit.get(); + final boolean beyondLimit = overflow || newLocal > allocationLimit; final boolean updatePeak = forceAllocation || (incomingUpdatePeak && !beyondLimit); if (details != null) { @@ -214,7 +222,7 @@ private AllocationOutcome.Status allocate( public void releaseBytes(long size) { // reduce local memory. all memory released above reservation should be released up the tree. - final long newSize = locallyHeldMemory.addAndGet(-size); + final long newSize = LOCALLY_HELD_MEMORY_UPDATER.addAndGet(this, -size); Preconditions.checkArgument(newSize >= 0, "Accounted size went negative."); @@ -255,7 +263,7 @@ public String getName() { * @return Limit in bytes. */ public long getLimit() { - return allocationLimit.get(); + return allocationLimit; } /** @@ -274,7 +282,7 @@ public long getInitReservation() { * @param newLimit The limit in bytes. */ public void setLimit(long newLimit) { - allocationLimit.set(newLimit); + ALLOCATION_LIMIT_UPDATER.set(this, newLimit); } /** @@ -284,7 +292,7 @@ public void setLimit(long newLimit) { * @return Currently allocate memory in bytes. */ public long getAllocatedMemory() { - return locallyHeldMemory.get(); + return locallyHeldMemory; } /** @@ -293,17 +301,17 @@ public long getAllocatedMemory() { * @return The peak allocated memory in bytes. */ public long getPeakMemoryAllocation() { - return peakAllocation.get(); + return peakAllocation; } public long getHeadroom() { - long localHeadroom = allocationLimit.get() - locallyHeldMemory.get(); + long localHeadroom = allocationLimit - locallyHeldMemory; if (parent == null) { return localHeadroom; } // Amount of reserved memory left on top of what parent has - long reservedHeadroom = Math.max(0, reservation - locallyHeldMemory.get()); + long reservedHeadroom = Math.max(0, reservation - locallyHeldMemory); return Math.min(localHeadroom, parent.getHeadroom() + reservedHeadroom); } } diff --git a/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java b/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java index b8012fe643..9712be34d7 100644 --- a/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java +++ b/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java @@ -24,7 +24,6 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.ReadOnlyBufferException; -import java.util.concurrent.atomic.AtomicLong; import org.apache.arrow.memory.BaseAllocator.Verbosity; import org.apache.arrow.memory.util.CommonUtil; import org.apache.arrow.memory.util.HistoricalLog; @@ -57,9 +56,8 @@ public final class ArrowBuf implements AutoCloseable { private static final int DOUBLE_SIZE = Double.BYTES; private static final int LONG_SIZE = Long.BYTES; - private static final AtomicLong idGenerator = new AtomicLong(0); private static final int LOG_BYTES_PER_ROW = 10; - private final long id = idGenerator.incrementAndGet(); + private final ReferenceManager referenceManager; private final @Nullable BufferManager bufferManager; private final long addr; @@ -67,7 +65,8 @@ public final class ArrowBuf implements AutoCloseable { private long writerIndex; private final @Nullable HistoricalLog historicalLog = BaseAllocator.DEBUG - ? new HistoricalLog(BaseAllocator.DEBUG_LOG_LENGTH, "ArrowBuf[%d]", id) + ? new HistoricalLog( + BaseAllocator.DEBUG_LOG_LENGTH, "ArrowBuf[%d]", System.identityHashCode(this)) : null; private volatile long capacity; @@ -218,7 +217,8 @@ public long memoryAddress() { @Override public String toString() { - return String.format("ArrowBuf[%d], address:%d, capacity:%d", id, memoryAddress(), capacity); + return String.format( + "ArrowBuf[%d], address:%d, capacity:%d", getId(), memoryAddress(), capacity); } @Override @@ -1080,12 +1080,15 @@ public String toHexString(final long start, final int length) { } /** - * Get the integer id assigned to this ArrowBuf for debugging purposes. + * Get the id assigned to this ArrowBuf for debugging purposes. + * + *

Returns {@link System#identityHashCode(Object)} which provides a unique identifier for this + * buffer without any per-instance memory overhead. * - * @return integer id + * @return the identity hash code for this buffer */ public long getId() { - return id; + return System.identityHashCode(this); } /** diff --git a/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferLedger.java b/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferLedger.java index b562a421e7..eb90efcbb5 100644 --- a/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferLedger.java +++ b/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferLedger.java @@ -17,8 +17,7 @@ package org.apache.arrow.memory; import java.util.IdentityHashMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import org.apache.arrow.memory.util.CommonUtil; import org.apache.arrow.memory.util.HistoricalLog; import org.apache.arrow.util.Preconditions; @@ -32,12 +31,13 @@ public class BufferLedger implements ValueWithKeyIncluded, ReferenceManager { private final @Nullable IdentityHashMap buffers = BaseAllocator.DEBUG ? new IdentityHashMap<>() : null; - private static final AtomicLong LEDGER_ID_GENERATOR = new AtomicLong(0); - // unique ID assigned to each ledger - private final long ledgerId = LEDGER_ID_GENERATOR.incrementAndGet(); - private final AtomicInteger bufRefCnt = new AtomicInteger(0); // start at zero so we can - // manage request for retain - // correctly + + // AtomicIntegerFieldUpdater for bufRefCnt to reduce memory overhead + private static final AtomicIntegerFieldUpdater BUF_REF_CNT_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(BufferLedger.class, "bufRefCnt"); + // start at zero so we can manage request for retain correctly + private volatile int bufRefCnt = 0; + private final long lCreationTime = System.nanoTime(); private final BufferAllocator allocator; private final AllocationManager allocationManager; @@ -78,7 +78,7 @@ public BufferAllocator getAllocator() { */ @Override public int getRefCount() { - return bufRefCnt.get(); + return bufRefCnt; } /** @@ -86,7 +86,7 @@ public int getRefCount() { * ArrowBufs managed by this ledger will share the ref count. */ void increment() { - bufRefCnt.incrementAndGet(); + BUF_REF_CNT_UPDATER.incrementAndGet(this); } /** @@ -144,7 +144,7 @@ private int decrement(int decrement) { allocator.assertOpen(); final int outcome; synchronized (allocationManager) { - outcome = bufRefCnt.addAndGet(-decrement); + outcome = BUF_REF_CNT_UPDATER.addAndGet(this, -decrement); if (outcome == 0) { lDestructionTime = System.nanoTime(); // refcount of this reference manager has dropped to 0 @@ -174,7 +174,7 @@ public void retain(int increment) { if (historicalLog != null) { historicalLog.recordEvent("retain(%d)", increment); } - final int originalReferenceCount = bufRefCnt.getAndAdd(increment); + final int originalReferenceCount = BUF_REF_CNT_UPDATER.getAndAdd(this, increment); Preconditions.checkArgument(originalReferenceCount > 0); } @@ -472,13 +472,13 @@ public long getAccountedSize() { void print(StringBuilder sb, int indent, BaseAllocator.Verbosity verbosity) { CommonUtil.indent(sb, indent) .append("ledger[") - .append(ledgerId) + .append(System.identityHashCode(this)) .append("] allocator: ") .append(allocator.getName()) .append("), isOwning: ") .append(", size: ") .append(", references: ") - .append(bufRefCnt.get()) + .append(bufRefCnt) .append(", life: ") .append(lCreationTime) .append("..") diff --git a/performance/src/main/java/org/apache/arrow/memory/MemoryFootprintBenchmarks.java b/performance/src/main/java/org/apache/arrow/memory/MemoryFootprintBenchmarks.java new file mode 100644 index 0000000000..395ba13b9d --- /dev/null +++ b/performance/src/main/java/org/apache/arrow/memory/MemoryFootprintBenchmarks.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.memory; + +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryUsage; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +/** + * Benchmarks for memory footprint of Arrow memory objects. + * + *

This benchmark measures the heap memory overhead of creating many ArrowBuf instances. The + * optimizations using AtomicFieldUpdater instead of AtomicLong/AtomicInteger objects should reduce + * memory overhead significantly. + * + *

Expected savings per instance: - ArrowBuf: 8 bytes (id field removed) - BufferLedger: 28 bytes + * (20 from AtomicInteger + 8 from ledgerId) - Accountant: 48 bytes (3 × 16 bytes from AtomicLong + * objects) + * + *

For 1M ArrowBuf instances, this should save approximately 8 MB of heap memory. + */ +@State(Scope.Benchmark) +@Fork( + value = 1, + jvmArgs = {"-Xms2g", "-Xmx2g"}) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +public class MemoryFootprintBenchmarks { + + /** Number of ArrowBuf instances to create for memory footprint measurement. */ + private static final int NUM_BUFFERS = 100_000; + + /** Size in bytes of each buffer allocation. */ + private static final int BUFFER_SIZE = 1024; + + /** Root allocator used for all buffer allocations in the benchmark. */ + private RootAllocator allocator; + + /** Array to hold references to allocated buffers, preventing garbage collection. */ + private ArrowBuf[] buffers; + + /** JMX bean for querying heap memory usage statistics. */ + private MemoryMXBean memoryBean; + + /** + * Sets up the benchmark state before each trial. + * + *

Initializes the memory monitoring bean, creates a root allocator with sufficient capacity, + * and allocates the buffer reference array. + */ + @Setup(Level.Trial) + public void setup() { + memoryBean = ManagementFactory.getMemoryMXBean(); + allocator = new RootAllocator((long) NUM_BUFFERS * BUFFER_SIZE); + buffers = new ArrowBuf[NUM_BUFFERS]; + } + + /** + * Cleans up buffers after each benchmark invocation. + * + *

Closes all allocated buffers to prevent memory leaks and ensure each iteration starts with a + * clean slate. This is critical for the memory footprint benchmark which allocates many buffers + * that would otherwise accumulate across warmup and measurement iterations. + */ + @TearDown(Level.Invocation) + public void tearDown() { + for (int i = 0; i < NUM_BUFFERS; i++) { + if (buffers[i] != null) { + buffers[i].close(); + buffers[i] = null; + } + } + } + + /** + * Cleans up the allocator after the trial completes. + * + *

Closes the root allocator to release all resources after all warmup and measurement + * iterations are complete. + */ + @TearDown(Level.Trial) + public void tearDownTrial() { + allocator.close(); + } + + /** + * Benchmark that measures heap memory usage when creating many ArrowBuf instances. + * + *

This benchmark creates {@value #NUM_BUFFERS} ArrowBuf instances and measures the heap memory + * used. With the AtomicFieldUpdater optimizations, we expect to save approximately 800 KB of heap + * memory (8 bytes × 100,000 instances) just from removing the id field in ArrowBuf. + * + *

The benchmark performs garbage collection before and after allocation to ensure accurate + * measurement of heap memory delta. Results are printed to stdout for analysis. + * + * @return the total heap memory used by the allocated buffers in bytes + */ + @Benchmark + @BenchmarkMode(Mode.SingleShotTime) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public long measureArrowBufMemoryFootprint() { + // Force GC before measurement + System.gc(); + System.gc(); + System.gc(); + + MemoryUsage heapBefore = memoryBean.getHeapMemoryUsage(); + long usedBefore = heapBefore.getUsed(); + + // Allocate buffers + for (int i = 0; i < NUM_BUFFERS; i++) { + buffers[i] = allocator.buffer(BUFFER_SIZE); + } + + // Force GC to get accurate measurement + System.gc(); + System.gc(); + System.gc(); + + MemoryUsage heapAfter = memoryBean.getHeapMemoryUsage(); + long usedAfter = heapAfter.getUsed(); + + long memoryUsed = usedAfter - usedBefore; + + // Print memory usage for analysis + System.out.printf( + "Created %d ArrowBuf instances. Heap memory used: %d bytes (%.2f MB)%n", + NUM_BUFFERS, memoryUsed, memoryUsed / (1024.0 * 1024.0)); + System.out.printf( + "Average memory per ArrowBuf: %.2f bytes%n", (double) memoryUsed / NUM_BUFFERS); + + return memoryUsed; + } + + /** + * Benchmark that measures allocation and deallocation performance. + * + *

This complements the memory footprint benchmark by measuring the time it takes to allocate + * and deallocate 1,000 buffers in a tight loop. This helps identify any performance regressions + * introduced by memory optimizations. + * + *

Uses a local buffer array to avoid interference with the shared {@link #buffers} array used + * by other benchmarks. + */ + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + public void measureAllocationPerformance() { + ArrowBuf[] localBuffers = new ArrowBuf[1000]; + + for (int i = 0; i < 1000; i++) { + localBuffers[i] = allocator.buffer(BUFFER_SIZE); + } + + for (int i = 0; i < 1000; i++) { + localBuffers[i].close(); + } + } + + /** + * Main entry point for running the benchmarks standalone. + * + *

This allows running the benchmarks directly from the command line or IDE without using the + * Maven JMH plugin. Example usage: + * + * 

{@code
+   * java -cp target/benchmarks.jar org.apache.arrow.memory.MemoryFootprintBenchmarks
+   * }
+ * + * @param args command line arguments (not used) + * @throws RunnerException if the benchmark runner encounters an error + */ + public static void main(String[] args) throws RunnerException { + Options opt = + new OptionsBuilder() + .include(MemoryFootprintBenchmarks.class.getSimpleName()) + .forks(1) + .build(); + + new Runner(opt).run(); + } +} From 7cbf15994ae7c82fbcb178dded6b3f128219d9ad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Mar 2026 21:18:49 +0100 Subject: [PATCH 165/271] MINOR: Bump org.codehaus.mojo:build-helper-maven-plugin from 3.6.0 to 3.6.1 (#1049) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.codehaus.mojo:build-helper-maven-plugin](https://github.com/mojohaus/build-helper-maven-plugin) from 3.6.0 to 3.6.1.
Release notes

Sourced from org.codehaus.mojo:build-helper-maven-plugin's releases.

3.6.1

📝 Documentation updates

👻 Maintenance

📦 Dependency updates

Commits
  • 908df59 [maven-release-plugin] prepare release 3.6.1
  • faafd8f Use common release-drafter configuration
  • a91b402 Rename Goals to Plugin Documentation in the site menu
  • 1e9136d Bump org.codehaus.mojo:mojo-parent from 87 to 91
  • 8700ddc Bump org.apache.maven.shared:file-management from 3.1.0 to 3.2.0
  • ab2c635 Bump org.codehaus.mojo:mojo-parent from 86 to 87
  • 611ce40 Typos.
  • 02d2b8e Bump org.codehaus.mojo:mojo-parent from 85 to 86
  • d742e5c Update site.xml to Doxia 2
  • 80b89b8 Bump org.codehaus.plexus:plexus-utils from 4.0.1 to 4.0.2
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.codehaus.mojo:build-helper-maven-plugin&package-manager=maven&previous-version=3.6.0&new-version=3.6.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2917d6cb9b..6b7003f31a 100644 --- a/pom.xml +++ b/pom.xml @@ -497,7 +497,7 @@ under the License. org.codehaus.mojo build-helper-maven-plugin - 3.6.0 + 3.6.1 org.codehaus.mojo From 15f50796b5603100702560fad4b4c843f4fa379c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Mon, 9 Mar 2026 15:12:41 +0100 Subject: [PATCH 166/271] MINOR: Fix flaky TestBasicAuth memory leak by waiting for async buffer release (#1058) ## What's Changed gRPC/Netty releases Arrow buffers asynchronously after server shutdown. Poll briefly for the allocator's memory to drain before closing it, preventing spurious "Memory was leaked" errors in CI. The fix adds a brief polling loop to wait for the allocator's memory to drain before closing it. --- .../java/org/apache/arrow/flight/auth/TestBasicAuth.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/flight/flight-core/src/test/java/org/apache/arrow/flight/auth/TestBasicAuth.java b/flight/flight-core/src/test/java/org/apache/arrow/flight/auth/TestBasicAuth.java index 0c63785c88..0f202ba2d9 100644 --- a/flight/flight-core/src/test/java/org/apache/arrow/flight/auth/TestBasicAuth.java +++ b/flight/flight-core/src/test/java/org/apache/arrow/flight/auth/TestBasicAuth.java @@ -178,6 +178,12 @@ public static void shutdown() throws Exception { AutoCloseables.close(server); allocator.getChildAllocators().forEach(BufferAllocator::close); + + // gRPC/Netty may still be releasing Arrow buffers asynchronously after server shutdown. + // Poll briefly to allow in-flight buffer releases to complete before closing the allocator. + for (int i = 0; i < 20 && allocator.getAllocatedMemory() > 0; i++) { + Thread.sleep(100); + } AutoCloseables.close(allocator); } } From 2f39438afd4a2c8bf7ba63b7e3aa726c680036e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 16:32:14 +0100 Subject: [PATCH 167/271] MINOR: Bump org.apache.orc:orc-core from 2.2.2 to 2.3.0 (#1056) Bumps org.apache.orc:orc-core from 2.2.2 to 2.3.0. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.orc:orc-core&package-manager=maven&previous-version=2.2.2&new-version=2.3.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- adapter/orc/pom.xml | 2 +- dataset/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/adapter/orc/pom.xml b/adapter/orc/pom.xml index 89d45e155c..c96ab36119 100644 --- a/adapter/orc/pom.xml +++ b/adapter/orc/pom.xml @@ -61,7 +61,7 @@ under the License. org.apache.orc orc-core - 2.2.2 + 2.3.0 test diff --git a/dataset/pom.xml b/dataset/pom.xml index 686a234358..1852c6eddc 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -130,7 +130,7 @@ under the License. org.apache.orc orc-core - 2.2.2 + 2.3.0 test From 07c5f48a16230275cb502b94ffe4a3ca70f9adad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Mon, 9 Mar 2026 17:32:10 +0100 Subject: [PATCH 168/271] MINOR: [CI] Increase JNI macOS job timeout from 45 to 60 minutes (#1060) As MacOS executor as slightly slower than other executors, this PR increase the JNI MacOS job timeout to 60 minutes (instead of 45 minutes). --- .github/workflows/rc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 8039f4c598..b866ff75f2 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -155,7 +155,7 @@ jobs: jni-macos: name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }} runs-on: ${{ matrix.platform.runs_on }} - timeout-minutes: 45 + timeout-minutes: 60 needs: - source strategy: From a7313c22c17211ecb666e44e97158a495a176778 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 17:32:56 +0100 Subject: [PATCH 169/271] MINOR: [CI] Bump docker/login-action from 3.7.0 to 4.0.0 (#1053) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [docker/login-action](https://github.com/docker/login-action) from 3.7.0 to 4.0.0.
Release notes

Sourced from docker/login-action's releases.

v4.0.0

Full Changelog: https://github.com/docker/login-action/compare/v3.7.0...v4.0.0

Commits
  • b45d80f Merge pull request #929 from crazy-max/node24
  • 176cb9c node 24 as default runtime
  • cad8984 Merge pull request #920 from docker/dependabot/npm_and_yarn/aws-sdk-dependenc...
  • 92cbcb2 chore: update generated content
  • 5a2d6a7 build(deps): bump the aws-sdk-dependencies group with 2 updates
  • 44512b6 Merge pull request #928 from docker/dependabot/npm_and_yarn/docker/actions-to...
  • 28737a5 chore: update generated content
  • dac0793 build(deps): bump @​docker/actions-toolkit from 0.76.0 to 0.77.0
  • 62029f3 Merge pull request #919 from docker/dependabot/npm_and_yarn/actions/core-3.0.0
  • 08c8f06 chore: update generated content
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=docker/login-action&package-manager=github_actions&previous-version=3.7.0&new-version=4.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index b866ff75f2..a202777143 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -127,7 +127,7 @@ jobs: with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing - - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: registry: ghcr.io username: ${{ github.actor }} From a53339b6ba2321f51f7f10f16a1ce06b12384498 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 18:59:48 +0100 Subject: [PATCH 170/271] MINOR: Bump dep.hadoop.version from 3.4.2 to 3.4.3 (#1055) Bumps `dep.hadoop.version` from 3.4.2 to 3.4.3. Updates `org.apache.hadoop:hadoop-client-runtime` from 3.4.2 to 3.4.3 Updates `org.apache.hadoop:hadoop-client-api` from 3.4.2 to 3.4.3 Updates `org.apache.hadoop:hadoop-common` from 3.4.2 to 3.4.3 Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6b7003f31a..e91e8c888f 100644 --- a/pom.xml +++ b/pom.xml @@ -102,7 +102,7 @@ under the License. 1.78.0 4.33.4 2.21.0 - 3.4.2 + 3.4.3 25.2.10 1.12.1 1.17.0 From 7390f551267798d4670eae6b2894c527dbc90403 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Mar 2026 14:46:43 +0100 Subject: [PATCH 171/271] MINOR: Bump io.grpc:grpc-bom from 1.78.0 to 1.79.0 (#1048) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [io.grpc:grpc-bom](https://github.com/grpc/grpc-java) from 1.78.0 to 1.79.0.
Release notes

Sourced from io.grpc:grpc-bom's releases.

v1.79.0

API Changes

  • core: Delete the never-used io.grpc.internal.ReadableBuffer.readBytes(ByteBuffer) (#12580) (738782fb0). This is deeply internal and not accessible, so shouldn’t impact anything. However, Apache Arrow Java uses reflection to access private fields; GH-939: Remove reflection for gRPC buffers is swapping to gRPC’s public zero-copy APIs

  • opentelemetry: Add target attribute filter for metrics (#12587). Introduce an optional Predicate targetAttributeFilter to control how grpc.target is recorded in OpenTelemetry client metrics. When a filter is provided, targets rejected by the predicate are normalized to "other" to reduce grpc.target metric cardinality, while accepted targets are recorded as-is. If no filter is set, existing behavior is preserved. This change adds a new Builder API on GrpcOpenTelemetry to allow applications to configure the filter. 

Behavior Changes

  • core: Convert AutoConfiguredLB to an actual LB (4bbf8eee5). This is an internal refactoring, but it does improve how errors are handled for broken binaries. Previously, not being able to load pick_first would result in a channel panic. Now it is handled as a regular load balancing error

  • okhttp: Assert no pending streams before transport READY (#12566) (ed6d175fc). No pending streams should exist when the transport transitions to READY. This PR adds an assertion to help verify this invariant.

Bug Fixes

  • core: PickFirstLB should not return a subchannel during CONNECTING (228fc8ecd). Pick-first in grpc-java has behaved this way since it was created, and it was of no consequence. However, now there are some load balancing policies (mainly RLS) that will do a pick() and hope the result to be reasonably accurate for metrics.

Improvements

  • core: Improve DEADLINE_EXCEEDED message for CallCreds delays (ead532b39). Previously the error message contained “buffered_nanos” and “waiting_for_connection” for connection delays. However, we discovered the same strings were also used if waiting on CallCredentials. Now you’ll see details like “connecting_and_lb_delay”, “call_credentials_delay”, and “was_still_waiting”.

  • opentelemetry: Add Android API checking (a9f73f4c0). Previously we assumed OpenTelemetry support would not be used on Android. It did happen to be compatible with Android, but since OpenTelemetry does have some Android support, we now have a check that it remains compatible

  • core: Catch Errors when calling complex config parsing code (a535ed799). Error (and any other Throwable) is now caught and handled when parsing configuration (e.g., service config, xds). This will cause such failures to be handled gracefully instead of panicking the channel

  • core: Implement LoadBalancer.Helper.createOobChannel() with the internals of createResolvingOobChannel() (3915d029c). This API is only expected to be relevant to the gRPC-LB lookaside load balancer, and is not believed to have behavior changes. Out-of-band channel had been implemented with its own stripped-down Channel without load balancing. Reimplementing using the resolving oob channel makes it a full-fledged channel and reduces the burden when integrating new features and allows us to have a ManagedChannelBuilder to use with efforts like gRFC A110: Child Channel Options.

  • xds: Implement the proactive connection logic in RingHashLoadBalancer as outlined in gRFC A61 (#12596). Previously, the Java implementation only initialized child balancers when a ring-chosen endpoint was in TRANSIENT_FAILURE during a picker's pickSubchannel call. This PR adds the missing logic: when a child balancer reports TRANSIENT_FAILURE, the LoadBalancer now proactively initializes the first available IDLE child if no other children are currently connecting or ready.

This ensures a backup subchannel starts warming up immediately outside the RPC flow, reducing failover latency and improving overall resilience. This behavior was previously present but was inadvertently lost after #10610.

  • api: Add RFC 3986 support to DnsNameResolverProvider (#12602) (f65127cf7) Experimental RFC 3986 target URI parsing mode (disabled by default)

New Features

Dependencies 

  • protobuf: Upgrade Bazel protobuf to 33.1 (#12553) (b61a8f49c) and load java_proto_library from the protobuf repo (c7f3cdbc3)

  • protobuf: Fix build with Bazel 9 by upgrading bazel_jar_jar and grpc-proto versions (#12569)

  • Upgrade dependencies (#12588) (6422092e3) Netty to 4.1.130, error-prone annotations to 2.45.0, google-auth-library to 1.41.0, tomcat-embed-core9 to 9.0.113, tomcat-embed-core to 10.1.50, opentelemetry to 1.57.0, jetty-ee10-servlet to 12.1.5, jetty-http2-server to 12.1.5, google-cloud-logging to 3.23.9, google-auth to 1.41.0, proto-google-common-protos to 2.63.2.

... (truncated)

Commits
  • 381593f Bump version to 1.79.0
  • f93ecb0 Update README etc to reference 1.79.0
  • f6d140f xds: Normalize weights before combining endpoint and locality weights
  • c589bef core: clarify dns javadoc/test about trailing path segments
  • 65596ae core: Move 4 test cases from DnsNameResolverTest to DnsNameResolverProviderTe...
  • 59a64f0 core: Use FlagResetRule to set/restore system properties in DnsNameResolverTe...
  • c5f5ee0 opentelemetry: Add target attribute filter for metrics (#12587)
  • f65127c api: Add RFC 3986 support to DnsNameResolverProvider (#12602)
  • a535ed7 Catch Errors when calling complex parsing code
  • ebb9420 xds: Merge ClusterResolverLB into CdsLB2
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=io.grpc:grpc-bom&package-manager=maven&previous-version=1.78.0&new-version=1.79.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: JB Onofré --- .../org/apache/arrow/flight/grpc/GetReadableBuffer.java | 6 +++--- pom.xml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/GetReadableBuffer.java b/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/GetReadableBuffer.java index 45c32a86c6..fcba88d212 100644 --- a/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/GetReadableBuffer.java +++ b/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/GetReadableBuffer.java @@ -87,13 +87,13 @@ public static void readIntoBuffer( final InputStream stream, final ArrowBuf buf, final int size, final boolean fastPath) throws IOException { ReadableBuffer readableBuffer = fastPath ? getReadableBuffer(stream) : null; + byte[] heapBytes = new byte[size]; if (readableBuffer != null) { - readableBuffer.readBytes(buf.nioBuffer(0, size)); + readableBuffer.readBytes(heapBytes, 0, size); } else { - byte[] heapBytes = new byte[size]; ByteStreams.readFully(stream, heapBytes); - buf.writeBytes(heapBytes); } + buf.writeBytes(heapBytes); buf.writerIndex(size); } } diff --git a/pom.xml b/pom.xml index e91e8c888f..19625617b1 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,7 @@ under the License. 2.0.17 33.4.8-jre 4.2.9.Final - 1.78.0 + 1.79.0 4.33.4 2.21.0 3.4.3 From e349a9a837aa9e3c5a56cbdd841f4e9655fa9ab2 Mon Sep 17 00:00:00 2001 From: Logan Riggs Date: Wed, 11 Mar 2026 00:09:23 -0700 Subject: [PATCH 172/271] GH-1061: Add codegen classifier jar for arrow-vector. (#1062) ## What's Changed Add a new codegen classifier jar for arrow-vector that contains tdd and other template files. Closes #1061 . --- docs/source/overview.rst | 3 +++ vector/pom.xml | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/docs/source/overview.rst b/docs/source/overview.rst index be579c1495..1188054114 100644 --- a/docs/source/overview.rst +++ b/docs/source/overview.rst @@ -45,6 +45,9 @@ but some modules are JNI bindings to the C++ library. * - arrow-vector - An off-heap reference implementation for Arrow columnar data format. - Native + * - arrow-vector-codegen + - Template files for Arrow datatypes suitable for code generation. + - Native * - arrow-tools - Java applications for working with Arrow ValueVectors. - Native diff --git a/vector/pom.xml b/vector/pom.xml index b24f37d5f9..f46bd0e7b4 100644 --- a/vector/pom.xml +++ b/vector/pom.xml @@ -194,6 +194,34 @@ under the License. + + org.apache.maven.plugins + maven-jar-plugin + + + codegen-jar + + jar + + package + + + codegen + ${basedir}/src/main/codegen + + **/*.tdd + **/*.fmpp + **/*.ftl + + + + + From bdec833fb69f945db9f3c767715c93d09214f2b5 Mon Sep 17 00:00:00 2001 From: Pedro Matias Date: Wed, 11 Mar 2026 07:34:47 +0000 Subject: [PATCH 173/271] GH-994: Fix DatabaseMetaData NPEs when SqlInfo is unavailable (#995) ## What's Changed Multiple DatabaseMetaData methods had NPEs when the method `ArrowDatabaseMetadata.getSqlInfoAndCacheIfCacheIsEmpty(final SqlInfo sqlInfoCommand, final Class desiredType)` returned null. Now the method never returns null. If the database server does not provide the requested info, either a sensible default is returned or a SQLException is thrown. Closes #994. --- .../driver/jdbc/ArrowDatabaseMetadata.java | 33 ++++++++- .../jdbc/ArrowDatabaseMetadataTest.java | 72 +++++++++++++++++++ 2 files changed, 104 insertions(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java index 502270e1cd..0110525fea 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadata.java @@ -45,6 +45,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.Arrays; +import java.util.Collections; import java.util.EnumMap; import java.util.HashMap; import java.util.List; @@ -86,9 +87,12 @@ import org.apache.arrow.vector.util.Text; import org.apache.calcite.avatica.AvaticaConnection; import org.apache.calcite.avatica.AvaticaDatabaseMetaData; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** Arrow Flight JDBC's implementation of {@link DatabaseMetaData}. */ public class ArrowDatabaseMetadata extends AvaticaDatabaseMetaData { + private static final Logger LOGGER = LoggerFactory.getLogger(ArrowDatabaseMetadata.class); private static final String JAVA_REGEX_SPECIALS = "[]()|^-+*?{}$\\."; private static final Charset CHARSET = StandardCharsets.UTF_8; private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; @@ -774,7 +778,34 @@ private T getSqlInfoAndCacheIfCacheIsEmpty( } } } - return desiredType.cast(cachedSqlInfo.get(sqlInfoCommand)); + T value = desiredType.cast(cachedSqlInfo.get(sqlInfoCommand)); + if (value != null) { + return value; + } + LOGGER.debug( + "SqlInfo {} not provided by server, returning default for type {}", + sqlInfoCommand.name(), + desiredType.getSimpleName()); + + // Return sensible defaults when SqlInfo is unavailable + if (desiredType == Long.class) { + return desiredType.cast(0L); + } else if (desiredType == Integer.class) { + return desiredType.cast(0); + } else if (desiredType == Boolean.class) { + return desiredType.cast(false); + } else if (desiredType == String.class) { + return desiredType.cast(""); + } else if (desiredType == Map.class) { + return desiredType.cast(Collections.emptyMap()); + } else if (desiredType == List.class) { + return desiredType.cast(Collections.emptyList()); + } + + throw new SQLException( + String.format( + "The value of the SqlInfo %s is null and it could not be cast to %s.", + sqlInfoCommand.name(), desiredType.getName())); } private Optional convertListSqlInfoToString(final List sqlInfoList) { diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java index 81579cc387..3ab1460b27 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowDatabaseMetadataTest.java @@ -1543,11 +1543,83 @@ public void testEmptySqlInfo() throws Exception { try (final Connection testConnection = FLIGHT_SERVER_EMPTY_SQLINFO_TEST_RULE.getConnection(false)) { final DatabaseMetaData metaData = testConnection.getMetaData(); + assertThat(metaData.getSQLKeywords(), is("")); assertThat(metaData.getNumericFunctions(), is("")); assertThat(metaData.getStringFunctions(), is("")); assertThat(metaData.getSystemFunctions(), is("")); assertThat(metaData.getTimeDateFunctions(), is("")); + + assertThat(metaData.getMaxBinaryLiteralLength(), is(0)); + assertThat(metaData.getMaxCharLiteralLength(), is(0)); + assertThat(metaData.getMaxColumnNameLength(), is(0)); + assertThat(metaData.getMaxColumnsInGroupBy(), is(0)); + assertThat(metaData.getMaxColumnsInIndex(), is(0)); + assertThat(metaData.getMaxColumnsInOrderBy(), is(0)); + assertThat(metaData.getMaxColumnsInSelect(), is(0)); + assertThat(metaData.getMaxColumnsInTable(), is(0)); + assertThat(metaData.getMaxConnections(), is(0)); + assertThat(metaData.getMaxCursorNameLength(), is(0)); + assertThat(metaData.getMaxIndexLength(), is(0)); + assertThat(metaData.getMaxSchemaNameLength(), is(0)); + assertThat(metaData.getMaxProcedureNameLength(), is(0)); + assertThat(metaData.getMaxCatalogNameLength(), is(0)); + assertThat(metaData.getMaxRowSize(), is(0)); + assertThat(metaData.getMaxStatementLength(), is(0)); + assertThat(metaData.getMaxStatements(), is(0)); + assertThat(metaData.getMaxTableNameLength(), is(0)); + assertThat(metaData.getMaxTablesInSelect(), is(0)); + assertThat(metaData.getMaxUserNameLength(), is(0)); + + assertThat(metaData.supportsColumnAliasing(), is(false)); + assertThat(metaData.nullPlusNonNullIsNull(), is(false)); + assertThat(metaData.supportsTableCorrelationNames(), is(false)); + assertThat(metaData.supportsDifferentTableCorrelationNames(), is(false)); + assertThat(metaData.supportsExpressionsInOrderBy(), is(false)); + assertThat(metaData.supportsOrderByUnrelated(), is(false)); + assertThat(metaData.supportsLikeEscapeClause(), is(false)); + assertThat(metaData.supportsNonNullableColumns(), is(false)); + assertThat(metaData.supportsIntegrityEnhancementFacility(), is(false)); + assertThat(metaData.isCatalogAtStart(), is(false)); + assertThat(metaData.supportsSelectForUpdate(), is(false)); + assertThat(metaData.supportsStoredProcedures(), is(false)); + assertThat(metaData.supportsCorrelatedSubqueries(), is(false)); + assertThat(metaData.doesMaxRowSizeIncludeBlobs(), is(false)); + assertThat(metaData.supportsTransactions(), is(false)); + assertThat(metaData.dataDefinitionCausesTransactionCommit(), is(false)); + assertThat(metaData.dataDefinitionIgnoredInTransactions(), is(false)); + assertThat(metaData.supportsBatchUpdates(), is(false)); + assertThat(metaData.supportsSavepoints(), is(false)); + assertThat(metaData.supportsNamedParameters(), is(false)); + assertThat(metaData.locatorsUpdateCopy(), is(false)); + assertThat(metaData.supportsStoredFunctionsUsingCallSyntax(), is(false)); + assertThat(metaData.supportsGroupBy(), is(false)); + assertThat(metaData.supportsGroupByUnrelated(), is(false)); + assertThat(metaData.supportsMinimumSQLGrammar(), is(false)); + assertThat(metaData.supportsCoreSQLGrammar(), is(false)); + assertThat(metaData.supportsExtendedSQLGrammar(), is(false)); + assertThat(metaData.supportsANSI92EntryLevelSQL(), is(false)); + assertThat(metaData.supportsANSI92IntermediateSQL(), is(false)); + assertThat(metaData.supportsANSI92FullSQL(), is(false)); + assertThat(metaData.supportsOuterJoins(), is(false)); + assertThat(metaData.supportsFullOuterJoins(), is(false)); + assertThat(metaData.supportsLimitedOuterJoins(), is(false)); + assertThat(metaData.supportsSchemasInProcedureCalls(), is(false)); + assertThat(metaData.supportsSchemasInIndexDefinitions(), is(false)); + assertThat(metaData.supportsSchemasInPrivilegeDefinitions(), is(false)); + assertThat(metaData.supportsCatalogsInIndexDefinitions(), is(false)); + assertThat(metaData.supportsCatalogsInPrivilegeDefinitions(), is(false)); + assertThat(metaData.supportsPositionedDelete(), is(false)); + assertThat(metaData.supportsPositionedUpdate(), is(false)); + assertThat(metaData.supportsSubqueriesInComparisons(), is(false)); + assertThat(metaData.supportsSubqueriesInExists(), is(false)); + assertThat(metaData.supportsSubqueriesInIns(), is(false)); + assertThat(metaData.supportsSubqueriesInQuantifieds(), is(false)); + assertThat(metaData.supportsUnion(), is(false)); + assertThat(metaData.supportsUnionAll(), is(false)); + assertThat(metaData.supportsConvert(), is(false)); + + assertThat(metaData.getDefaultTransactionIsolation(), is(Connection.TRANSACTION_NONE)); } } } From c8666f28569ca7e825b45f8ca39434c12428bec6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Mar 2026 11:09:05 +0100 Subject: [PATCH 174/271] MINOR: Bump com.gradle:common-custom-user-data-maven-extension from 2.0.3 to 2.1.0 (#998) Bumps [com.gradle:common-custom-user-data-maven-extension](https://github.com/gradle/common-custom-user-data-maven-extension) from 2.0.3 to 2.1.0.
Release notes

Sourced from com.gradle:common-custom-user-data-maven-extension's releases.

2.1.0

  • [NEW] Add support for evaluating one or more Groovy scripts in the Develocity storage directory

2.0.7

  • [FIX] Added a null-safety check to handle cases where the Maven session may be null

2.0.6

  • [FIX] GitHub Actions build link doesn't include run attempt

2.0.5

  • [FIX] Add GitHub run attempt as custom value to precisely identify GitHub Action run

2.0.4

  • [FIX] Add GitHub run number as custom value to precisely identify GitHub Action run
Commits
  • 0bb5838 [maven-release-plugin] prepare release v2.1.0
  • 4b1a27c Update changes.md
  • b9010d0 Merge pull request #329 from gradle/erichaagdev/groovy-scripts-m2-directory
  • 82281ec Switch Groovy script evaluation order
  • 7cdb3cc Clarify script locations in README
  • 2c511ae Add support for evaluating one or more Groovy scripts in the Develocity stora...
  • f02dbca Update to use version 2.0.7 of the Common Custom User Data Maven Extension
  • 88e0f41 Prepare for next round of development
  • 36edaf8 [maven-release-plugin] prepare for next development iteration
  • 683a966 [maven-release-plugin] prepare release v2.0.7
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.gradle:common-custom-user-data-maven-extension&package-manager=maven&previous-version=2.0.3&new-version=2.1.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .mvn/extensions.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index 0e25cc84f8..4585435b49 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -28,6 +28,6 @@ com.gradle common-custom-user-data-maven-extension - 2.0.3 + 2.1.0 From 6ffb2d0450effccf115203ae31da10708a35dda8 Mon Sep 17 00:00:00 2001 From: Aleksei Starikov Date: Wed, 11 Mar 2026 11:23:45 +0100 Subject: [PATCH 175/271] GH-301: [Vector] Allow adding a vector at the end of VectorSchemaRoot (#1013) ## What's Changed Allow adding a vector at the end of VectorSchemaRoot in the `VectorSchemaRoot#addVector()` method. Previously, the precondition `index < fieldVectors.size()` rejected `index == fieldVectors.size()`, so appending was impossible. The precondition is now `index <= fieldVectors.size()`, and when `index == fieldVectors.size()` the new vector is appended after all existing vectors. The implementation of `VectorSchemaRoot#addVector()` is now aligned with [BaseTable#insertVector()](https://github.com/apache/arrow-java/blob/main/vector/src/main/java/org/apache/arrow/vector/table/BaseTable.java#L156) The change is backward compatible, as it extends the functionality of the `VectorSchemaRoot#addVector()` method. Closes #301. --- .../apache/arrow/vector/VectorSchemaRoot.java | 15 +++++++++----- .../arrow/vector/TestVectorSchemaRoot.java | 20 +++++++++++++++++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java b/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java index a7cb9ced72..4c1fbf761a 100644 --- a/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java +++ b/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java @@ -199,13 +199,18 @@ public FieldVector getVector(int index) { */ public VectorSchemaRoot addVector(int index, FieldVector vector) { Preconditions.checkNotNull(vector); - Preconditions.checkArgument(index >= 0 && index < fieldVectors.size()); + Preconditions.checkArgument(index >= 0 && index <= fieldVectors.size()); List newVectors = new ArrayList<>(); - for (int i = 0; i < fieldVectors.size(); i++) { - if (i == index) { - newVectors.add(vector); + if (index == fieldVectors.size()) { + newVectors.addAll(fieldVectors); + newVectors.add(vector); + } else { + for (int i = 0; i < fieldVectors.size(); i++) { + if (i == index) { + newVectors.add(vector); + } + newVectors.add(fieldVectors.get(i)); } - newVectors.add(fieldVectors.get(i)); } return new VectorSchemaRoot(newVectors); } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestVectorSchemaRoot.java b/vector/src/test/java/org/apache/arrow/vector/TestVectorSchemaRoot.java index c121d94892..bd3113f8bc 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestVectorSchemaRoot.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestVectorSchemaRoot.java @@ -171,6 +171,26 @@ public void testAddVector() { } } + @Test + public void testAddVectorAtEnd() { + try (final IntVector intVector1 = new IntVector("intVector1", allocator); + final IntVector intVector2 = new IntVector("intVector2", allocator); + final IntVector intVector3 = new IntVector("intVector3", allocator); ) { + + VectorSchemaRoot original = new VectorSchemaRoot(Arrays.asList(intVector1, intVector2)); + assertEquals(2, original.getFieldVectors().size()); + + VectorSchemaRoot newRecordBatch = original.addVector(2, intVector3); + assertEquals(3, newRecordBatch.getFieldVectors().size()); + assertEquals(intVector1, newRecordBatch.getFieldVectors().get(0)); + assertEquals(intVector2, newRecordBatch.getFieldVectors().get(1)); + assertEquals(intVector3, newRecordBatch.getFieldVectors().get(2)); + + original.close(); + newRecordBatch.close(); + } + } + @Test public void testRemoveVector() { try (final IntVector intVector1 = new IntVector("intVector1", allocator); From 18de621ff2e7a72f54d416b9ef6e6a4a96b2aa8d Mon Sep 17 00:00:00 2001 From: Pedro Matias Date: Wed, 11 Mar 2026 10:59:07 +0000 Subject: [PATCH 176/271] =?UTF-8?q?GH-1004:=20=20[JDBC]=20Fix=20NPE=20in?= =?UTF-8?q?=20ArrowFlightJdbcDriver#connect=E2=80=8B(final=20String=20url,?= =?UTF-8?q?=20final=20Properties=20info)=20=20(#1005)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What's Changed `ArrowFlightJdbcDriver.connect(final String url, final Properties info)` now properly ignores a null value for `info`, obtaining the properties solely from the URL. Closes #1004. --- .../driver/jdbc/ArrowFlightJdbcDriver.java | 4 +++- .../jdbc/ArrowFlightJdbcDriverTest.java | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriver.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriver.java index 53e6120f62..12ef8030d7 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriver.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriver.java @@ -75,7 +75,9 @@ public Logger getParentLogger() { public ArrowFlightConnection connect(final String url, final Properties info) throws SQLException { final Properties properties = new Properties(info); - properties.putAll(info); + if (info != null) { + properties.putAll(info); + } if (url != null) { final Optional> maybeProperties = getUrlsArgs(url); diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriverTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriverTest.java index ae355829d7..88fb9889b6 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriverTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriverTest.java @@ -201,6 +201,30 @@ public void testConnectWithInsensitiveCasePropertyKeys2() throws Exception { } } + /** + * Tests whether the {@link ArrowFlightJdbcDriver} can establish a successful connection to the + * Arrow Flight client when provided with null properties. + */ + @Test + public void testConnectWithNullProperties() throws Exception { + final Driver driver = new ArrowFlightJdbcDriver(); + try (Connection connection = + driver.connect( + "jdbc:arrow-flight://" + + dataSource.getConfig().getHost() + + ":" + + dataSource.getConfig().getPort() + + "?" + + "useEncryption=false" + + "&user=" + + dataSource.getConfig().getUser() + + "&password=" + + dataSource.getConfig().getPassword(), + null)) { + assertTrue(connection.isValid(300)); + } + } + /** * Tests whether an exception is thrown upon attempting to connect to a malformed URI. * From 4a7fb4ef95d9386a0a7102fb18fbe52547527c21 Mon Sep 17 00:00:00 2001 From: Aleksei Starikov Date: Wed, 11 Mar 2026 14:59:21 +0100 Subject: [PATCH 177/271] GH-552: [Vector] Add absent methods to the UnionFixedSizeListWriter (#1052) ## What's Changed Add absent methods to the `UnionFixedSizeListWriter`. 1. Aligned the `UnionFixedSizeListWriter` template with the `UnionListWriter` template, which added the following previously absent methods to the generated `UnionFixedSizeListWriter` class: ``` - duration() methods - DurationWriter duration() - DurationWriter duration(String name, org.apache.arrow.vector.types.TimeUnit unit) - DurationWriter duration(String name) - timeStampSecTZ() methods - TimeStampSecTZWriter timeStampSecTZ() - TimeStampSecTZWriter timeStampSecTZ(String name, String timezone) - TimeStampSecTZWriter timeStampSecTZ(String name) - timeStampMilliTZ() methods - TimeStampMilliTZWriter timeStampMilliTZ() - TimeStampMilliTZWriter timeStampMilliTZ(String name, String timezone) - TimeStampMilliTZWriter timeStampMilliTZ(String name) - timeStampMicroTZ() methods - TimeStampMicroTZWriter timeStampMicroTZ() - TimeStampMicroTZWriter timeStampMicroTZ(String name, String timezone) - TimeStampMicroTZWriter timeStampMicroTZ(String name) - timeStampNanoTZ() methods - TimeStampNanoTZWriter timeStampNanoTZ() - TimeStampNanoTZWriter timeStampNanoTZ(String name, String timezone) - TimeStampNanoTZWriter timeStampNanoTZ(String name) - fixedSizeBinary() methods - FixedSizeBinaryWriter fixedSizeBinary() - FixedSizeBinaryWriter fixedSizeBinary(String name, int byteWidth) - FixedSizeBinaryWriter fixedSizeBinary(String name) - write() methods for Duration - void writeDuration(long value) - void write(DurationHolder holder) - write() methods for TimeStampSecTZ - void writeTimeStampSecTZ(long value) - void write(TimeStampSecTZHolder holder) - write() methods for TimeStampMilliTZ - void writeTimeStampMilliTZ(long value) - void write(TimeStampMilliTZHolder holder) - write() methods for TimeStampMicroTZ - void writeTimeStampMicroTZ(long value) - void write(TimeStampMicroTZHolder holder) - write() methods for TimeStampNanoTZ - void writeTimeStampNanoTZ(long value) - void write(TimeStampNanoTZHolder holder) - write() methods for FixedSizeBinary - void writeFixedSizeBinary(ArrowBuf buffer) - void write(FixedSizeBinaryHolder holder) ``` 2. Add `structName = name;` for 2 existing methods (align them with other similar methods): ``` - DecimalWriter decimal(String name) - Decimal256Writer decimal256(String name) ``` 3. Remove unused assignments from the `UnionListWriter` template + add missing overrides. This fix adds override annotations to the `UnionListWriter` generated class and extend/fix the code of the `UnionFixedSizeListWriter` generated class. So, the change is backward compatible. See the gists for the generated writer classes: - [UnionFixedSizeListWriter](https://gist.github.com/axreldable/9908f98d75ec4c0a62e4ccfa176cbbe1) - [UnionListWriter](https://gist.github.com/axreldable/881f3cf1ed001c513870daf3ea4f3bbd) --- Inspired by this [PR](https://github.com/apache/arrow/pull/35353). --- Closes #552 . --- .../templates/UnionFixedSizeListWriter.java | 151 +++++-------- .../codegen/templates/UnionListWriter.java | 6 +- .../arrow/vector/TestFixedSizeListVector.java | 207 ++++++++++++++++++ .../apache/arrow/vector/TestMapVector.java | 35 ++- .../org/apache/arrow/vector/TestUtils.java | 13 ++ 5 files changed, 293 insertions(+), 119 deletions(-) diff --git a/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java b/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java index f6e3f63caf..484199ab2a 100644 --- a/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java +++ b/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java @@ -35,6 +35,10 @@ <#include "/@includes/vv_imports.ftl" /> +<#function is_timestamp_tz type> + <#return type?starts_with("TimeStamp") && type?ends_with("TZ")> + + /* * This class is generated using freemarker and the ${.template_name} template. */ @@ -96,55 +100,30 @@ public void close() throws Exception { public void setPosition(int index) { super.setPosition(index); } - <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> - <#assign fields = minor.fields!type.fields /> - <#assign uncappedName = name?uncap_first/> - <#if uncappedName == "int" ><#assign uncappedName = "integer" /> - <#if !minor.typeParams?? > + <#list vv.types as type><#list type.minor as minor> + <#assign lowerName = minor.class?uncap_first /> + <#if lowerName == "int" ><#assign lowerName = "integer" /> + <#assign upperName = minor.class?upper_case /> @Override - public ${name}Writer ${uncappedName}() { + public ${minor.class}Writer ${lowerName}() { return this; } + <#if minor.typeParams?? > @Override - public ${name}Writer ${uncappedName}(String name) { - structName = name; - return writer.${uncappedName}(name); + public ${minor.class}Writer ${lowerName}(String name<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}) { + return writer.${lowerName}(name<#list minor.typeParams as typeParam>, ${typeParam.name}); } - - - @Override - public DecimalWriter decimal() { - return this; - } - - @Override - public DecimalWriter decimal(String name, int scale, int precision) { - return writer.decimal(name, scale, precision); - } - - @Override - public DecimalWriter decimal(String name) { - return writer.decimal(name); - } - @Override - public Decimal256Writer decimal256() { - return this; - } - - @Override - public Decimal256Writer decimal256(String name, int scale, int precision) { - return writer.decimal256(name, scale, precision); + public ${minor.class}Writer ${lowerName}(String name) { + structName = name; + return writer.${lowerName}(name); } - @Override - public Decimal256Writer decimal256(String name) { - return writer.decimal256(name); - } + @Override public StructWriter struct() { @@ -215,87 +194,86 @@ public void end() { } @Override - public void write(DecimalHolder holder) { - if (writer.idx() >= (idx() + 1) * listSize) { - throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); - } - writer.write(holder); - writer.setPosition(writer.idx() + 1); - } - - @Override - public void write(Decimal256Holder holder) { + public void writeNull() { if (writer.idx() >= (idx() + 1) * listSize) { throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); } - writer.write(holder); - writer.setPosition(writer.idx() + 1); + writer.writeNull(); } + <#list vv.types as type> + <#list type.minor as minor> + <#assign name = minor.class?cap_first /> + <#assign fields = minor.fields!type.fields /> + <#assign uncappedName = name?uncap_first/> @Override - public void writeNull() { + public void write${name}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, ) { if (writer.idx() >= (idx() + 1) * listSize) { throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); } - writer.writeNull(); + writer.write${name}(<#list fields as field>${field.name}<#if field_has_next>, ); + writer.setPosition(writer.idx()+1); } - public void writeDecimal(long start, ArrowBuf buffer, ArrowType arrowType) { + <#if is_timestamp_tz(minor.class) || minor.class == "Duration" || minor.class == "FixedSizeBinary"> + @Override + public void write(${name}Holder holder) { if (writer.idx() >= (idx() + 1) * listSize) { throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); } - writer.writeDecimal(start, buffer, arrowType); - writer.setPosition(writer.idx() + 1); + writer.write(holder); + writer.setPosition(writer.idx()+1); } - public void writeDecimal(BigDecimal value) { + <#elseif minor.class?starts_with("Decimal")> + @Override + public void write${name}(long start, ArrowBuf buffer, ArrowType arrowType) { if (writer.idx() >= (idx() + 1) * listSize) { throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); } - writer.writeDecimal(value); - writer.setPosition(writer.idx() + 1); + writer.write${name}(start, buffer, arrowType); + writer.setPosition(writer.idx()+1); } - public void writeBigEndianBytesToDecimal(byte[] value, ArrowType arrowType) { + @Override + public void write(${name}Holder holder) { if (writer.idx() >= (idx() + 1) * listSize) { throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); } - writer.writeBigEndianBytesToDecimal(value, arrowType); - writer.setPosition(writer.idx() + 1); + writer.write(holder); + writer.setPosition(writer.idx()+1); } - public void writeDecimal256(long start, ArrowBuf buffer, ArrowType arrowType) { + @Override + public void write${name}(BigDecimal value) { if (writer.idx() >= (idx() + 1) * listSize) { throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); } - writer.writeDecimal256(start, buffer, arrowType); - writer.setPosition(writer.idx() + 1); + writer.write${name}(value); + writer.setPosition(writer.idx()+1); } - public void writeDecimal256(BigDecimal value) { + @Override + public void writeBigEndianBytesTo${name}(byte[] value, ArrowType arrowType){ if (writer.idx() >= (idx() + 1) * listSize) { throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); } - writer.writeDecimal256(value); + writer.writeBigEndianBytesTo${name}(value, arrowType); writer.setPosition(writer.idx() + 1); } - - public void writeBigEndianBytesToDecimal256(byte[] value, ArrowType arrowType) { + <#else> + @Override + public void write(${name}Holder holder) { if (writer.idx() >= (idx() + 1) * listSize) { throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); } - writer.writeBigEndianBytesToDecimal256(value, arrowType); - writer.setPosition(writer.idx() + 1); + writer.write${name}(<#list fields as field>holder.${field.name}<#if field_has_next>, ); + writer.setPosition(writer.idx()+1); } + - - <#list vv.types as type> - <#list type.minor as minor> - <#assign name = minor.class?cap_first /> - <#assign fields = minor.fields!type.fields /> - <#assign uncappedName = name?uncap_first/> - <#if minor.class?ends_with("VarBinary")> + <#if minor.class?ends_with("VarBinary")> @Override public void write${minor.class}(byte[] value) { if (writer.idx() >= (idx() + 1) * listSize) { @@ -349,27 +327,8 @@ public void writeBigEndianBytesToDecimal256(byte[] value, ArrowType arrowType) { writer.write${minor.class}(value); writer.setPosition(writer.idx() + 1); } - - - <#if !minor.typeParams?? > - @Override - public void write${name}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, ) { - if (writer.idx() >= (idx() + 1) * listSize) { - throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); - } - writer.write${name}(<#list fields as field>${field.name}<#if field_has_next>, ); - writer.setPosition(writer.idx() + 1); - } - - public void write(${name}Holder holder) { - if (writer.idx() >= (idx() + 1) * listSize) { - throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); - } - writer.write${name}(<#list fields as field>holder.${field.name}<#if field_has_next>, ); - writer.setPosition(writer.idx() + 1); - } + - } diff --git a/vector/src/main/codegen/templates/UnionListWriter.java b/vector/src/main/codegen/templates/UnionListWriter.java index 4b54739230..394348f029 100644 --- a/vector/src/main/codegen/templates/UnionListWriter.java +++ b/vector/src/main/codegen/templates/UnionListWriter.java @@ -123,8 +123,6 @@ public void setPosition(int index) { <#assign lowerName = minor.class?uncap_first /> <#if lowerName == "int" ><#assign lowerName = "integer" /> <#assign upperName = minor.class?upper_case /> - <#assign capName = minor.class?cap_first /> - <#assign vectName = capName /> @Override public ${minor.class}Writer ${lowerName}() { return this; @@ -370,6 +368,7 @@ public void write(${name}Holder holder) { } <#elseif minor.class?starts_with("Decimal")> + @Override public void write${name}(long start, ArrowBuf buffer, ArrowType arrowType) { writer.write${name}(start, buffer, arrowType); writer.setPosition(writer.idx()+1); @@ -381,11 +380,13 @@ public void write(${name}Holder holder) { writer.setPosition(writer.idx()+1); } + @Override public void write${name}(BigDecimal value) { writer.write${name}(value); writer.setPosition(writer.idx()+1); } + @Override public void writeBigEndianBytesTo${name}(byte[] value, ArrowType arrowType){ writer.writeBigEndianBytesTo${name}(value, arrowType); writer.setPosition(writer.idx() + 1); @@ -429,6 +430,7 @@ public void write(${name}Holder holder) { writer.setPosition(writer.idx() + 1); } + @Override public void write${minor.class}(String value) { writer.write${minor.class}(value); writer.setPosition(writer.idx() + 1); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java index 73a88b3a1e..b3455fe52c 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java @@ -30,14 +30,21 @@ import java.util.Arrays; import java.util.List; import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.BaseRepeatedValueVector; import org.apache.arrow.vector.complex.FixedSizeListVector; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.impl.UnionFixedSizeListReader; import org.apache.arrow.vector.complex.impl.UnionFixedSizeListWriter; import org.apache.arrow.vector.complex.impl.UnionListReader; import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.DurationHolder; +import org.apache.arrow.vector.holders.FixedSizeBinaryHolder; +import org.apache.arrow.vector.holders.TimeStampMilliTZHolder; +import org.apache.arrow.vector.holders.TimeStampNanoTZHolder; +import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.Text; import org.apache.arrow.vector.util.TransferPair; @@ -628,6 +635,206 @@ public void testWriteLargeVarBinaryHelpers() throws Exception { } } + @Test + public void testWriterTimeStampNanoTZField() { + try (final FixedSizeListVector vector = + FixedSizeListVector.empty("vector", /* size= */ 3, allocator)) { + UnionFixedSizeListWriter writer = vector.getWriter(); + writer.allocate(); + + final int valueCount = 10; + + for (int i = 0; i < valueCount; i++) { + writer.startList(); + writer.timeStampNanoTZ().writeTimeStampNanoTZ(i * 1000L); + writer.timeStampNanoTZ().writeTimeStampNanoTZ((i + 1) * 1000L); + writer.timeStampNanoTZ().writeTimeStampNanoTZ((i + 2) * 1000L); + writer.endList(); + } + vector.setValueCount(valueCount); + + UnionFixedSizeListReader reader = vector.getReader(); + for (int i = 0; i < valueCount; i++) { + reader.setPosition(i); + assertTrue(reader.isSet()); + assertTrue(reader.next()); + assertEquals(i * 1000L, reader.reader().readLong().longValue()); + assertTrue(reader.next()); + assertEquals((i + 1) * 1000L, reader.reader().readLong().longValue()); + assertTrue(reader.next()); + assertEquals((i + 2) * 1000L, reader.reader().readLong().longValue()); + assertFalse(reader.next()); + } + } + } + + @Test + public void testWriterUsingHolderTimeStampNanoTZField() { + try (final FixedSizeListVector vector = + FixedSizeListVector.empty("vector", /* size= */ 3, allocator)) { + UnionFixedSizeListWriter writer = vector.getWriter(); + writer.allocate(); + + TimeStampNanoTZHolder holder = new TimeStampNanoTZHolder(); + holder.timezone = "SomeFakeTimeZone"; + writer.startList(); + holder.value = 12341234L; + writer.timeStampNanoTZ().write(holder); + holder.value = 55555L; + writer.timeStampNanoTZ().write(holder); + + // Writing with a different timezone should throw + holder.timezone = "AsdfTimeZone"; + holder.value = 77777; + IllegalArgumentException ex = + assertThrows( + IllegalArgumentException.class, () -> writer.timeStampNanoTZ().write(holder)); + assertEquals( + "holder.timezone: AsdfTimeZone not equal to vector timezone: SomeFakeTimeZone", + ex.getMessage()); + + writer.endList(); + vector.setValueCount(1); + + Field expectedDataField = + new Field( + BaseRepeatedValueVector.DATA_VECTOR_NAME, + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.NANOSECOND, "SomeFakeTimeZone")), + null); + Field expectedField = + new Field( + vector.getName(), + FieldType.nullable(new ArrowType.FixedSizeList(3)), + List.of(expectedDataField)); + + assertEquals(expectedField, writer.getField()); + } + } + + @Test + public void testWriterUsingHolderTimestampMilliTZField() { + try (final FixedSizeListVector vector = + FixedSizeListVector.empty("vector", /* size= */ 3, allocator)) { + UnionFixedSizeListWriter writer = vector.getWriter(); + writer.allocate(); + + TimeStampMilliTZHolder holder = new TimeStampMilliTZHolder(); + holder.timezone = "SomeFakeTimeZone"; + writer.startList(); + holder.value = 12341234L; + writer.timeStampMilliTZ().write(holder); + holder.value = 55555L; + writer.timeStampMilliTZ().write(holder); + + // Writing with a different timezone should throw + holder.timezone = "AsdfTimeZone"; + holder.value = 77777; + IllegalArgumentException ex = + assertThrows( + IllegalArgumentException.class, () -> writer.timeStampMilliTZ().write(holder)); + assertEquals( + "holder.timezone: AsdfTimeZone not equal to vector timezone: SomeFakeTimeZone", + ex.getMessage()); + + writer.endList(); + vector.setValueCount(1); + + Field expectedDataField = + new Field( + BaseRepeatedValueVector.DATA_VECTOR_NAME, + FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, "SomeFakeTimeZone")), + null); + Field expectedField = + new Field( + vector.getName(), + FieldType.nullable(new ArrowType.FixedSizeList(3)), + List.of(expectedDataField)); + + assertEquals(expectedField, writer.getField()); + } + } + + @Test + public void testWriterUsingHolderDurationField() { + try (final FixedSizeListVector vector = + FixedSizeListVector.empty("vector", /* size= */ 3, allocator)) { + UnionFixedSizeListWriter writer = vector.getWriter(); + writer.allocate(); + + DurationHolder durationHolder = new DurationHolder(); + durationHolder.unit = TimeUnit.MILLISECOND; + + writer.startList(); + durationHolder.value = 812374L; + writer.duration().write(durationHolder); + durationHolder.value = 143451L; + writer.duration().write(durationHolder); + + // Writing with a different unit should throw + durationHolder.unit = TimeUnit.SECOND; + durationHolder.value = 8888888; + IllegalArgumentException ex = + assertThrows( + IllegalArgumentException.class, () -> writer.duration().write(durationHolder)); + assertEquals("holder.unit: SECOND not equal to vector unit: MILLISECOND", ex.getMessage()); + + writer.endList(); + vector.setValueCount(1); + + Field expectedDataField = + new Field( + BaseRepeatedValueVector.DATA_VECTOR_NAME, + FieldType.nullable(new ArrowType.Duration(TimeUnit.MILLISECOND)), + null); + Field expectedField = + new Field( + vector.getName(), + FieldType.nullable(new ArrowType.FixedSizeList(3)), + List.of(expectedDataField)); + + assertEquals(expectedField, writer.getField()); + } + } + + @Test + public void testWriterUsingHolderFixedSizeBinaryField() { + try (final FixedSizeListVector vector = + FixedSizeListVector.empty("vector", /* size= */ 2, allocator)) { + UnionFixedSizeListWriter writer = vector.getWriter(); + writer.allocate(); + + FixedSizeBinaryHolder holder1 = + TestUtils.fixedSizeBinaryHolder(allocator, new byte[] {11, 22}); + FixedSizeBinaryHolder holder2 = + TestUtils.fixedSizeBinaryHolder(allocator, new byte[] {32, 21}); + + writer.startList(); + writer.fixedSizeBinary().write(holder1); + holder1.buffer.close(); + writer.fixedSizeBinary().write(holder2); + holder2.buffer.close(); + + writer.endList(); + vector.setValueCount(1); + + FieldReader reader = vector.getReader(); + assertTrue(reader.isSet(), "shouldn't be null"); + + Field expectedDataField = + new Field( + BaseRepeatedValueVector.DATA_VECTOR_NAME, + FieldType.nullable(new ArrowType.FixedSizeBinary(2)), + null); + Field expectedField = + new Field( + vector.getName(), + FieldType.nullable(new ArrowType.FixedSizeList(2)), + List.of(expectedDataField)); + + assertEquals(expectedField, writer.getField()); + } + } + private int[] convertListToIntArray(List list) { int[] values = new int[list.size()]; for (int i = 0; i < list.size(); i++) { diff --git a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java index 274d2973bd..2f520f3882 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java @@ -1353,17 +1353,6 @@ public void testCopyFromForExtensionType() throws Exception { } } - private FixedSizeBinaryHolder getFixedSizeBinaryHolder(byte[] array) { - FixedSizeBinaryHolder holder = new FixedSizeBinaryHolder(); - holder.byteWidth = array.length; - holder.buffer = allocator.buffer(array.length); - for (int i = 0; i < array.length; i++) { - holder.buffer.setByte(i, array[i]); - } - - return holder; - } - /** * Regression test for GH-586: UnionMapWriter.fixedSizeBinary() should properly delegate to the * entry writer for both key and value paths. @@ -1382,8 +1371,10 @@ public void testFixedSizeBinaryWriter() { // {[11, 22] -> null} // {null -> [32, 21]} - wrong "for a given entry, the "key" is non-nullable" - todo: it // shouldn't work. Should it? - FixedSizeBinaryHolder holder1 = getFixedSizeBinaryHolder(new byte[] {11, 22}); - FixedSizeBinaryHolder holder2 = getFixedSizeBinaryHolder(new byte[] {32, 21}); + FixedSizeBinaryHolder holder1 = + TestUtils.fixedSizeBinaryHolder(allocator, new byte[] {11, 22}); + FixedSizeBinaryHolder holder2 = + TestUtils.fixedSizeBinaryHolder(allocator, new byte[] {32, 21}); writer.setPosition(0); // optional writer.startMap(); @@ -1399,8 +1390,8 @@ public void testFixedSizeBinaryWriter() { writer.endMap(); // {1 -> [11, 22], 2 -> [32, 21]} - holder1 = getFixedSizeBinaryHolder(new byte[] {11, 22}); - holder2 = getFixedSizeBinaryHolder(new byte[] {32, 21}); + holder1 = TestUtils.fixedSizeBinaryHolder(allocator, new byte[] {11, 22}); + holder2 = TestUtils.fixedSizeBinaryHolder(allocator, new byte[] {32, 21}); writer.setPosition(1); writer.startMap(); writer.startEntry(); @@ -1416,8 +1407,8 @@ public void testFixedSizeBinaryWriter() { holder2.buffer.close(); // {[11, 22] -> 1, [32, 21] -> 2} - holder1 = getFixedSizeBinaryHolder(new byte[] {11, 22}); - holder2 = getFixedSizeBinaryHolder(new byte[] {32, 21}); + holder1 = TestUtils.fixedSizeBinaryHolder(allocator, new byte[] {11, 22}); + holder2 = TestUtils.fixedSizeBinaryHolder(allocator, new byte[] {32, 21}); writer.setPosition(3); writer.startMap(); writer.startEntry(); @@ -1433,7 +1424,7 @@ public void testFixedSizeBinaryWriter() { holder2.buffer.close(); // {[11, 22] -> null} - holder1 = getFixedSizeBinaryHolder(new byte[] {11, 22}); + holder1 = TestUtils.fixedSizeBinaryHolder(allocator, new byte[] {11, 22}); writer.setPosition(4); writer.startMap(); writer.startEntry(); @@ -1443,7 +1434,7 @@ public void testFixedSizeBinaryWriter() { holder1.buffer.close(); // {null -> [32, 21]} - holder2 = getFixedSizeBinaryHolder(new byte[] {32, 21}); + holder2 = TestUtils.fixedSizeBinaryHolder(allocator, new byte[] {32, 21}); writer.setPosition(5); writer.startMap(); writer.startEntry(); @@ -1536,8 +1527,10 @@ public void testFixedSizeBinaryFirstInitialization() { // populate input vector with the following records // {[11, 22] -> [32, 21]} - FixedSizeBinaryHolder holder1 = getFixedSizeBinaryHolder(new byte[] {11, 22}); - FixedSizeBinaryHolder holder2 = getFixedSizeBinaryHolder(new byte[] {32, 21}); + FixedSizeBinaryHolder holder1 = + TestUtils.fixedSizeBinaryHolder(allocator, new byte[] {11, 22}); + FixedSizeBinaryHolder holder2 = + TestUtils.fixedSizeBinaryHolder(allocator, new byte[] {32, 21}); writer.setPosition(0); // optional writer.startMap(); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestUtils.java b/vector/src/test/java/org/apache/arrow/vector/TestUtils.java index c28751aa58..d91b2004c0 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestUtils.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestUtils.java @@ -18,6 +18,7 @@ import java.util.Random; import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.holders.FixedSizeBinaryHolder; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.ExtensionTypeRegistry; @@ -73,4 +74,16 @@ public static void ensureRegistered(ArrowType.ExtensionType type) { ExtensionTypeRegistry.register(type); } } + + public static FixedSizeBinaryHolder fixedSizeBinaryHolder( + BufferAllocator allocator, byte[] array) { + FixedSizeBinaryHolder holder = new FixedSizeBinaryHolder(); + holder.byteWidth = array.length; + holder.buffer = allocator.buffer(array.length); + for (int i = 0; i < array.length; i++) { + holder.buffer.setByte(i, array[i]); + } + + return holder; + } } From 77df3ecb2cf5517fb5d37a4b2806844e3b4700df Mon Sep 17 00:00:00 2001 From: Yicong Huang <17627829+Yicong-Huang@users.noreply.github.com> Date: Thu, 12 Mar 2026 01:39:44 -0700 Subject: [PATCH 178/271] GH-343: Fix BaseVariableWidthVector and BaseLargeVariableWidthVector offset buffer serialization (#989) ## What's Changed Fix `BaseVariableWidthVector`/`BaseLargeVariableWidthVector` IPC serialization when `valueCount` is 0. ### Problem When `valueCount == 0`, `setReaderAndWriterIndex()` was setting `offsetBuffer.writerIndex(0)`, which means `readableBytes() == 0`. IPC serializer uses `readableBytes()` to determine buffer size, so 0 bytes were written to the IPC stream. This crashes IPC readers in other libraries because Arrow spec requires offset buffer to have at least one entry `[0]`. This is a follow-up to #967 which fixed the same issue in `ListVector`/`LargeListVector`. ### Fix Modify `setReaderAndWriterIndex()` to always use `(valueCount + 1) * OFFSET_WIDTH` for the offset buffer's `writerIndex`, moved outside the if/else branch. When the offset buffer capacity is insufficient (e.g., empty buffer from constructor or loaded via `loadFieldBuffers()`), it reallocates a properly sized buffer on demand. ### Testing Added tests for empty `VarCharVector` and `LargeVarCharVector` verifying offset buffer has correct `readableBytes()` after `setValueCount(0)`. Closes #343 --------- Co-authored-by: Yicong Huang --- .../adapter/jdbc/ResultSetUtilityTest.java | 22 ++++++----- .../vector/BaseLargeVariableWidthVector.java | 16 +++++++- .../arrow/vector/BaseVariableWidthVector.java | 16 +++++++- .../apache/arrow/vector/TestValueVector.java | 38 +++++++++++++++++++ 4 files changed, 79 insertions(+), 13 deletions(-) diff --git a/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ResultSetUtilityTest.java b/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ResultSetUtilityTest.java index c7dc9b2791..e5039ccf59 100644 --- a/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ResultSetUtilityTest.java +++ b/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ResultSetUtilityTest.java @@ -43,15 +43,19 @@ public void testZeroRowResultSet() throws Exception { .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) .build(); - ArrowVectorIterator iter = JdbcToArrow.sqlToArrowVectorIterator(rs, config); - assertTrue(iter.hasNext(), "Iterator on zero row ResultSet should haveNext() before use"); - VectorSchemaRoot root = iter.next(); - assertNotNull(root, "VectorSchemaRoot from first next() result should never be null"); - assertEquals( - 0, root.getRowCount(), "VectorSchemaRoot from empty ResultSet should have zero rows"); - assertFalse( - iter.hasNext(), - "hasNext() should return false on empty ResultSets after initial next() call"); + try (ArrowVectorIterator iter = JdbcToArrow.sqlToArrowVectorIterator(rs, config)) { + assertTrue(iter.hasNext(), "Iterator on zero row ResultSet should haveNext() before use"); + VectorSchemaRoot root = iter.next(); + assertNotNull(root, "VectorSchemaRoot from first next() result should never be null"); + assertEquals( + 0, root.getRowCount(), "VectorSchemaRoot from empty ResultSet should have zero rows"); + assertFalse( + iter.hasNext(), + "hasNext() should return false on empty ResultSets after initial next() call"); + if (!reuseVectorSchemaRoot) { + root.close(); + } + } } } } diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java index 6c451f10a7..3fac195786 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java @@ -373,14 +373,26 @@ private void setReaderAndWriterIndex() { valueBuffer.readerIndex(0); if (valueCount == 0) { validityBuffer.writerIndex(0); - offsetBuffer.writerIndex(0); valueBuffer.writerIndex(0); } else { final long lastDataOffset = getStartOffset(valueCount); validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); - offsetBuffer.writerIndex((long) (valueCount + 1) * OFFSET_WIDTH); valueBuffer.writerIndex(lastDataOffset); } + // IPC serializer will determine readable bytes based on `readerIndex` and `writerIndex`. + // Both are set to 0 means 0 bytes are written to the IPC stream which will crash IPC readers + // in other libraries. According to Arrow spec, we should still output the offset buffer which + // is [0]. + final long requiredOffsetBufferSize = (long) (valueCount + 1) * OFFSET_WIDTH; + if (offsetBuffer.capacity() < requiredOffsetBufferSize) { + ArrowBuf newOffsetBuffer = allocateOffsetBuffer(requiredOffsetBufferSize); + if (offsetBuffer.capacity() > 0) { + newOffsetBuffer.setBytes(0, offsetBuffer, 0, offsetBuffer.capacity()); + } + offsetBuffer.getReferenceManager().release(); + offsetBuffer = newOffsetBuffer; + } + offsetBuffer.writerIndex(requiredOffsetBufferSize); } /** Same as {@link #allocateNewSafe()}. */ diff --git a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java index 96e2afbd29..d5bd167256 100644 --- a/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java @@ -389,14 +389,26 @@ private void setReaderAndWriterIndex() { valueBuffer.readerIndex(0); if (valueCount == 0) { validityBuffer.writerIndex(0); - offsetBuffer.writerIndex(0); valueBuffer.writerIndex(0); } else { final int lastDataOffset = getStartOffset(valueCount); validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount)); - offsetBuffer.writerIndex((long) (valueCount + 1) * OFFSET_WIDTH); valueBuffer.writerIndex(lastDataOffset); } + // IPC serializer will determine readable bytes based on `readerIndex` and `writerIndex`. + // Both are set to 0 means 0 bytes are written to the IPC stream which will crash IPC readers + // in other libraries. According to Arrow spec, we should still output the offset buffer which + // is [0]. + final long requiredOffsetBufferSize = (long) (valueCount + 1) * OFFSET_WIDTH; + if (offsetBuffer.capacity() < requiredOffsetBufferSize) { + ArrowBuf newOffsetBuffer = allocateOffsetBuffer(requiredOffsetBufferSize); + if (offsetBuffer.capacity() > 0) { + newOffsetBuffer.setBytes(0, offsetBuffer, 0, offsetBuffer.capacity()); + } + offsetBuffer.getReferenceManager().release(); + offsetBuffer = newOffsetBuffer; + } + offsetBuffer.writerIndex(requiredOffsetBufferSize); } /** Same as {@link #allocateNewSafe()}. */ diff --git a/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index df42d04e60..22c93b0cbe 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -3940,4 +3940,42 @@ public void testVectorLoadUnloadOnNonVariadicVectors() { } } } + + @Test + public void testEmptyVarCharOffsetBuffer() { + // Validates that offset buffer has at least OFFSET_WIDTH bytes (for offset[0]=0) + // even when valueCount is 0, per Arrow specification. + try (VarCharVector vector = newVarCharVector("varchar", allocator)) { + vector.allocateNew(); + vector.setValueCount(0); + + List buffers = vector.getFieldBuffers(); + // buffers: [validity, offset, data] + assertTrue( + buffers.get(1).readableBytes() >= BaseVariableWidthVector.OFFSET_WIDTH, + "Offset buffer should have at least " + + BaseVariableWidthVector.OFFSET_WIDTH + + " bytes for offset[0]"); + assertEquals(0, vector.getOffsetBuffer().getInt(0)); + } + } + + @Test + public void testEmptyLargeVarCharOffsetBuffer() { + // Validates that offset buffer has at least OFFSET_WIDTH bytes (for offset[0]=0) + // even when valueCount is 0, per Arrow specification. + try (LargeVarCharVector vector = new LargeVarCharVector("largevarchar", allocator)) { + vector.allocateNew(); + vector.setValueCount(0); + + List buffers = vector.getFieldBuffers(); + // buffers: [validity, offset, data] + assertTrue( + buffers.get(1).readableBytes() >= BaseLargeVariableWidthVector.OFFSET_WIDTH, + "Offset buffer should have at least " + + BaseLargeVariableWidthVector.OFFSET_WIDTH + + " bytes for offset[0]"); + assertEquals(0, vector.getOffsetBuffer().getLong(0)); + } + } } From b410fb26d01c7cefc4c6e3443a53562164001371 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Thu, 12 Mar 2026 11:58:50 +0100 Subject: [PATCH 179/271] MINOR: Bump version to 19.0.0 (#1066) --- adapter/avro/pom.xml | 2 +- adapter/jdbc/pom.xml | 2 +- adapter/orc/pom.xml | 2 +- algorithm/pom.xml | 2 +- arrow-variant/pom.xml | 2 +- bom/pom.xml | 4 ++-- c/pom.xml | 2 +- compression/pom.xml | 2 +- dataset/pom.xml | 2 +- flight/flight-core/pom.xml | 2 +- flight/flight-integration-tests/pom.xml | 2 +- flight/flight-sql-jdbc-core/pom.xml | 2 +- flight/flight-sql-jdbc-driver/pom.xml | 2 +- flight/flight-sql/pom.xml | 2 +- flight/pom.xml | 2 +- format/pom.xml | 2 +- gandiva/pom.xml | 2 +- memory/memory-core/pom.xml | 2 +- memory/memory-netty-buffer-patch/pom.xml | 2 +- memory/memory-netty/pom.xml | 2 +- memory/memory-unsafe/pom.xml | 2 +- memory/pom.xml | 2 +- performance/pom.xml | 2 +- pom.xml | 6 +++--- tools/pom.xml | 2 +- vector/pom.xml | 2 +- 26 files changed, 29 insertions(+), 29 deletions(-) diff --git a/adapter/avro/pom.xml b/adapter/avro/pom.xml index 827d19f2a2..18c48a0e8f 100644 --- a/adapter/avro/pom.xml +++ b/adapter/avro/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 ../../pom.xml diff --git a/adapter/jdbc/pom.xml b/adapter/jdbc/pom.xml index 2f621d7a05..a0819d7aef 100644 --- a/adapter/jdbc/pom.xml +++ b/adapter/jdbc/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 ../../pom.xml diff --git a/adapter/orc/pom.xml b/adapter/orc/pom.xml index c96ab36119..fbb72d6c19 100644 --- a/adapter/orc/pom.xml +++ b/adapter/orc/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 ../../pom.xml diff --git a/algorithm/pom.xml b/algorithm/pom.xml index 898c2605b6..116be9aebf 100644 --- a/algorithm/pom.xml +++ b/algorithm/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 arrow-algorithm Arrow Algorithms diff --git a/arrow-variant/pom.xml b/arrow-variant/pom.xml index 3a842178a4..fea724824b 100644 --- a/arrow-variant/pom.xml +++ b/arrow-variant/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 arrow-variant Arrow Variant diff --git a/bom/pom.xml b/bom/pom.xml index 0de43a1217..6a4f741fca 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -29,7 +29,7 @@ under the License. org.apache.arrow arrow-bom - 19.0.0-SNAPSHOT + 19.0.0 pom Arrow Bill of Materials @@ -68,7 +68,7 @@ under the License. scm:git:https://github.com/apache/arrow-java.git scm:git:https://github.com/apache/arrow-java.git - main + v19.0.0 https://github.com/apache/arrow-java/tree/${project.scm.tag} diff --git a/c/pom.xml b/c/pom.xml index c90b6dc0ef..b0a7ffe41d 100644 --- a/c/pom.xml +++ b/c/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 arrow-c-data diff --git a/compression/pom.xml b/compression/pom.xml index 29f8b41788..92144addc4 100644 --- a/compression/pom.xml +++ b/compression/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 arrow-compression Arrow Compression diff --git a/dataset/pom.xml b/dataset/pom.xml index 1852c6eddc..df5620c641 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 arrow-dataset diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml index f1d58a0cad..fbed544a1b 100644 --- a/flight/flight-core/pom.xml +++ b/flight/flight-core/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 19.0.0-SNAPSHOT + 19.0.0 flight-core diff --git a/flight/flight-integration-tests/pom.xml b/flight/flight-integration-tests/pom.xml index f0f10ada43..5ee7c6fc14 100644 --- a/flight/flight-integration-tests/pom.xml +++ b/flight/flight-integration-tests/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 19.0.0-SNAPSHOT + 19.0.0 flight-integration-tests diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index da00baf32a..d6fa11688d 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 19.0.0-SNAPSHOT + 19.0.0 flight-sql-jdbc-core diff --git a/flight/flight-sql-jdbc-driver/pom.xml b/flight/flight-sql-jdbc-driver/pom.xml index 559c42597d..e6f23bcb08 100644 --- a/flight/flight-sql-jdbc-driver/pom.xml +++ b/flight/flight-sql-jdbc-driver/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 19.0.0-SNAPSHOT + 19.0.0 flight-sql-jdbc-driver diff --git a/flight/flight-sql/pom.xml b/flight/flight-sql/pom.xml index a5954819c3..a58b76acda 100644 --- a/flight/flight-sql/pom.xml +++ b/flight/flight-sql/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 19.0.0-SNAPSHOT + 19.0.0 flight-sql diff --git a/flight/pom.xml b/flight/pom.xml index 2fc3e89ef8..a5a40a834a 100644 --- a/flight/pom.xml +++ b/flight/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 arrow-flight diff --git a/format/pom.xml b/format/pom.xml index d3578b63d2..c09fad32fb 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 arrow-format diff --git a/gandiva/pom.xml b/gandiva/pom.xml index 5367bfdedf..d26edeb9d6 100644 --- a/gandiva/pom.xml +++ b/gandiva/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 org.apache.arrow.gandiva diff --git a/memory/memory-core/pom.xml b/memory/memory-core/pom.xml index 72ee69d60a..586fddae1a 100644 --- a/memory/memory-core/pom.xml +++ b/memory/memory-core/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 19.0.0-SNAPSHOT + 19.0.0 arrow-memory-core diff --git a/memory/memory-netty-buffer-patch/pom.xml b/memory/memory-netty-buffer-patch/pom.xml index 07dc7d2403..cb38efa345 100644 --- a/memory/memory-netty-buffer-patch/pom.xml +++ b/memory/memory-netty-buffer-patch/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 19.0.0-SNAPSHOT + 19.0.0 arrow-memory-netty-buffer-patch diff --git a/memory/memory-netty/pom.xml b/memory/memory-netty/pom.xml index 6d660da117..f33eb95e44 100644 --- a/memory/memory-netty/pom.xml +++ b/memory/memory-netty/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 19.0.0-SNAPSHOT + 19.0.0 arrow-memory-netty diff --git a/memory/memory-unsafe/pom.xml b/memory/memory-unsafe/pom.xml index 92dc0c9fe5..d941fee645 100644 --- a/memory/memory-unsafe/pom.xml +++ b/memory/memory-unsafe/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 19.0.0-SNAPSHOT + 19.0.0 arrow-memory-unsafe diff --git a/memory/pom.xml b/memory/pom.xml index bc34c26050..af953ccd21 100644 --- a/memory/pom.xml +++ b/memory/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 arrow-memory pom diff --git a/performance/pom.xml b/performance/pom.xml index 3f18188e3a..685f433f05 100644 --- a/performance/pom.xml +++ b/performance/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 arrow-performance jar diff --git a/pom.xml b/pom.xml index 19625617b1..0b0aa58232 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 pom Apache Arrow Java Root POM @@ -82,7 +82,7 @@ under the License. scm:git:https://github.com/apache/arrow-java.git scm:git:https://github.com/apache/arrow-java.git - main + v19.0.0 https://github.com/apache/arrow-java/tree/${project.scm.tag} @@ -92,7 +92,7 @@ under the License. - 1695310533 + 1773307790 ${project.build.directory}/generated-sources 1.9.0 5.12.2 diff --git a/tools/pom.xml b/tools/pom.xml index d43adb1fdf..9e557bab76 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 arrow-tools Arrow Tools diff --git a/vector/pom.xml b/vector/pom.xml index f46bd0e7b4..bc64cbc76b 100644 --- a/vector/pom.xml +++ b/vector/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0-SNAPSHOT + 19.0.0 arrow-vector Arrow Vectors From 77127ef54272b578075c574071f24ee8c1bba22e Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 16 Mar 2026 16:17:40 +0900 Subject: [PATCH 180/271] GH-1077: Add missing `export GH_TOKEN` to release scripts --- dev/release/bump_version.sh | 1 + dev/release/release.sh | 1 + dev/release/release_rc.sh | 1 + 3 files changed, 3 insertions(+) diff --git a/dev/release/bump_version.sh b/dev/release/bump_version.sh index 458e930f98..68cafb99bd 100755 --- a/dev/release/bump_version.sh +++ b/dev/release/bump_version.sh @@ -37,6 +37,7 @@ if [ ! -f "${SOURCE_DIR}/.env" ]; then exit 1 fi . "${SOURCE_DIR}/.env" +export GH_TOKEN cd "${SOURCE_TOP_DIR}" diff --git a/dev/release/release.sh b/dev/release/release.sh index f08a618c4f..d1db7ad05a 100755 --- a/dev/release/release.sh +++ b/dev/release/release.sh @@ -36,6 +36,7 @@ if [ ! -f "${SOURCE_DIR}/.env" ]; then exit 1 fi . "${SOURCE_DIR}/.env" +export GH_TOKEN git_origin_url="$(git remote get-url origin)" repository="${git_origin_url#*github.com?}" diff --git a/dev/release/release_rc.sh b/dev/release/release_rc.sh index ff77718b8d..0920edbe35 100755 --- a/dev/release/release_rc.sh +++ b/dev/release/release_rc.sh @@ -42,6 +42,7 @@ if [ ! -f "${SOURCE_DIR}/.env" ]; then exit 1 fi . "${SOURCE_DIR}/.env" +export GH_TOKEN cd "${SOURCE_TOP_DIR}" From d5e132b96500646ea4021d3b032d00a65233b6b5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 08:45:29 +0100 Subject: [PATCH 181/271] MINOR: [CI] Bump actions/download-artifact from 8.0.0 to 8.0.1 (#1068) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 8.0.0 to 8.0.1.
Release notes

Sourced from actions/download-artifact's releases.

v8.0.1

What's Changed

Full Changelog: https://github.com/actions/download-artifact/compare/v8...v8.0.1

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/download-artifact&package-manager=github_actions&previous-version=8.0.0&new-version=8.0.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index a202777143..4973b1afb5 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -101,7 +101,7 @@ jobs: packages: write steps: - name: Download source archive - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: release-source - name: Extract source archive @@ -168,7 +168,7 @@ jobs: MACOSX_DEPLOYMENT_TARGET: "14.0" steps: - name: Download source archive - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: release-source - name: Extract source archive @@ -296,7 +296,7 @@ jobs: arch: "x86_64" steps: - name: Download source archive - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: release-source - name: Extract source archive @@ -369,7 +369,7 @@ jobs: - jni-windows steps: - name: Download artifacts - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: artifacts - name: Decompress artifacts @@ -450,11 +450,11 @@ jobs: with: cache: 'pip' - name: Download source archive - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: release-source - name: Download Javadocs - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: reference - name: Extract source archive @@ -519,7 +519,7 @@ jobs: cp ../.asf.yaml ./ git add .nojekyll .asf.yaml - name: Download - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: release-html - name: Extract @@ -555,7 +555,7 @@ jobs: - ubuntu-latest steps: - name: Download release artifacts - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: release-* - name: Verify @@ -589,7 +589,7 @@ jobs: contents: write steps: - name: Download release artifacts - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: release-* path: artifacts From fa781dcc3c42ab98214743e3aef1eb8fd5297209 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 08:45:54 +0100 Subject: [PATCH 182/271] MINOR: Bump com.squareup.okio:okio-jvm from 3.16.4 to 3.17.0 (#1075) Bumps [com.squareup.okio:okio-jvm](https://github.com/square/okio) from 3.16.4 to 3.17.0.
Changelog

Sourced from com.squareup.okio:okio-jvm's changelog.

Version 3.17.0

2026-03-11

  • New: Adjust down the Kotlin stdlib dependency to [Kotlin 2.1.21][kotlin_2_1_21]. Okio is built with an up-to-date Kotlin compiler (2.2.21), but depends on an older kotlin-stdlib. We're doing this so you can update Okio and Kotlin independently.

  • Fix: Return the correct timestamp in FileMetadata.createdAtMillis on Kotlin/Native on UNIX platforms. We were incorrectly using the POSIX ctime (change time) instead of the birthtime. With this fix Okio now prefers statx() over stat() on native platforms. This API first appeared in Linux in 4.11 (2017) and Android in API 30 (2020).

Commits
  • 80a5023 Prepare for release 3.17.0.
  • 65c0c26 Switch to FileMetadata to use statx instead of stat on Linux and Apple platfo...
  • b11f17b Remove Kotlin/JS IR default parameter workarounds. (#1786)
  • b35f473 Update Gradle to v9.4.0 (#1785)
  • cbcee31 Update actions/upload-artifact action to v7 (#1783)
  • fc7aecb Update dependency com.android.tools.build:gradle to v9.0.1 (#1781)
  • 79aa267 Drop isWasm() early return workaround for KT-60212. (#1777)
  • 45459dc Fix result of an 'errnoToIOException' call is not thrown. inside `PosixFileSy...
  • 9fbab0f Decode env variables in WASI tests (#1773)
  • 50abe89 Stop using AssertJ (#1771)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.squareup.okio:okio-jvm&package-manager=maven&previous-version=3.16.4&new-version=3.17.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index d6fa11688d..4b8122fa4d 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -141,7 +141,7 @@ under the License. com.squareup.okio okio-jvm - 3.16.4 + 3.17.0 test From 47cd032939f15edbfc7c87fbcb02d585f82e5948 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 08:46:10 +0100 Subject: [PATCH 183/271] MINOR: Bump org.codehaus.mojo:properties-maven-plugin from 1.2.1 to 1.3.0 (#1072) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.codehaus.mojo:properties-maven-plugin](https://github.com/mojohaus/properties-maven-plugin) from 1.2.1 to 1.3.0.
Release notes

Sourced from org.codehaus.mojo:properties-maven-plugin's releases.

1.3.0

🚀 New features and improvements

👻 Maintenance

🔧 Build

📦 Dependency updates

Commits
  • 91a2ade [maven-release-plugin] prepare release properties-maven-plugin-1.3.0
  • bf56d32 Bump org.codehaus.mojo:mojo-parent from 94 to 95
  • 80e20be Bump org.codehaus.mojo:mojo-parent from 93 to 94
  • e8ae7a5 Bump org.yaml:snakeyaml from 2.4 to 2.5
  • bb39c3d Bump org.codehaus.mojo:mojo-parent from 92 to 93
  • b41267b Bump org.codehaus.mojo:mojo-parent from 91 to 92
  • bb548c5 Bump org.codehaus.mojo:mojo-parent from 87 to 91
  • 3ffa9cb Use Sisu plugin
  • 7adbe3b Require Maven 3.6.3
  • 407342f Bump org.yaml:snakeyaml from 2.3 to 2.4
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.codehaus.mojo:properties-maven-plugin&package-manager=maven&previous-version=1.2.1&new-version=1.3.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0b0aa58232..a414dcd229 100644 --- a/pom.xml +++ b/pom.xml @@ -502,7 +502,7 @@ under the License. org.codehaus.mojo properties-maven-plugin - 1.2.1 + 1.3.0 org.codehaus.mojo From 4242d587b25a74e944f2a01e79d52dcd401d5cf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Mon, 16 Mar 2026 09:37:09 +0100 Subject: [PATCH 184/271] MINOR: Bump version to 20.0.0-SNAPSHOT (#1076) --- adapter/avro/pom.xml | 2 +- adapter/jdbc/pom.xml | 2 +- adapter/orc/pom.xml | 2 +- algorithm/pom.xml | 2 +- arrow-variant/pom.xml | 2 +- bom/pom.xml | 4 ++-- c/pom.xml | 2 +- compression/pom.xml | 2 +- dataset/pom.xml | 2 +- flight/flight-core/pom.xml | 2 +- flight/flight-integration-tests/pom.xml | 2 +- flight/flight-sql-jdbc-core/pom.xml | 2 +- flight/flight-sql-jdbc-driver/pom.xml | 2 +- flight/flight-sql/pom.xml | 2 +- flight/pom.xml | 2 +- format/pom.xml | 2 +- gandiva/pom.xml | 2 +- memory/memory-core/pom.xml | 2 +- memory/memory-netty-buffer-patch/pom.xml | 2 +- memory/memory-netty/pom.xml | 2 +- memory/memory-unsafe/pom.xml | 2 +- memory/pom.xml | 2 +- performance/pom.xml | 2 +- pom.xml | 6 +++--- tools/pom.xml | 2 +- vector/pom.xml | 2 +- 26 files changed, 29 insertions(+), 29 deletions(-) diff --git a/adapter/avro/pom.xml b/adapter/avro/pom.xml index 18c48a0e8f..4f7f90d7a9 100644 --- a/adapter/avro/pom.xml +++ b/adapter/avro/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT ../../pom.xml diff --git a/adapter/jdbc/pom.xml b/adapter/jdbc/pom.xml index a0819d7aef..9ff44593ff 100644 --- a/adapter/jdbc/pom.xml +++ b/adapter/jdbc/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT ../../pom.xml diff --git a/adapter/orc/pom.xml b/adapter/orc/pom.xml index fbb72d6c19..50a9b3a603 100644 --- a/adapter/orc/pom.xml +++ b/adapter/orc/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT ../../pom.xml diff --git a/algorithm/pom.xml b/algorithm/pom.xml index 116be9aebf..24adcefa6f 100644 --- a/algorithm/pom.xml +++ b/algorithm/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT arrow-algorithm Arrow Algorithms diff --git a/arrow-variant/pom.xml b/arrow-variant/pom.xml index fea724824b..e578626dd4 100644 --- a/arrow-variant/pom.xml +++ b/arrow-variant/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT arrow-variant Arrow Variant diff --git a/bom/pom.xml b/bom/pom.xml index 6a4f741fca..f9200a7e8d 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -29,7 +29,7 @@ under the License. org.apache.arrow arrow-bom - 19.0.0 + 20.0.0-SNAPSHOT pom Arrow Bill of Materials @@ -68,7 +68,7 @@ under the License. scm:git:https://github.com/apache/arrow-java.git scm:git:https://github.com/apache/arrow-java.git - v19.0.0 + main https://github.com/apache/arrow-java/tree/${project.scm.tag} diff --git a/c/pom.xml b/c/pom.xml index b0a7ffe41d..27b6619c4c 100644 --- a/c/pom.xml +++ b/c/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT arrow-c-data diff --git a/compression/pom.xml b/compression/pom.xml index 92144addc4..945738d2b8 100644 --- a/compression/pom.xml +++ b/compression/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT arrow-compression Arrow Compression diff --git a/dataset/pom.xml b/dataset/pom.xml index df5620c641..7a0210ce95 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT arrow-dataset diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml index fbed544a1b..92490dd67b 100644 --- a/flight/flight-core/pom.xml +++ b/flight/flight-core/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 19.0.0 + 20.0.0-SNAPSHOT flight-core diff --git a/flight/flight-integration-tests/pom.xml b/flight/flight-integration-tests/pom.xml index 5ee7c6fc14..ec81162e59 100644 --- a/flight/flight-integration-tests/pom.xml +++ b/flight/flight-integration-tests/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 19.0.0 + 20.0.0-SNAPSHOT flight-integration-tests diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 4b8122fa4d..53c630ab40 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 19.0.0 + 20.0.0-SNAPSHOT flight-sql-jdbc-core diff --git a/flight/flight-sql-jdbc-driver/pom.xml b/flight/flight-sql-jdbc-driver/pom.xml index e6f23bcb08..55de7221ec 100644 --- a/flight/flight-sql-jdbc-driver/pom.xml +++ b/flight/flight-sql-jdbc-driver/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 19.0.0 + 20.0.0-SNAPSHOT flight-sql-jdbc-driver diff --git a/flight/flight-sql/pom.xml b/flight/flight-sql/pom.xml index a58b76acda..b7c8931391 100644 --- a/flight/flight-sql/pom.xml +++ b/flight/flight-sql/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-flight - 19.0.0 + 20.0.0-SNAPSHOT flight-sql diff --git a/flight/pom.xml b/flight/pom.xml index a5a40a834a..30f75fa27e 100644 --- a/flight/pom.xml +++ b/flight/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT arrow-flight diff --git a/format/pom.xml b/format/pom.xml index c09fad32fb..8c2f2d3387 100644 --- a/format/pom.xml +++ b/format/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT arrow-format diff --git a/gandiva/pom.xml b/gandiva/pom.xml index d26edeb9d6..190bf016ce 100644 --- a/gandiva/pom.xml +++ b/gandiva/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT org.apache.arrow.gandiva diff --git a/memory/memory-core/pom.xml b/memory/memory-core/pom.xml index 586fddae1a..1c7b6f8834 100644 --- a/memory/memory-core/pom.xml +++ b/memory/memory-core/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 19.0.0 + 20.0.0-SNAPSHOT arrow-memory-core diff --git a/memory/memory-netty-buffer-patch/pom.xml b/memory/memory-netty-buffer-patch/pom.xml index cb38efa345..039b2aa04a 100644 --- a/memory/memory-netty-buffer-patch/pom.xml +++ b/memory/memory-netty-buffer-patch/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 19.0.0 + 20.0.0-SNAPSHOT arrow-memory-netty-buffer-patch diff --git a/memory/memory-netty/pom.xml b/memory/memory-netty/pom.xml index f33eb95e44..4218910980 100644 --- a/memory/memory-netty/pom.xml +++ b/memory/memory-netty/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 19.0.0 + 20.0.0-SNAPSHOT arrow-memory-netty diff --git a/memory/memory-unsafe/pom.xml b/memory/memory-unsafe/pom.xml index d941fee645..3fafb42802 100644 --- a/memory/memory-unsafe/pom.xml +++ b/memory/memory-unsafe/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-memory - 19.0.0 + 20.0.0-SNAPSHOT arrow-memory-unsafe diff --git a/memory/pom.xml b/memory/pom.xml index af953ccd21..4ea3d1f9ca 100644 --- a/memory/pom.xml +++ b/memory/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT arrow-memory pom diff --git a/performance/pom.xml b/performance/pom.xml index 685f433f05..96ea5291ad 100644 --- a/performance/pom.xml +++ b/performance/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT arrow-performance jar diff --git a/pom.xml b/pom.xml index a414dcd229..43149bc957 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT pom Apache Arrow Java Root POM @@ -82,7 +82,7 @@ under the License. scm:git:https://github.com/apache/arrow-java.git scm:git:https://github.com/apache/arrow-java.git - v19.0.0 + main https://github.com/apache/arrow-java/tree/${project.scm.tag} @@ -92,7 +92,7 @@ under the License. - 1773307790 + 1773644827 ${project.build.directory}/generated-sources 1.9.0 5.12.2 diff --git a/tools/pom.xml b/tools/pom.xml index 9e557bab76..64634b9abe 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT arrow-tools Arrow Tools diff --git a/vector/pom.xml b/vector/pom.xml index bc64cbc76b..4d247961c9 100644 --- a/vector/pom.xml +++ b/vector/pom.xml @@ -22,7 +22,7 @@ under the License. org.apache.arrow arrow-java-root - 19.0.0 + 20.0.0-SNAPSHOT arrow-vector Arrow Vectors From 74a7996d40226c659c3b0ae03c0ae75a4964f282 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 09:55:27 +0100 Subject: [PATCH 185/271] MINOR: Bump com.nimbusds:oauth2-oidc-sdk from 11.20.1 to 11.34 (#1074) Bumps [com.nimbusds:oauth2-oidc-sdk](https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions) from 11.20.1 to 11.34.
Changelog

Sourced from com.nimbusds:oauth2-oidc-sdk's changelog.

version 1.0 (2012-05-29) * First official release with authorisation endpoint, token endpoint, check ID endpoint and UserInfo endpoint support. * JSON Web Tokens (JWTs) support through the Nimbus-JWT library. * Language Tags (RFC 5646) support through the Nimbus-LangTag library. * JSON support through the JSON Smart library.

version 2.0 (2013-05-13) * Intermediary development release with Maven build, published to Maven Central.

version 2.1 (2013-06-06) * Updates the APIs to OpenID Connect Messages draft 20, OpenID Connect Standard draft 21, OpenID Connect Discovery draft 17 and OpenID Connect Registration draft 19. * Major refactoring of the APIs for greater simplicity. * Adds JUnit tests.

version 2.2 (2013-06-18) * Refactors dynamic OpenID Connect client registration. * Adds partial support of the OAuth 2.0 Dynamic Client Registration Protocol (draft-ietf-oauth-dyn-reg-12). * Optimises parsing of request parameters consisting of one or more tokens (scope, response type, etc).

version 2.3 (2013-06-19) * Renames OAuth 2.0 dynamic client registration package. * Adds ClientInformation.getClientMetadata() method. * Adds OIDCClientInformation class.

version 2.4 (2013-06-20) * Adds static OIDCClientInformation.parse(JSONObject) method.

version 2.5 (2013-06-22) * Adds support OAuth 2.0 dynamic client update. * Adds OpenID Connect dynamic client registration classes.

version 2.6 (2013-06-25) * Enforces order of preference of ACR values in OpenID Connect client metadata, as required by the specification. * Documentation and performance improvements.

version 2.7 (2013-06-26) * Switches Identifier generation to java.security.SecureRandom.

version 2.8 (2013-06-30) * Fixes serialisation and assignment bugs in ClientMetadata. * Switches Secret generation to java.security.SecureRandom.

version 2.9 (2013-09-17)

... (truncated)

Commits
  • 668f6d8 The ParseException message thrown by Prompt.Type.parse must not include parse...
  • 75cde87 Updates test sample X.509 cert chain resource
  • a7a9623 [maven-release-plugin] prepare release 11.30.2
  • e03c9bb [maven-release-plugin] prepare for next development iteration
  • 6f11e30 Expands AMR test coverage
  • afba676 Adds static AMR.parseList(Collection<String>) method
  • 4b700b3 [maven-release-plugin] prepare release 11.31
  • b214cfa [maven-release-plugin] prepare for next development iteration
  • 28628f9 The DPoPCommonVerifier must instantiate the DPoPProofClaimsSetVerifier with t...
  • 4df4d53 The DPoPCommonVerifier must instantiate the DPoPProofClaimsSetVerifier with t...
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.nimbusds:oauth2-oidc-sdk&package-manager=maven&previous-version=11.20.1&new-version=11.34)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 53c630ab40..ffeff12462 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -182,7 +182,7 @@ under the License. com.nimbusds oauth2-oidc-sdk - 11.20.1 + 11.34 From 97e491399382215e248b3ba33e7f0d7d9650da5e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 09:59:10 +0100 Subject: [PATCH 186/271] MINOR: Bump com.github.luben:zstd-jni from 1.5.7-6 to 1.5.7-7 (#1073) Bumps [com.github.luben:zstd-jni](https://github.com/luben/zstd-jni) from 1.5.7-6 to 1.5.7-7.
Commits
  • 73bfa27 Bump version to v1.5.7-7
  • 322f6bc Update GH actions
  • d18bc0a Use latest MacOS runners
  • 1051112 Fix typo in ZstdDictCompress.java
  • 2d94b35 address
  • 7c2e3ff Avoid SetLongField call when GetPrimitiveArrayCritical return NULL
  • d82feda fix: ZstdInputStream decompression failure when underlying stream returns 0 t...
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.github.luben:zstd-jni&package-manager=maven&previous-version=1.5.7-6&new-version=1.5.7-7)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- compression/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compression/pom.xml b/compression/pom.xml index 945738d2b8..9014b6913a 100644 --- a/compression/pom.xml +++ b/compression/pom.xml @@ -55,7 +55,7 @@ under the License. com.github.luben zstd-jni - 1.5.7-6 + 1.5.7-7 From 394d12ef455fad83f76ec4b3a79ac6b5c6fe1139 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 10:04:36 +0100 Subject: [PATCH 187/271] MINOR: Bump com.fasterxml.jackson:jackson-bom from 2.21.0 to 2.21.1 (#1071) Bumps [com.fasterxml.jackson:jackson-bom](https://github.com/FasterXML/jackson-bom) from 2.21.0 to 2.21.1.
Commits
  • 08a5a9a [maven-release-plugin] prepare release jackson-bom-2.21.1
  • 5b03376 Prep for 2.21.1 release
  • 1d78778 Merge branch '2.20' into 2.21
  • cd46b24 Post-release dep version bump
  • 17179ff [maven-release-plugin] prepare for next development iteration
  • 2a26844 [maven-release-plugin] prepare release jackson-bom-2.20.2
  • 6adf11b Prep for 2.20.2 release
  • 441df8a Post-release version bump
  • a1b4814 [maven-release-plugin] prepare for next development iteration
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.fasterxml.jackson:jackson-bom&package-manager=maven&previous-version=2.21.0&new-version=2.21.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 43149bc957..eb7768780f 100644 --- a/pom.xml +++ b/pom.xml @@ -101,7 +101,7 @@ under the License. 4.2.9.Final 1.79.0 4.33.4 - 2.21.0 + 2.21.1 3.4.3 25.2.10 1.12.1 From 27382cd92baf29762dba2454d7e9e911c58821e3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 10:07:18 +0100 Subject: [PATCH 188/271] MINOR: Bump checker.framework.version from 3.53.1 to 3.54.0 (#1070) Bumps `checker.framework.version` from 3.53.1 to 3.54.0. Updates `org.checkerframework:checker-qual` from 3.53.1 to 3.54.0
Release notes

Sourced from org.checkerframework:checker-qual's releases.

Checker Framework 3.54.0

Version 3.54.0 (2026-03-02)

User-visible changes

Command-line arguments:

  • Added -AinferOutputDirectory.
  • Removed long-deprecated -Alint=forbidnonnullarraycomponents.

New command-line argument -Aonelinemsg puts error messages on a single line. This is useful when using a tool that only shows the first line of the error.

The command-line argument -Anomsgtext surrounds the error key with brackets instead of parenthesis. This matches Java error messages.

Implementation details

In AnnotatedTypeFactory, canonicalAnnotation() returns a non-null value.

In AnnotationClassLoader:

  • Renamed hasWellDefinedTargetMetaAnnotation() to isTypeQualifierAnnotation(). The method now returns true for annotations bearing @InvisibleQualifier or @SubtypeOf, in addition to the existing @Target(TYPE_USE) check.

In TestDiagnostic:

  • Renamed field message to key.
  • Added new nullable field message for the full message without the key.

Removed classes and methods that have been deprecated for more than two years.

Closed issues

#6874, #7471, #7475, #7486.

Changelog

Sourced from org.checkerframework:checker-qual's changelog.

Version 3.54.0 (2026-03-02)

User-visible changes

Command-line arguments:

  • Added -AinferOutputDirectory.
  • Removed long-deprecated -Alint=forbidnonnullarraycomponents.

New command-line argument -Aonelinemsg puts error messages on a single line. This is useful when using a tool that only shows the first line of the error.

The command-line argument -Anomsgtext surrounds the error key with brackets instead of parenthesis. This matches Java error messages.

Implementation details

In AnnotatedTypeFactory, canonicalAnnotation() returns a non-null value.

In AnnotationClassLoader:

  • Renamed hasWellDefinedTargetMetaAnnotation() to isTypeQualifierAnnotation(). The method now returns true for annotations bearing @InvisibleQualifier or @SubtypeOf, in addition to the existing @Target(TYPE_USE) check.

In TestDiagnostic:

  • Renamed field message to key.
  • Added new nullable field message for the full message without the key.

Removed classes and methods that have been deprecated for more than two years.

Closed issues

#6874, #7471, #7475, #7486.

Commits
  • a6eff70 new release 3.54.0
  • fd34700 Prep for release.
  • edb6e7a Print error key in brackets (#7525)
  • a79b1de Show details of the error message in test failures (#7513)
  • a5ecc22 Clone the JDK using the same fork and branch as CF (#7491)
  • 2770c52 Update cimg/base Docker tag to v2026.03
  • bba6bc9 Update plugin com-gradleup-shadow to v9.3.2
  • 3a6d4d4 Update error-prone monorepo to v2.48.0
  • 70aa5f3 Update plugin net-ltgt-errorprone to v5.1.0
  • 0dbd3e7 Prepare for javac AST changes
  • Additional commits viewable in compare view

Updates `org.checkerframework:checker` from 3.53.1 to 3.54.0
Release notes

Sourced from org.checkerframework:checker's releases.

Checker Framework 3.54.0

Version 3.54.0 (2026-03-02)

User-visible changes

Command-line arguments:

  • Added -AinferOutputDirectory.
  • Removed long-deprecated -Alint=forbidnonnullarraycomponents.

New command-line argument -Aonelinemsg puts error messages on a single line. This is useful when using a tool that only shows the first line of the error.

The command-line argument -Anomsgtext surrounds the error key with brackets instead of parenthesis. This matches Java error messages.

Implementation details

In AnnotatedTypeFactory, canonicalAnnotation() returns a non-null value.

In AnnotationClassLoader:

  • Renamed hasWellDefinedTargetMetaAnnotation() to isTypeQualifierAnnotation(). The method now returns true for annotations bearing @InvisibleQualifier or @SubtypeOf, in addition to the existing @Target(TYPE_USE) check.

In TestDiagnostic:

  • Renamed field message to key.
  • Added new nullable field message for the full message without the key.

Removed classes and methods that have been deprecated for more than two years.

Closed issues

#6874, #7471, #7475, #7486.

Changelog

Sourced from org.checkerframework:checker's changelog.

Version 3.54.0 (2026-03-02)

User-visible changes

Command-line arguments:

  • Added -AinferOutputDirectory.
  • Removed long-deprecated -Alint=forbidnonnullarraycomponents.

New command-line argument -Aonelinemsg puts error messages on a single line. This is useful when using a tool that only shows the first line of the error.

The command-line argument -Anomsgtext surrounds the error key with brackets instead of parenthesis. This matches Java error messages.

Implementation details

In AnnotatedTypeFactory, canonicalAnnotation() returns a non-null value.

In AnnotationClassLoader:

  • Renamed hasWellDefinedTargetMetaAnnotation() to isTypeQualifierAnnotation(). The method now returns true for annotations bearing @InvisibleQualifier or @SubtypeOf, in addition to the existing @Target(TYPE_USE) check.

In TestDiagnostic:

  • Renamed field message to key.
  • Added new nullable field message for the full message without the key.

Removed classes and methods that have been deprecated for more than two years.

Closed issues

#6874, #7471, #7475, #7486.

Commits
  • a6eff70 new release 3.54.0
  • fd34700 Prep for release.
  • edb6e7a Print error key in brackets (#7525)
  • a79b1de Show details of the error message in test failures (#7513)
  • a5ecc22 Clone the JDK using the same fork and branch as CF (#7491)
  • 2770c52 Update cimg/base Docker tag to v2026.03
  • bba6bc9 Update plugin com-gradleup-shadow to v9.3.2
  • 3a6d4d4 Update error-prone monorepo to v2.48.0
  • 70aa5f3 Update plugin net-ltgt-errorprone to v5.1.0
  • 0dbd3e7 Prepare for javac AST changes
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index eb7768780f..27aa503b99 100644 --- a/pom.xml +++ b/pom.xml @@ -112,7 +112,7 @@ under the License. 10.23.0 true 2.42.0 - 3.53.1 + 3.54.0 1.5.32 none -Xdoclint:none From 899e883428ee067da4542393e7ecc958cfb5d90c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 13:14:24 +0100 Subject: [PATCH 189/271] MINOR: Bump com.google.protobuf:protobuf-bom from 4.33.4 to 4.34.1 (#1086) Bumps [com.google.protobuf:protobuf-bom](https://github.com/protocolbuffers/protobuf) from 4.33.4 to 4.34.1.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.protobuf:protobuf-bom&package-manager=maven&previous-version=4.33.4&new-version=4.34.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 27aa503b99..61ecddf867 100644 --- a/pom.xml +++ b/pom.xml @@ -100,7 +100,7 @@ under the License. 33.4.8-jre 4.2.9.Final 1.79.0 - 4.33.4 + 4.34.1 2.21.1 3.4.3 25.2.10 From a8b238061bfdab433d25ee223ce11d697a572efd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 13:15:02 +0100 Subject: [PATCH 190/271] MINOR: Bump org.mockito:mockito-bom from 5.21.0 to 5.23.0 (#1087) Bumps [org.mockito:mockito-bom](https://github.com/mockito/mockito) from 5.21.0 to 5.23.0.
Release notes

Sourced from org.mockito:mockito-bom's releases.

v5.23.0

NOTE: Breaking change for Android

The mockito-android artifact has a breaking change: tests now require a device or emulator based on API 28+ (Android P). This is to enable new support for mocking Kotlin classes. See #3788 for more details.

Changelog generated by Shipkit Changelog Gradle Plugin

5.23.0

v5.22.0

Changelog generated by Shipkit Changelog Gradle Plugin

5.22.0

Commits
  • a231205 Fix StackOverflowError with AbstractList after using mockSingleton (#3790)
  • f6a91a6 Replace mockito-android mock maker implementation with dexmaker-mockito-inlin...
  • aa2298a fix: make spotless happy
  • a6729d6 chore: update BDDMockito with jspecify annotation
  • bb83c92 chore: move jspecify as a compile only dependency
  • 47a4695 chore: add jspecify with minimal change. Fixes #3503
  • 25f1395 Add core API to enable Kotlin singleton mocking (#3762)
  • ef9ee55 Avoids mocking private static methods, as well as package-private static meth...
  • d16fcfc Bump graalvm/setup-graalvm from 1.4.4 to 1.4.5 (#3780)
  • 27eb8a3 Clarify RETURNS_MOCKS behavior with sealed abstract enums (Java 15+) (#3773)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.mockito:mockito-bom&package-manager=maven&previous-version=5.21.0&new-version=5.23.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 61ecddf867..c96656a67d 100644 --- a/pom.xml +++ b/pom.xml @@ -106,7 +106,7 @@ under the License. 25.2.10 1.12.1 1.17.0 - 5.21.0 + 5.23.0 2 10.23.0 From 1348b8dc8302daebe7bb76df9f3c13bdcf2766e9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 13:16:04 +0100 Subject: [PATCH 191/271] MINOR: Bump io.netty:netty-bom from 4.2.9.Final to 4.2.10.Final (#1085) Bumps [io.netty:netty-bom](https://github.com/netty/netty) from 4.2.9.Final to 4.2.10.Final.
Commits
  • 4cc9873 [maven-release-plugin] prepare release netty-4.2.10.Final
  • 54b8663 Remove unnecessary allocations and abstractions in HttpContentCompressor (#16...
  • 961f427 Update to netty-tcnative 2.0.75.Final (#16227)
  • 3007ba9 Use recommanded finalize chain pattern when override finalize() method (#16212)
  • b918042 Update some dependencies (#16198) (#16215)
  • 874c995 Reduce allocations on DefaultHeaders::containsValue (#15843)
  • e0fe794 Remove unnecessary null check in WebSocketServerExtensionHandler (#16201)
  • 1b0636b Move default compression options into static variable in HttpContentCompresso...
  • 85a3a0e codec-http2: move the accessors from Http2Headers to DefaultHttp2Headers (#16...
  • f44a88d Improve chunk picking for the large-size buddy allocator (#16179)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=io.netty:netty-bom&package-manager=maven&previous-version=4.2.9.Final&new-version=4.2.10.Final)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c96656a67d..c0a01c6050 100644 --- a/pom.xml +++ b/pom.xml @@ -98,7 +98,7 @@ under the License. 5.12.2 2.0.17 33.4.8-jre - 4.2.9.Final + 4.2.10.Final 1.79.0 4.34.1 2.21.1 From d9d6112b3e4037d72aa44e978bc72c2f7706c52c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 18:46:51 +0100 Subject: [PATCH 192/271] MINOR: Bump com.google.guava:guava-bom from 33.4.8-jre to 33.5.0-jre (#1083) Bumps [com.google.guava:guava-bom](https://github.com/google/guava) from 33.4.8-jre to 33.5.0-jre.
Release notes

Sourced from com.google.guava:guava-bom's releases.

33.5.0

Maven

<dependency>
  <groupId>com.google.guava</groupId>
  <artifactId>guava</artifactId>
  <version>33.5.0-jre</version>
  <!-- or, for Android: -->
  <version>33.5.0-android</version>
</dependency>

Jar files

Guava requires one runtime dependency, which you can download here:

Javadoc

JDiff

Changelog

  • Restored the Automatic-Module-Name to guava-android. (It, unlike, guava-jre, is not a proper module.) (7a04a8a955)
  • For users of guava-gwt: Google has moved off GWT internally. We plan to continue to release guava-gwt for users of GWT and J2CL, but the artifact is no longer tested for GWT-specific issues, and we have limited resources to fix any unexpected issues that might arise. While we do not anticipate any specific problems, we can't guarantee how long support will continue.
  • Increased our Android minSdkVersion to 23 (Marshmallow). This follows the minimum of Google's foundational Android libraries, and we expect it to have no practical impact on users. (5c23347cc1)
  • Listed the JSpecify annotations as an optional dependency in our OSGi metadata. (2dfd572981)
  • cache: Improved the handling of exceptions from compute functions in Cache.asMap(). (We do still recommend using Caffeine rather than com.google.common.cache.) (087f2c4a80)
  • collect: Improved Iterators.mergeSorted() to preserve stability for equal elements. (4dc93be9a8)
  • math: Added saturatedAbs methods to IntMath and LongMath. (ed0e518f20)
  • net: Added image/avif to MediaType. (53344caba6)
  • testing: Made CollectorTester available to Android users. (294c251079)
  • util.concurrent: Added Striped.custom. (1586eb271d)
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.guava:guava-bom&package-manager=maven&previous-version=33.4.8-jre&new-version=33.5.0-jre)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c0a01c6050..200371e265 100644 --- a/pom.xml +++ b/pom.xml @@ -97,7 +97,7 @@ under the License. 1.9.0 5.12.2 2.0.17 - 33.4.8-jre + 33.5.0-jre 4.2.10.Final 1.79.0 4.34.1 From d5497915c604582e66613c96599b32a812fda4bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 18:47:12 +0100 Subject: [PATCH 193/271] MINOR: Bump com.fasterxml.jackson:jackson-bom from 2.21.1 to 2.21.2 (#1082) Bumps [com.fasterxml.jackson:jackson-bom](https://github.com/FasterXML/jackson-bom) from 2.21.1 to 2.21.2.
Commits
  • 10e12a5 [maven-release-plugin] prepare release jackson-bom-2.21.2
  • d754903 Prep for 2.21.2 release
  • 63e1b3b Post-release dep version bump
  • 716ab0d [maven-release-plugin] prepare for next development iteration
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.fasterxml.jackson:jackson-bom&package-manager=maven&previous-version=2.21.1&new-version=2.21.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 200371e265..fe5716a5f1 100644 --- a/pom.xml +++ b/pom.xml @@ -101,7 +101,7 @@ under the License. 4.2.10.Final 1.79.0 4.34.1 - 2.21.1 + 2.21.2 3.4.3 25.2.10 1.12.1 From 01affd741864c493cb76b9357fb32ead7f977672 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Tue, 24 Mar 2026 07:49:08 +0100 Subject: [PATCH 194/271] GH-1078: Upgrade minimum JDK version from 11 to 17 (#1079) Update compiler source/target/release to 17 across build config (pom.xml, bom), CI workflows, Docker images, Brewfile, and documentation. Replace deprecated boxed-type constructors with valueOf() in HolderReaderImpl codegen template to fix -Werror under release=17. ## What's Changed JDK 11 would not be supported in some cases. **This contains breaking changes.** Closes #1078. --- .env | 2 +- .github/workflows/rc.yml | 4 ++-- .github/workflows/test.yml | 8 ++++---- Brewfile | 2 +- bom/pom.xml | 8 ++++---- ci/docker/conda-jni.dockerfile | 2 +- ci/docker/vcpkg-jni.dockerfile | 2 +- compose.yaml | 4 ++-- docs/source/cdata.rst | 8 ++++---- docs/source/developers/building.rst | 16 ++++++++-------- docs/source/flight_sql_jdbc_driver.rst | 2 +- docs/source/install.rst | 4 ++-- docs/source/jdbc.rst | 2 +- docs/source/memory.rst | 2 +- pom.xml | 10 +++++----- .../main/codegen/templates/HolderReaderImpl.java | 4 ++-- 16 files changed, 40 insertions(+), 40 deletions(-) diff --git a/.env b/.env index a7783537d0..2bd7255476 100644 --- a/.env +++ b/.env @@ -47,7 +47,7 @@ ARROW_REPO=ghcr.io/apache/arrow-dev ULIMIT_CORE=-1 # Default versions for various dependencies -JDK=11 +JDK=17 MAVEN=3.9.9 # Versions for various dependencies used to build artifacts diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 4973b1afb5..9c75b8c807 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -273,7 +273,7 @@ jobs: run: | set -e # make brew Java available to CMake - export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home + export JAVA_HOME=$(brew --prefix openjdk@17)/libexec/openjdk.jdk/Contents/Home ci/scripts/jni_macos_build.sh . arrow build jni - name: Compress into single artifact to keep directory structure run: tar -cvzf jni-macos-${{ matrix.platform.arch }}.tar.gz jni/ @@ -317,7 +317,7 @@ jobs: - name: Set up Java uses: actions/setup-java@v5 with: - java-version: '11' + java-version: '17' distribution: 'temurin' - name: Download Timezone Database shell: bash diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2602592799..8c437a056d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,7 +45,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17, 21, 23] + jdk: [17, 21, 23] maven: [3.9.9] image: [ubuntu, conda-jni-cdata] include: @@ -88,10 +88,10 @@ jobs: matrix: include: - arch: AMD64 - jdk: 11 + jdk: 17 macos: 15-intel - arch: AArch64 - jdk: 11 + jdk: 17 macos: latest steps: - name: Set up Java @@ -123,7 +123,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11] + jdk: [17] steps: - name: Set up Java uses: actions/setup-java@v5 diff --git a/Brewfile b/Brewfile index af6bd65615..2c47a38af5 100644 --- a/Brewfile +++ b/Brewfile @@ -15,5 +15,5 @@ # specific language governing permissions and limitations # under the License. -brew "openjdk@11" +brew "openjdk@17" brew "sccache" diff --git a/bom/pom.xml b/bom/pom.xml index f9200a7e8d..e4ccee02db 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -80,10 +80,10 @@ under the License. - 11 - 11 - 11 - 11 + 17 + 17 + 17 + 17 diff --git a/ci/docker/conda-jni.dockerfile b/ci/docker/conda-jni.dockerfile index e14db73688..3f31b74052 100644 --- a/ci/docker/conda-jni.dockerfile +++ b/ci/docker/conda-jni.dockerfile @@ -17,7 +17,7 @@ FROM ghcr.io/mamba-org/micromamba:ubuntu24.04 -ARG jdk=11 +ARG jdk=17 ARG maven=3.9.9 RUN micromamba install -y \ diff --git a/ci/docker/vcpkg-jni.dockerfile b/ci/docker/vcpkg-jni.dockerfile index d6bd322a39..f2f5d0d45a 100644 --- a/ci/docker/vcpkg-jni.dockerfile +++ b/ci/docker/vcpkg-jni.dockerfile @@ -20,7 +20,7 @@ FROM ${base} # Install Java # We need Java for JNI headers, but we don't invoke Maven in this build. -ARG java=11 +ARG java=17 RUN dnf install -y java-$java-openjdk-devel && dnf clean all # For ci/scripts/{cpp,java}_*.sh diff --git a/compose.yaml b/compose.yaml index f5082a22aa..fb290b22fe 100644 --- a/compose.yaml +++ b/compose.yaml @@ -40,7 +40,7 @@ services: # docker compose run ubuntu # Parameters: # MAVEN: 3.9.9 - # JDK: 11, 17, 21 + # JDK: 17, 21 image: ${ARCH}/maven:${MAVEN}-eclipse-temurin-${JDK} volumes: - .:/arrow-java:delegated @@ -60,7 +60,7 @@ services: # docker compose run conda-jni-cdata # Parameters: # MAVEN: 3.9.9 - # JDK: 11, 17, 21 + # JDK: 17, 21 image: ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration build: context: . diff --git a/docs/source/cdata.rst b/docs/source/cdata.rst index 9643d88df3..7b2924d259 100644 --- a/docs/source/cdata.rst +++ b/docs/source/cdata.rst @@ -101,8 +101,8 @@ without writing JNI bindings ourselves. 1.0-SNAPSHOT - 8 - 8 + 17 + 17 9.0.0 @@ -237,8 +237,8 @@ For this example, we will build a JAR with all dependencies bundled. cpptojava 1.0-SNAPSHOT - 8 - 8 + 17 + 17 9.0.0 diff --git a/docs/source/developers/building.rst b/docs/source/developers/building.rst index f9ef7daea8..b682957714 100644 --- a/docs/source/developers/building.rst +++ b/docs/source/developers/building.rst @@ -32,7 +32,7 @@ Arrow Java uses the `Maven `_ build system. Building requires: -* JDK 11+ +* JDK 17+ * Maven 3+ .. note:: @@ -345,7 +345,7 @@ configuration file usually located under ``${HOME}/.m2`` with the following snip jdk - 21 + 21 temurin @@ -383,11 +383,11 @@ Arrow repository, and update the following settings: right click the directory, and select Mark Directory as > Generated Sources Root. There is no need to mark other generated sources directories, as only the ``vector`` module generates sources. -* For JDK 11, due to an `IntelliJ bug - `__, you must go into +* Due to an `IntelliJ bug + `__, you may need to go into Settings > Build, Execution, Deployment > Compiler > Java Compiler and disable "Use '--release' option for cross-compilation (Java 9 and later)". Otherwise - you will get an error like "package sun.misc does not exist". + you may get an error like "package sun.misc does not exist". * You may want to disable error-prone entirely if it gives spurious warnings (disable both error-prone profiles in the Maven tool window and "Reload All Maven Projects"). @@ -397,7 +397,7 @@ Arrow repository, and update the following settings: * To enable debugging JNI-based modules like ``dataset``, activate specific profiles in the Maven tab under "Profiles". Ensure the profiles ``arrow-c-data``, ``arrow-jni``, ``generate-libs-cdata-all-os``, - ``generate-libs-jni-macos-linux``, and ``jdk11+`` are enabled, so that the + ``generate-libs-jni-macos-linux``, and ``jdk17+`` are enabled, so that the IDE can build them and enable debugging. You may not need to update all of these settings if you build/test with the @@ -478,8 +478,8 @@ Installing Manually .. code-block:: xml - 8 - 8 + 17 + 17 9.0.0.dev501 diff --git a/docs/source/flight_sql_jdbc_driver.rst b/docs/source/flight_sql_jdbc_driver.rst index 4deb726b33..6d40434a22 100644 --- a/docs/source/flight_sql_jdbc_driver.rst +++ b/docs/source/flight_sql_jdbc_driver.rst @@ -27,7 +27,7 @@ Flight SQL. Installation and Requirements ============================= -The driver is compatible with JDK 11+. Note that the following JVM +The driver is compatible with JDK 17+. Note that the following JVM parameter is required: .. code-block:: shell diff --git a/docs/source/install.rst b/docs/source/install.rst index b2b1c7163f..e0b34515ef 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -27,8 +27,8 @@ Java modules are regularly built and tested on macOS and Linux distributions. Java Compatibility ================== -Java modules are compatible with JDK 11 and above. Currently, JDK versions -11, 17, 21, and latest are tested in CI. +Java modules are compatible with JDK 17 and above. Currently, JDK versions +17, 21, and latest are tested in CI. Note that some JDK internals must be exposed by adding ``--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED`` to the ``java`` command: diff --git a/docs/source/jdbc.rst b/docs/source/jdbc.rst index 2f57c34bf8..e054127faa 100644 --- a/docs/source/jdbc.rst +++ b/docs/source/jdbc.rst @@ -276,7 +276,7 @@ mapping, with additional support for the UUID extension type noted below. JDBC value, because a JDBC Timestamp is in UTC, and we have no timezone information. In this case, the default binder will call `setTimestamp(int, Timestamp) - `_, + `_, which will lead to the driver using the "default timezone" (that of the Java VM). * \(3) For the Flight SQL JDBC driver, the Arrow UUID extension type diff --git a/docs/source/memory.rst b/docs/source/memory.rst index 4a71ed846a..5b9148223a 100644 --- a/docs/source/memory.rst +++ b/docs/source/memory.rst @@ -341,7 +341,7 @@ How this works: .. _`newChildAllocator`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/RootAllocator.html#newChildAllocator-java.lang.String-org.apache.arrow.memory.AllocationListener-long-long- .. _`Netty`: https://netty.io/wiki/ .. _`sun.misc.unsafe`: https://web.archive.org/web/20210929024401/http://www.docjar.com/html/api/sun/misc/Unsafe.java.html -.. _`Direct Memory`: https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/nio/ByteBuffer.html +.. _`Direct Memory`: https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/nio/ByteBuffer.html .. _`ReferenceManager`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/ReferenceManager.html .. _`ReferenceManager.release`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/ReferenceManager.html#release-- .. _`ReferenceManager.retain`: https://arrow.apache.org/java/current/reference/org.apache.arrow.memory.core/org/apache/arrow/memory/ReferenceManager.html#retain-- diff --git a/pom.xml b/pom.xml index fe5716a5f1..863169b47b 100644 --- a/pom.xml +++ b/pom.xml @@ -119,10 +119,10 @@ under the License. --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED - 11 - 11 - 11 - 11 + 17 + 17 + 17 + 17

... (truncated)

Commits
  • c8eb81b in preparation for a release
  • 7ff5ee5 Merge pull request #4260 from JackPGreen/update-maven-central-url
  • da4c337 Merge pull request #4271 from IrisesD/master
  • d053544 feat: allow CATALOG in CREATE SCHEMA and DROP SCHEMA (#4277)
  • a448d91 Merge pull request #4273 from naive924/feat/compactThreads
  • 6672123 fix: change log
  • 858e74a fix: MVStore.compact: run map copy in parallel by default (¼ cores, override ...
  • bce0ec1 feat: parallel map copy option for MVStore.compact()
  • d8a6cc3 Fix command syntax in help.csv
  • c45413c Merge pull request #4266 from andreitokar/issue-4208-2
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.h2database:h2&package-manager=maven&previous-version=2.3.232&new-version=2.4.240)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- adapter/jdbc/pom.xml | 2 +- performance/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/adapter/jdbc/pom.xml b/adapter/jdbc/pom.xml index 9ff44593ff..a8ac19721d 100644 --- a/adapter/jdbc/pom.xml +++ b/adapter/jdbc/pom.xml @@ -59,7 +59,7 @@ under the License. com.h2database h2 - 2.3.232 + 2.4.240 test diff --git a/performance/pom.xml b/performance/pom.xml index 96ea5291ad..413d5ba1e0 100644 --- a/performance/pom.xml +++ b/performance/pom.xml @@ -75,7 +75,7 @@ under the License. com.h2database h2 - 2.3.232 + 2.4.240 runtime From 168a969147ea77a6fe8f12fa15c6bc25b1e671f2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 10:21:57 +0200 Subject: [PATCH 196/271] MINOR: Bump io.netty:netty-bom from 4.2.10.Final to 4.2.12.Final (#1091) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [io.netty:netty-bom](https://github.com/netty/netty) from 4.2.10.Final to 4.2.12.Final.
Release notes

Sourced from io.netty:netty-bom's releases.

netty-4.2.12.Final

What's Changed

Full Changelog: https://github.com/netty/netty/compare/netty-4.2.11.Final...netty-4.2.12.Final

netty-4.2.11.Final

Security

What's Changed

... (truncated)

Commits
  • 67ce541 [maven-release-plugin] prepare release netty-4.2.12.Final
  • 7074624 Revert "Eliminate redundant bounds checks in CompositeByteBuf accessors" (#16...
  • c3b0a43 [maven-release-plugin] prepare for next development iteration
  • c94a818 [maven-release-plugin] prepare release netty-4.2.11.Final
  • 3b76df1 Merge commit from fork
  • aae944a Auto-port 4.2: Limit the number of Continuation frames per HTTP2 Headers (#16...
  • 6001499 Eliminate redundant bounds checks in CompositeByteBuf accessors (#16525)
  • a7fbb6f JdkZlibDecoder: accumulate decompressed output before firing channelRead (#16...
  • 7937553 Enforce io.netty.maxDirectMemory accounting on all Java versions (#16489)
  • 893ea2e Allocate less in QueryStringDecoder.addParam for typical use case (#16527)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=io.netty:netty-bom&package-manager=maven&previous-version=4.2.10.Final&new-version=4.2.12.Final)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 863169b47b..61b5b3f0b2 100644 --- a/pom.xml +++ b/pom.xml @@ -98,7 +98,7 @@ under the License. 5.12.2 2.0.17 33.5.0-jre - 4.2.10.Final + 4.2.12.Final 1.79.0 4.34.1 2.21.2 From 89fa995eac2aa84e0195341bff601dfce66b942e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 10:22:13 +0200 Subject: [PATCH 197/271] MINOR: Bump com.nimbusds:oauth2-oidc-sdk from 11.34 to 11.37 (#1096) Bumps [com.nimbusds:oauth2-oidc-sdk](https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions) from 11.34 to 11.37.
Changelog

Sourced from com.nimbusds:oauth2-oidc-sdk's changelog.

version 1.0 (2012-05-29) * First official release with authorisation endpoint, token endpoint, check ID endpoint and UserInfo endpoint support. * JSON Web Tokens (JWTs) support through the Nimbus-JWT library. * Language Tags (RFC 5646) support through the Nimbus-LangTag library. * JSON support through the JSON Smart library.

version 2.0 (2013-05-13) * Intermediary development release with Maven build, published to Maven Central.

version 2.1 (2013-06-06) * Updates the APIs to OpenID Connect Messages draft 20, OpenID Connect Standard draft 21, OpenID Connect Discovery draft 17 and OpenID Connect Registration draft 19. * Major refactoring of the APIs for greater simplicity. * Adds JUnit tests.

version 2.2 (2013-06-18) * Refactors dynamic OpenID Connect client registration. * Adds partial support of the OAuth 2.0 Dynamic Client Registration Protocol (draft-ietf-oauth-dyn-reg-12). * Optimises parsing of request parameters consisting of one or more tokens (scope, response type, etc).

version 2.3 (2013-06-19) * Renames OAuth 2.0 dynamic client registration package. * Adds ClientInformation.getClientMetadata() method. * Adds OIDCClientInformation class.

version 2.4 (2013-06-20) * Adds static OIDCClientInformation.parse(JSONObject) method.

version 2.5 (2013-06-22) * Adds support OAuth 2.0 dynamic client update. * Adds OpenID Connect dynamic client registration classes.

version 2.6 (2013-06-25) * Enforces order of preference of ACR values in OpenID Connect client metadata, as required by the specification. * Documentation and performance improvements.

version 2.7 (2013-06-26) * Switches Identifier generation to java.security.SecureRandom.

version 2.8 (2013-06-30) * Fixes serialisation and assignment bugs in ClientMetadata. * Switches Secret generation to java.security.SecureRandom.

version 2.9 (2013-09-17)

... (truncated)

Commits
  • d98de1a [maven-release-plugin] prepare for next development iteration
  • 2ea716f Shortens InvalidClientException messages
  • ed5773c TokenRevocationRequest receives custom form parameters support
  • e133559 Updates tests for shortened InvalidClientException messages
  • fe43e1f [maven-release-plugin] prepare release 11.35
  • 73224c9 [maven-release-plugin] prepare for next development iteration
  • f3f7286 Adds static JSONObjectUtils.getNonNegativeLong methods
  • d6899e0 Cleans up JSONObjectUtils.getEnum(net.minidev.json.JSONObject, java.lang.Stri...
  • 9b05d23 Adds non-negative checks when parsing Date instances from Unix timestamps (is...
  • 592d8f4 Adds "acr" and "auth_time" parameter (RFC 9470) support to TokenIntrospection...
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.nimbusds:oauth2-oidc-sdk&package-manager=maven&previous-version=11.34&new-version=11.37)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index ffeff12462..483c019cdc 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -182,7 +182,7 @@ under the License. com.nimbusds oauth2-oidc-sdk - 11.34 + 11.37 From 4297733dae36ca83234a239b25068db775a09743 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 10:30:50 +0200 Subject: [PATCH 198/271] MINOR: Bump com.gradle:develocity-maven-extension from 2.3.4 to 2.4.0 (#1095) Bumps com.gradle:develocity-maven-extension from 2.3.4 to 2.4.0. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.gradle:develocity-maven-extension&package-manager=maven&previous-version=2.3.4&new-version=2.4.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .mvn/extensions.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index 4585435b49..f52dfafb4a 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -23,7 +23,7 @@ com.gradle develocity-maven-extension - 2.3.4 + 2.4.0 com.gradle From 447372ce4caa5a53387018861e3ea9d1f7be795c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 10:57:29 +0200 Subject: [PATCH 199/271] MINOR: Bump com.google.api.grpc:proto-google-common-protos from 2.66.0 to 2.67.0 (#1092) Bumps [com.google.api.grpc:proto-google-common-protos](https://github.com/googleapis/sdk-platform-java) from 2.66.0 to 2.67.0.
Release notes

Sourced from com.google.api.grpc:proto-google-common-protos's releases.

v2.67.0

2.67.0 (2026-02-18)

Features

  • observability: introduce minimal tracing implementation (#4105) (e4e5e89)

Dependencies

v2.66.1

2.66.1 (2026-02-04)

Documentation

  • [common-protos] update reference documentation for SelectionInput.DROPDOWN to include dynamic data sources and autosuggestion (9960262)
Changelog

Sourced from com.google.api.grpc:proto-google-common-protos's changelog.

2.67.0 (2026-02-18)

Features

  • observability: introduce minimal tracing implementation (#4105) (e4e5e89)

Dependencies

2.66.1 (2026-02-04)

Documentation

  • [common-protos] update reference documentation for SelectionInput.DROPDOWN to include dynamic data sources and autosuggestion (9960262)
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.api.grpc:proto-google-common-protos&package-manager=maven&previous-version=2.66.0&new-version=2.67.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml index 92490dd67b..5c6c3378c1 100644 --- a/flight/flight-core/pom.xml +++ b/flight/flight-core/pom.xml @@ -134,7 +134,7 @@ under the License. com.google.api.grpc proto-google-common-protos - 2.66.0 + 2.67.0 test From 0703021c8315a55651b6f7446da8cc781f039f72 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 14:56:30 +0200 Subject: [PATCH 200/271] MINOR: Bump org.apache:apache from 33 to 37 (#1033) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.apache:apache](https://github.com/apache/maven-apache-parent) from 33 to 37.
Release notes

Sourced from org.apache:apache's releases.

Apache Parent POM version 37

🚀 New features and improvements

Apache Parent POM version 36

:boom: Breaking changes

🚀 New features and improvements

📝 Documentation updates

👻 Maintenance

📦 Dependency updates

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache:apache&package-manager=maven&previous-version=33&new-version=37)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- bom/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index e4ccee02db..267dcef73c 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache apache - 33 + 37 diff --git a/pom.xml b/pom.xml index 61b5b3f0b2..25d49a0704 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache apache - 35 + 37 org.apache.arrow From e54681b80e84573f4cb6e7fdcb77b4247768965f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 16:20:40 +0200 Subject: [PATCH 201/271] MINOR: Bump com.diffplug.spotless:spotless-maven-plugin from 2.44.4 to 3.4.0 (#1088) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [com.diffplug.spotless:spotless-maven-plugin](https://github.com/diffplug/spotless) from 2.44.4 to 3.4.0.
Release notes

Sourced from com.diffplug.spotless:spotless-maven-plugin's releases.

Maven Plugin v3.4.0

Added

  • Add tableTest format type for standalone .table files. (#2880)

Changes

  • Bump default tabletest-formatter version 1.0.1 -> 1.1.1, now works with Java 17+. (#2880)

Lib v3.3.1

Fixed

  • GitPrePushHookInstaller didn't work on windows, now fixed. (#2562)

Lib v3.3.0

Added

  • Allow specifying path to Biome JSON config file directly in biome step. Requires biome 2.x. (#2548)
  • GitPrePushHookInstaller, a reusable library component for installing a Git pre-push hook that runs formatter checks. (#2553)
  • Allow setting Eclipse XML config from a string, not only from files (#2361)

Changed

  • Bump default gson version to latest 2.11.0 -> 2.13.1. (#2414)
  • Bump default jackson version to latest 2.18.1 -> 2.19.2. (#2558)
  • Bump default gherkin-utils version to latest 9.0.0 -> 9.2.0. (#2408)
  • Bump default cleanthat version to latest 2.22 -> 2.23. (#2556)

Maven Plugin v3.3.0

Added

  • Add tabletest-formatter support for Java and Kotlin. (#2860)

Fixed

  • Fix the ability to specify a wildcard version (*) for external formatter executables, which did not work. (#2848)
  • [fix] ConcurrentModificationException in expandWildcardImports (#2830)

Maven Plugin v3.2.1

Fixed

  • removeSemicolons() should not be applied to multiline strings in groovy #2780 (#2792)

Lib v3.2.0

Added

  • Support for idea (#2020, #2535)
  • Add support for removing wildcard imports via removeWildcardImports step. (#2517)
  • scalafmt: enforce version consistency between the version configured in Spotless and the version declared in Scalafmt config file (#2460)

Fixed

  • SortPom disable expandEmptyElements, to avoid empty body warnings. (#2520)
  • Fix biome formatter for new major release 2.x of biome (#2537)
  • Make sure npm-based formatters use the correct node_modules directory when running in parallel. (#2542)

Changed

  • Bump internal dependencies for npm-based formatters (#2542)

Maven Plugin v3.2.0

Added

  • Add the ability to specify a wildcard version (*) for external formatter executables. (#2757)

Changes

  • Dramatic (~100x) performance improvement when using git ratchetFrom. (#2805)

Fixed

... (truncated)

Commits
  • 708a1b0 Published maven/3.4.0
  • 1cc0163 Published gradle/8.4.0
  • a4cd808 Published lib/4.5.0
  • 9066bf6 Add links to the changelog.
  • db8dc1c Fix for illegal mutation issue with predeclareDeps (#2892)
  • 0eb98a9 chore: Updated gradle plugin change
  • 3f7f12e chore: Removes check for predeclare as it's not needed anymore
  • 55c0c5c fix: IsolatedProjectTest.predeclaredIsUnsupported() is now actually supported...
  • 47489af fix: avoid IllegalMutationException when root project uses predeclareDeps() w...
  • 4010e8b test: Introduce a test harnessing predeclared deps
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.diffplug.spotless:spotless-maven-plugin&package-manager=maven&previous-version=2.44.4&new-version=3.4.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: JB Onofré Co-authored-by: JB Onofré --- bom/pom.xml | 8 ++++---- flight/flight-integration-tests/pom.xml | 2 +- flight/flight-sql-jdbc-driver/pom.xml | 2 +- memory/memory-core/pom.xml | 4 ++-- performance/pom.xml | 6 +++--- pom.xml | 14 +++++++------- tools/pom.xml | 2 +- 7 files changed, 19 insertions(+), 19 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index 267dcef73c..d97cc291c2 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -24,7 +24,7 @@ under the License. org.apache apache 37 - + org.apache.arrow @@ -78,7 +78,7 @@ under the License. - + 17 17 @@ -208,7 +208,7 @@ under the License. com.diffplug.spotless spotless-maven-plugin - 2.44.4 + 3.4.0 org.codehaus.mojo @@ -235,7 +235,7 @@ under the License. ${maven.multiModuleProjectDirectory}/dev/license/asf-xml.license (<configuration|<project) - + diff --git a/flight/flight-integration-tests/pom.xml b/flight/flight-integration-tests/pom.xml index ec81162e59..f6ae8e16a5 100644 --- a/flight/flight-integration-tests/pom.xml +++ b/flight/flight-integration-tests/pom.xml @@ -101,7 +101,7 @@ under the License. - + META-INF/LICENSE.txt src/shade/LICENSE.txt diff --git a/flight/flight-sql-jdbc-driver/pom.xml b/flight/flight-sql-jdbc-driver/pom.xml index 55de7221ec..ff5763702b 100644 --- a/flight/flight-sql-jdbc-driver/pom.xml +++ b/flight/flight-sql-jdbc-driver/pom.xml @@ -138,7 +138,7 @@ under the License. - + META-INF/LICENSE.txt src/shade/LICENSE.txt diff --git a/memory/memory-core/pom.xml b/memory/memory-core/pom.xml index 1c7b6f8834..825b3dae4b 100644 --- a/memory/memory-core/pom.xml +++ b/memory/memory-core/pom.xml @@ -100,8 +100,8 @@ under the License. test - - + + **/TestOpens.java diff --git a/performance/pom.xml b/performance/pom.xml index 413d5ba1e0..d994bff45b 100644 --- a/performance/pom.xml +++ b/performance/pom.xml @@ -35,10 +35,10 @@ under the License. true .* 1 - + 5 5 - + jmh-result.json json @@ -143,7 +143,7 @@ under the License. java -classpath - + org.openjdk.jmh.Main ${benchmark.filter} -f diff --git a/pom.xml b/pom.xml index 25d49a0704..06cb916b45 100644 --- a/pom.xml +++ b/pom.xml @@ -107,7 +107,7 @@ under the License. 1.12.1 1.17.0 5.23.0 - + 2 10.23.0 true @@ -374,7 +374,7 @@ under the License. - + @@ -387,7 +387,7 @@ under the License. - + @@ -400,7 +400,7 @@ under the License. - + @@ -413,7 +413,7 @@ under the License. - + @@ -492,7 +492,7 @@ under the License. com.diffplug.spotless spotless-maven-plugin - 2.44.4 + 3.4.0 org.codehaus.mojo @@ -734,7 +734,7 @@ under the License. ${maven.multiModuleProjectDirectory}/dev/license/asf-xml.license (<configuration|<project) - + diff --git a/tools/pom.xml b/tools/pom.xml index 64634b9abe..b4d64fd435 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -126,7 +126,7 @@ under the License. - + META-INF/LICENSE.txt src/shade/LICENSE.txt From d8ad7b6765620c4f3f3078ee87c3d2cfb7423945 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2026 17:41:35 +0900 Subject: [PATCH 202/271] MINOR: [CI] Bump docker/login-action from 4.0.0 to 4.1.0 (#1103) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [docker/login-action](https://github.com/docker/login-action) from 4.0.0 to 4.1.0.
Release notes

Sourced from docker/login-action's releases.

v4.1.0

Full Changelog: https://github.com/docker/login-action/compare/v4.0.0...v4.1.0

Commits
  • 4907a6d Merge pull request #930 from docker/dependabot/npm_and_yarn/aws-sdk-dependenc...
  • 1e233e6 chore: update generated content
  • 6c24ead build(deps): bump the aws-sdk-dependencies group with 2 updates
  • ee034d7 Merge pull request #958 from docker/dependabot/npm_and_yarn/lodash-4.18.1
  • 1527209 Merge pull request #937 from docker/dependabot/npm_and_yarn/proxy-agent-depen...
  • d39362a build(deps): bump lodash from 4.17.23 to 4.18.1
  • a6f092b chore: update generated content
  • 60953f0 build(deps): bump the proxy-agent-dependencies group with 2 updates
  • 62c6885 Merge pull request #936 from docker/dependabot/npm_and_yarn/docker/actions-to...
  • 102c0e6 chore: update generated content
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=docker/login-action&package-manager=github_actions&previous-version=4.0.0&new-version=4.1.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 9c75b8c807..8f52d54bd6 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -127,7 +127,7 @@ jobs: with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing - - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: registry: ghcr.io username: ${{ github.actor }} From 13c8b9353b80f553e720ff50fce0cb103de87c55 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2026 14:23:37 +0200 Subject: [PATCH 203/271] MINOR: Bump checker.framework.version from 3.54.0 to 3.55.1 (#1105) Bumps `checker.framework.version` from 3.54.0 to 3.55.1. Updates `org.checkerframework:checker-qual` from 3.54.0 to 3.55.1
Release notes

Sourced from org.checkerframework:checker-qual's releases.

Checker Framework 3.55.1

Version 3.55.1 (2026-04-03)

No user-visible changes.

Checker Framework 3.54.1

Version 3.55.0 (2026-04-02)

User-visible changes

The Checker Framework runs under JDK 26 -- that is, it runs on a version 26 JVM.

Removed deprecated command-line option -AskipDirs; use -AskipFiles.

Implementation details

In AnnotatedTypeMirror:

  • Renamed getEffectiveAnnotation*() to getAnnotation*().
  • Renamed hasEffectiveAnnotation*() to hasAnnotation*().

Removed deprecated method ObjectCreationNode.getConstructor(); use getTypeToInstantiate().

Closed issues

#7079, #7489, #7539.

Changelog

Sourced from org.checkerframework:checker-qual's changelog.

Version 3.55.1 (2026-04-03)

No user-visible changes.

Version 3.55.0 (2026-04-02)

User-visible changes

The Checker Framework runs under JDK 26 -- that is, it runs on a version 26 JVM.

Removed deprecated command-line option -AskipDirs; use -AskipFiles.

Implementation details

In AnnotatedTypeMirror:

  • Renamed getEffectiveAnnotation*() to getAnnotation*().
  • Renamed hasEffectiveAnnotation*() to hasAnnotation*().

Removed deprecated method ObjectCreationNode.getConstructor(); use getTypeToInstantiate().

Closed issues

#7079, #7489, #7539.

Commits

Updates `org.checkerframework:checker` from 3.54.0 to 3.55.1
Release notes

Sourced from org.checkerframework:checker's releases.

Checker Framework 3.55.1

Version 3.55.1 (2026-04-03)

No user-visible changes.

Checker Framework 3.54.1

Version 3.55.0 (2026-04-02)

User-visible changes

The Checker Framework runs under JDK 26 -- that is, it runs on a version 26 JVM.

Removed deprecated command-line option -AskipDirs; use -AskipFiles.

Implementation details

In AnnotatedTypeMirror:

  • Renamed getEffectiveAnnotation*() to getAnnotation*().
  • Renamed hasEffectiveAnnotation*() to hasAnnotation*().

Removed deprecated method ObjectCreationNode.getConstructor(); use getTypeToInstantiate().

Closed issues

#7079, #7489, #7539.

Changelog

Sourced from org.checkerframework:checker's changelog.

Version 3.55.1 (2026-04-03)

No user-visible changes.

Version 3.55.0 (2026-04-02)

User-visible changes

The Checker Framework runs under JDK 26 -- that is, it runs on a version 26 JVM.

Removed deprecated command-line option -AskipDirs; use -AskipFiles.

Implementation details

In AnnotatedTypeMirror:

  • Renamed getEffectiveAnnotation*() to getAnnotation*().
  • Renamed hasEffectiveAnnotation*() to hasAnnotation*().

Removed deprecated method ObjectCreationNode.getConstructor(); use getTypeToInstantiate().

Closed issues

#7079, #7489, #7539.

Commits

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 06cb916b45..363ad7bd7d 100644 --- a/pom.xml +++ b/pom.xml @@ -112,7 +112,7 @@ under the License. 10.23.0 true 2.42.0 - 3.54.0 + 3.55.1 1.5.32 none -Xdoclint:none From d952ed48ef257b90a2085bca574ddd9213b67c91 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2026 14:24:24 +0200 Subject: [PATCH 204/271] MINOR: Bump dep.hadoop.version from 3.4.3 to 3.5.0 (#1104) Bumps `dep.hadoop.version` from 3.4.3 to 3.5.0. Updates `org.apache.hadoop:hadoop-client-runtime` from 3.4.3 to 3.5.0 Updates `org.apache.hadoop:hadoop-client-api` from 3.4.3 to 3.5.0 Updates `org.apache.hadoop:hadoop-common` from 3.4.3 to 3.5.0 Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 363ad7bd7d..5ef8364176 100644 --- a/pom.xml +++ b/pom.xml @@ -102,7 +102,7 @@ under the License. 1.79.0 4.34.1 2.21.2 - 3.4.3 + 3.5.0 25.2.10 1.12.1 1.17.0 From e4f8c9251229846190716cce5a9772292952c761 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 10 Apr 2026 14:31:54 +0900 Subject: [PATCH 205/271] GH-1107: Increase publish job timeout (#1108) ## What's Changed We have many artifacts and need `sleep 1` per upload. So 5min timeout is too short. Closes #1107. --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5ca9cf9b73..a2c5a55544 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,7 +32,7 @@ jobs: publish: name: Publish runs-on: ubuntu-latest - timeout-minutes: 5 + timeout-minutes: 30 steps: - name: Download RC contents run: | From 4ca6017bdd8590a6ad0a0a4154d993ec0c416076 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2026 17:37:40 +0900 Subject: [PATCH 206/271] MINOR: [CI] Bump actions/github-script from 8 to 9 (#1110) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/github-script](https://github.com/actions/github-script) from 8 to 9.
Release notes

Sourced from actions/github-script's releases.

v9.0.0

New features:

  • getOctokit factory function — Available directly in the script context. Create additional authenticated Octokit clients with different tokens for multi-token workflows, GitHub App tokens, and cross-org access. See Creating additional clients with getOctokit for details and examples.
  • Orchestration ID in user-agent — The ACTIONS_ORCHESTRATION_ID environment variable is automatically appended to the user-agent string for request tracing.

Breaking changes:

  • require('@actions/github') no longer works in scripts. The upgrade to @actions/github v9 (ESM-only) means require('@actions/github') will fail at runtime. If you previously used patterns like const { getOctokit } = require('@actions/github') to create secondary clients, use the new injected getOctokit function instead — it's available directly in the script context with no imports needed.
  • getOctokit is now an injected function parameter. Scripts that declare const getOctokit = ... or let getOctokit = ... will get a SyntaxError because JavaScript does not allow const/let redeclaration of function parameters. Use the injected getOctokit directly, or use var getOctokit = ... if you need to redeclare it.
  • If your script accesses other @actions/github internals beyond the standard github/octokit client, you may need to update those references for v9 compatibility.

What's Changed

New Contributors

Full Changelog: https://github.com/actions/github-script/compare/v8.0.0...v9.0.0

Commits
  • 3a2844b Merge pull request #700 from actions/salmanmkc/expose-getoctokit + prepare re...
  • ca10bbd fix: use @​octokit/core/types import for v7 compatibility
  • 86e48e2 merge: incorporate main branch changes
  • c108472 chore: rebuild dist for v9 upgrade and getOctokit factory
  • afff112 Merge pull request #712 from actions/salmanmkc/deployment-false + fix user-ag...
  • ff8117e ci: fix user-agent test to handle orchestration ID
  • 81c6b78 ci: use deployment: false to suppress deployment noise from integration tests
  • 3953caf docs: update README examples from @​v8 to @​v9, add getOctokit docs and v9 brea...
  • c17d55b ci: add getOctokit integration test job
  • a047196 test: add getOctokit integration tests via callAsyncFunction
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/github-script&package-manager=github_actions&previous-version=8&new-version=9)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/comment_bot.yml | 2 +- .github/workflows/dev_pr.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml index b4dbc92dfb..507d6a969c 100644 --- a/.github/workflows/comment_bot.yml +++ b/.github/workflows/comment_bot.yml @@ -30,7 +30,7 @@ jobs: if: github.event.comment.body == 'take' runs-on: ubuntu-latest steps: - - uses: actions/github-script@v8 + - uses: actions/github-script@v9 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: |- diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml index 84740628ee..20946c8185 100644 --- a/.github/workflows/dev_pr.yml +++ b/.github/workflows/dev_pr.yml @@ -50,28 +50,28 @@ jobs: - name: Ensure PR title format id: title-format - uses: actions/github-script@v8 + uses: actions/github-script@v9 with: script: | const scripts = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr.js`); return scripts.check_title_format({core, github, context}); - name: Label PR - uses: actions/github-script@v8 + uses: actions/github-script@v9 with: script: | const scripts = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr.js`); await scripts.apply_labels({core, github, context}); - name: Ensure PR is labeled - uses: actions/github-script@v8 + uses: actions/github-script@v9 with: script: | const scripts = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr.js`); await scripts.check_labels({core, github, context}); - name: Ensure PR is linked to an issue - uses: actions/github-script@v8 + uses: actions/github-script@v9 with: script: | const scripts = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr.js`); From 0d55ba78aeed3e6b28e3d65ced37c360f5e0ed4e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2026 17:37:53 +0900 Subject: [PATCH 207/271] MINOR: [CI] Bump actions/upload-artifact from 7.0.0 to 7.0.1 (#1111) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 7.0.0 to 7.0.1.
Release notes

Sourced from actions/upload-artifact's releases.

v7.0.1

What's Changed

Full Changelog: https://github.com/actions/upload-artifact/compare/v7...v7.0.1

Commits
  • 043fb46 Merge pull request #797 from actions/yacaovsnc/update-dependency
  • 634250c Include changes in typespec/ts-http-runtime 0.3.5
  • e454baa Readme: bump all the example versions to v7 (#796)
  • 74fad66 Update the readme with direct upload details (#795)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/upload-artifact&package-manager=github_actions&previous-version=7.0.0&new-version=7.0.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 8f52d54bd6..2658f1da00 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -71,7 +71,7 @@ jobs: run: | dev/release/run_rat.sh "${TAR_GZ}" - name: Upload source archive - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: release-source path: | @@ -148,7 +148,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf jni-linux-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: jni-linux-${{ matrix.platform.arch }} path: jni-linux-${{ matrix.platform.arch }}.tar.gz @@ -278,7 +278,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf jni-macos-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: jni-macos-${{ matrix.platform.arch }} path: jni-macos-${{ matrix.platform.arch }}.tar.gz @@ -356,7 +356,7 @@ jobs: shell: bash run: tar -cvzf jni-windows-${{ matrix.platform.arch }}.tar.gz jni/ - name: Upload artifacts - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: jni-windows-${{ matrix.platform.arch }} path: jni-windows-${{ matrix.platform.arch }}.tar.gz @@ -428,12 +428,12 @@ jobs: cp -a target/site/apidocs reference tar -cvzf reference.tar.gz reference - name: Upload binaries - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: release-binaries path: binaries/* - name: Upload docs - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: reference path: reference.tar.gz @@ -471,7 +471,7 @@ jobs: - name: Compress into single artifact to keep directory structure run: tar -cvzf html.tar.gz -C docs/build html - name: Upload artifacts - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: release-html path: html.tar.gz From 8d3e8dd2f01f7ddc22cda5aa04cde30eb5f6d308 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Apr 2026 15:18:55 +0200 Subject: [PATCH 208/271] MINOR: Bump com.google.api.grpc:proto-google-common-protos from 2.67.0 to 2.70.0 (#1124) Bumps [com.google.api.grpc:proto-google-common-protos](https://github.com/googleapis/sdk-platform-java) from 2.67.0 to 2.70.0.
Release notes

Sourced from com.google.api.grpc:proto-google-common-protos's releases.

v2.68.0

2.68.0 (2026-03-17)

Features

  • Add client request duration metric. (#4132) (6a76397)
  • Add more attributes to golden signals metrics. (#4135) (59d0624)
  • gax-httpjson: add HttpJsonErrorParser utility (#4137) (a1b7565)
  • generator: add extra allowed modules that will not be removed from the monorepo if they are present (#4124) (774fe6e)
  • o11y: introduce gcp.client.repo and gcp.client.artifact attributes (#4120) (105f644)
  • o11y: Introduce rpc.system.name and rpc.method in gRPC (#4121) (7ab6d2e)
  • o11y: introduce server.port attribute (#4128) (56aa343)

Bug Fixes

  • add null checks for ApiTracerFactory in ClientContext (#4122) (4b3dbe2)
  • Decrease log level for directpath warnings outside GCE (#4139) (c9651e7)
  • gax-grpc: add pick_first fallback to direct path service config (#4143) (b150fe9)
  • Populate method level attributes in metrics recording (#4149) (7b7e6c9)
  • suppress warnings in generated projects for non-idiomatic durations (#4119) (4206e6e)
  • Use ServiceName + MethodName as the regex for Otel (#2543) (b9ae73f)

Documentation

  • hermetic_build: fix config field name in readme (#4130) (a0c8f67)
Changelog

Sourced from com.google.api.grpc:proto-google-common-protos's changelog.

Changelog

2.68.0 (2026-03-17)

Features

  • Add client request duration metric. (#4132) (6a76397)
  • Add more attributes to golden signals metrics. (#4135) (59d0624)
  • gax-httpjson: add HttpJsonErrorParser utility (#4137) (a1b7565)
  • generator: add extra allowed modules that will not be removed from the monorepo if they are present (#4124) (774fe6e)
  • o11y: introduce gcp.client.repo and gcp.client.artifact attributes (#4120) (105f644)
  • o11y: Introduce rpc.system.name and rpc.method in gRPC (#4121) (7ab6d2e)
  • o11y: introduce server.port attribute (#4128) (56aa343)

Bug Fixes

  • add null checks for ApiTracerFactory in ClientContext (#4122) (4b3dbe2)
  • Decrease log level for directpath warnings outside GCE (#4139) (c9651e7)
  • gax-grpc: add pick_first fallback to direct path service config (#4143) (b150fe9)
  • Populate method level attributes in metrics recording (#4149) (7b7e6c9)
  • suppress warnings in generated projects for non-idiomatic durations (#4119) (4206e6e)
  • Use ServiceName + MethodName as the regex for Otel (#2543) (b9ae73f)

Documentation

  • hermetic_build: fix config field name in readme (#4130) (a0c8f67)
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.api.grpc:proto-google-common-protos&package-manager=maven&previous-version=2.67.0&new-version=2.70.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml index 5c6c3378c1..d199f70da4 100644 --- a/flight/flight-core/pom.xml +++ b/flight/flight-core/pom.xml @@ -134,7 +134,7 @@ under the License. com.google.api.grpc proto-google-common-protos - 2.67.0 + 2.70.0 test From 3dca92baf2a806802e437f84bfe06947d4433343 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Apr 2026 15:46:36 +0200 Subject: [PATCH 209/271] MINOR: Bump com.google.guava:guava-bom from 33.5.0-jre to 33.6.0-jre (#1123) Bumps [com.google.guava:guava-bom](https://github.com/google/guava) from 33.5.0-jre to 33.6.0-jre.
Release notes

Sourced from com.google.guava:guava-bom's releases.

33.6.0

Maven

<dependency>
  <groupId>com.google.guava</groupId>
  <artifactId>guava</artifactId>
  <version>33.6.0-jre</version>
  <!-- or, for Android: -->
  <version>33.6.0-android</version>
</dependency>

Jar files

Guava requires one runtime dependency, which you can download here:

Javadoc

JDiff

Changelog

  • Migrated some classes from finalize() to PhantomReference in preparation for the removal of finalization. (786b619dd6, 7c6b17c, aeef90988d)
  • cache: Deprecated CacheBuilder APIs that use TimeUnit in favor of those that use Duration. (73f8b0bb84)
  • collect: Added toImmutableSortedMap collectors that use the natural comparator. (64d70b9f94)
  • collect: Changed ConcurrentHashMultiset, ImmutableMap and TreeMultiset deserialization to avoid mutating final fields. In extremely unlikely scenarios in which an instance of that type contains an object that refers back to that instance, this could lead to a broken instance that throws NullPointerException when used. (8240c7e596, 046468055f)
  • graph: Removed @Beta from all APIs in the package. (dae9566b73)
  • graph: Added support to Graphs.transitiveClosure() for different strategies for adding self-loops. (2e13df25b2)
  • graph: Added an asNetwork() view to Graph and ValueGraph. (909c593c61)
  • hash: Added BloomFilter.serializedSize(). (df9bcc251a)
  • net: Added HttpHeaders.CDN_CACHE_CONTROL. (75331b5030)
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.guava:guava-bom&package-manager=maven&previous-version=33.5.0-jre&new-version=33.6.0-jre)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5ef8364176..a388c90fee 100644 --- a/pom.xml +++ b/pom.xml @@ -97,7 +97,7 @@ under the License. 1.9.0 5.12.2 2.0.17 - 33.5.0-jre + 33.6.0-jre 4.2.12.Final 1.79.0 4.34.1 From 980b5146b5c659e5a06c4592b59cef0f2514ea72 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Apr 2026 15:57:26 +0200 Subject: [PATCH 210/271] MINOR: Bump org.bouncycastle:bcpkix-jdk18on from 1.83 to 1.84 (#1122) Bumps [org.bouncycastle:bcpkix-jdk18on](https://github.com/bcgit/bc-java) from 1.83 to 1.84.
Changelog

Sourced from org.bouncycastle:bcpkix-jdk18on's changelog.

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.bouncycastle:bcpkix-jdk18on&package-manager=maven&previous-version=1.83&new-version=1.84)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 483c019cdc..1f9bc3e00e 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -165,7 +165,7 @@ under the License. org.bouncycastle bcpkix-jdk18on - 1.83 + 1.84 From 09038fbb578d38e98878359557b1e13dcb96b04f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 06:15:04 +0200 Subject: [PATCH 211/271] MINOR: Bump checker.framework.version from 3.55.1 to 4.0.0 (#1113) Bumps `checker.framework.version` from 3.55.1 to 4.0.0. Updates `org.checkerframework:checker-qual` from 3.55.1 to 4.0.0
Release notes

Sourced from org.checkerframework:checker-qual's releases.

Release 4.0.0 of the Checker Framework

Version 4.0.0 (2026-04-07)

User-visible changes

To run the Checker Framework, you need to use a JDK 17 or later version of javac. That is, you need to use JDK 17 or later when compiling your code.

The Checker Framework can type-check any Java project, including projects that compile to Java 8 or 11 bytecodes and run on JRE versions 8 or 11. That is, your code can run under any release of Java, from Java 8 onward.

The type qualifiers and utility libraries in checker-qual.jar and checker-util.jar still use Java 11 bytecode. Thus, they may be used in projects that run under Java 11 or later.

Changes since version 3.0.0

Since version 3.0.0, 91 authors have made over 4500 commits and closed over 600 issues. Thanks to everyone who contributed!

New checkers include:

  • The Index Checker warns about out-of-bounds accesses to arrays and strings.
  • The Initialized Fields Checker warns if a constructor does not initialize a field.
  • The Resource Leak Checker guarantees that every resource is closed rather than leaked. Examples of resources are a channel, executor, ExecutionControl, file, FileLock, Formatter, reader, Scanner, socket, stream, writer, etc.
  • The SQL Quotes Checker helps prevent SQL injection vulnerabilities.

New command-line arguments include:

  • -AskipFiles, -AonlyFiles
  • -AassumeSideEffectFree, -AassumeDeterministic, -AassumePure, -AassumePureGetters
  • -AuseConservativeDefaultsForUncheckedCode
  • -AignoreRawTypeArguments
  • -AwarnRedundantAnnotations
  • -Ainfer=ajava, -AinferOutputDirectory, -AinferOutputOriginal, -AshowWpiFailedInferences
  • -AshowSuppressWarningsStrings, -AwarnUnneededSuppressionsExceptions
  • -AshowPrefixInWarningMessages
  • -AstubNoWarnIfNotFound, -AstubWarnNote, -AmergeStubsWithSource
  • -Aonelinemsg, -AdumpOnErrors, -AexceptionLineSeparator
  • -ApermitMissingJdk, -AparseAllJdk
  • -AslowTypecheckingSeconds
  • -Aversion, -AprintGitProperties
  • You can pass an option to only a particular checker (not all checkers) by using an underscore prefix.

Other improvements include thousands of enhancements and bug fixes -- too many to list here.

Implementation details

All previously-deprecated methods and classes have been removed. If your project builds upon the Checker Framework, we suggest that you upgrade to version 3.55.1, resolve all the deprecation warnings, then upgrade to version 4.0.0.

Changelog

Sourced from org.checkerframework:checker-qual's changelog.

Version 4.0.0 (2026-04-07)

User-visible changes

To run the Checker Framework, you need to use a JDK 17 or later version of javac. That is, you need to use JDK 17 or later when compiling your code.

The Checker Framework can type-check any Java project, including projects that compile to Java 8 or 11 bytecodes and run on JRE versions 8 or 11. That is, your code can run under any release of Java, from Java 8 onward.

The type qualifiers and utility libraries in checker-qual.jar and checker-util.jar still use Java 11 bytecode. Thus, they may be used in projects that run under Java 11 or later.

Changes since version 3.0.0

Since version 3.0.0, 91 authors have made over 4500 commits and closed over 600 issues. Thanks to everyone who contributed!

New checkers include:

  • The Index Checker warns about out-of-bounds accesses to arrays and strings.
  • The Initialized Fields Checker warns if a constructor does not initialize a field.
  • The Resource Leak Checker guarantees that every resource is closed rather than leaked. Examples of resources are a channel, executor, ExecutionControl, file, FileLock, Formatter, reader, Scanner, socket, stream, writer, etc.
  • The SQL Quotes Checker helps prevent SQL injection vulnerabilities.

New command-line arguments include:

  • -AskipFiles, -AonlyFiles
  • -AassumeSideEffectFree, -AassumeDeterministic, -AassumePure, -AassumePureGetters
  • -AuseConservativeDefaultsForUncheckedCode
  • -AignoreRawTypeArguments
  • -AwarnRedundantAnnotations
  • -Ainfer=ajava, -AinferOutputDirectory, -AinferOutputOriginal, -AshowWpiFailedInferences
  • -AshowSuppressWarningsStrings, -AwarnUnneededSuppressionsExceptions
  • -AshowPrefixInWarningMessages
  • -AstubNoWarnIfNotFound, -AstubWarnNote, -AmergeStubsWithSource
  • -Aonelinemsg, -AdumpOnErrors, -AexceptionLineSeparator
  • -ApermitMissingJdk, -AparseAllJdk
  • -AslowTypecheckingSeconds

... (truncated)

Commits
  • 479d087 new release 4.0.0
  • bfff757 Put the manual in the right place.
  • c532f6d Put a copy of manual.pdf at top level of website as expected.
  • 5e53e6c No closed issues.
  • e67ae85 Prep for release.
  • 4192d0d Remove file SKIP-REQUIRE-JAVADOC
  • 7d6d856 Remove or update references to JDK 8-16
  • b1e3761 Remove all deprecated methods
  • a1b3064 Directly use Java 17 and below Javac APIs. (#7582)
  • 4efdbdb Remove support for Java 8 from scripts and build scripts. (#7575)
  • Additional commits viewable in compare view

Updates `org.checkerframework:checker` from 3.55.1 to 4.0.0
Release notes

Sourced from org.checkerframework:checker's releases.

Release 4.0.0 of the Checker Framework

Version 4.0.0 (2026-04-07)

User-visible changes

To run the Checker Framework, you need to use a JDK 17 or later version of javac. That is, you need to use JDK 17 or later when compiling your code.

The Checker Framework can type-check any Java project, including projects that compile to Java 8 or 11 bytecodes and run on JRE versions 8 or 11. That is, your code can run under any release of Java, from Java 8 onward.

The type qualifiers and utility libraries in checker-qual.jar and checker-util.jar still use Java 11 bytecode. Thus, they may be used in projects that run under Java 11 or later.

Changes since version 3.0.0

Since version 3.0.0, 91 authors have made over 4500 commits and closed over 600 issues. Thanks to everyone who contributed!

New checkers include:

  • The Index Checker warns about out-of-bounds accesses to arrays and strings.
  • The Initialized Fields Checker warns if a constructor does not initialize a field.
  • The Resource Leak Checker guarantees that every resource is closed rather than leaked. Examples of resources are a channel, executor, ExecutionControl, file, FileLock, Formatter, reader, Scanner, socket, stream, writer, etc.
  • The SQL Quotes Checker helps prevent SQL injection vulnerabilities.

New command-line arguments include:

  • -AskipFiles, -AonlyFiles
  • -AassumeSideEffectFree, -AassumeDeterministic, -AassumePure, -AassumePureGetters
  • -AuseConservativeDefaultsForUncheckedCode
  • -AignoreRawTypeArguments
  • -AwarnRedundantAnnotations
  • -Ainfer=ajava, -AinferOutputDirectory, -AinferOutputOriginal, -AshowWpiFailedInferences
  • -AshowSuppressWarningsStrings, -AwarnUnneededSuppressionsExceptions
  • -AshowPrefixInWarningMessages
  • -AstubNoWarnIfNotFound, -AstubWarnNote, -AmergeStubsWithSource
  • -Aonelinemsg, -AdumpOnErrors, -AexceptionLineSeparator
  • -ApermitMissingJdk, -AparseAllJdk
  • -AslowTypecheckingSeconds
  • -Aversion, -AprintGitProperties
  • You can pass an option to only a particular checker (not all checkers) by using an underscore prefix.

Other improvements include thousands of enhancements and bug fixes -- too many to list here.

Implementation details

All previously-deprecated methods and classes have been removed. If your project builds upon the Checker Framework, we suggest that you upgrade to version 3.55.1, resolve all the deprecation warnings, then upgrade to version 4.0.0.

Changelog

Sourced from org.checkerframework:checker's changelog.

Version 4.0.0 (2026-04-07)

User-visible changes

To run the Checker Framework, you need to use a JDK 17 or later version of javac. That is, you need to use JDK 17 or later when compiling your code.

The Checker Framework can type-check any Java project, including projects that compile to Java 8 or 11 bytecodes and run on JRE versions 8 or 11. That is, your code can run under any release of Java, from Java 8 onward.

The type qualifiers and utility libraries in checker-qual.jar and checker-util.jar still use Java 11 bytecode. Thus, they may be used in projects that run under Java 11 or later.

Changes since version 3.0.0

Since version 3.0.0, 91 authors have made over 4500 commits and closed over 600 issues. Thanks to everyone who contributed!

New checkers include:

  • The Index Checker warns about out-of-bounds accesses to arrays and strings.
  • The Initialized Fields Checker warns if a constructor does not initialize a field.
  • The Resource Leak Checker guarantees that every resource is closed rather than leaked. Examples of resources are a channel, executor, ExecutionControl, file, FileLock, Formatter, reader, Scanner, socket, stream, writer, etc.
  • The SQL Quotes Checker helps prevent SQL injection vulnerabilities.

New command-line arguments include:

  • -AskipFiles, -AonlyFiles
  • -AassumeSideEffectFree, -AassumeDeterministic, -AassumePure, -AassumePureGetters
  • -AuseConservativeDefaultsForUncheckedCode
  • -AignoreRawTypeArguments
  • -AwarnRedundantAnnotations
  • -Ainfer=ajava, -AinferOutputDirectory, -AinferOutputOriginal, -AshowWpiFailedInferences
  • -AshowSuppressWarningsStrings, -AwarnUnneededSuppressionsExceptions
  • -AshowPrefixInWarningMessages
  • -AstubNoWarnIfNotFound, -AstubWarnNote, -AmergeStubsWithSource
  • -Aonelinemsg, -AdumpOnErrors, -AexceptionLineSeparator
  • -ApermitMissingJdk, -AparseAllJdk
  • -AslowTypecheckingSeconds

... (truncated)

Commits
  • 479d087 new release 4.0.0
  • bfff757 Put the manual in the right place.
  • c532f6d Put a copy of manual.pdf at top level of website as expected.
  • 5e53e6c No closed issues.
  • e67ae85 Prep for release.
  • 4192d0d Remove file SKIP-REQUIRE-JAVADOC
  • 7d6d856 Remove or update references to JDK 8-16
  • b1e3761 Remove all deprecated methods
  • a1b3064 Directly use Java 17 and below Javac APIs. (#7582)
  • 4efdbdb Remove support for Java 8 from scripts and build scripts. (#7575)
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a388c90fee..0c5a00ded4 100644 --- a/pom.xml +++ b/pom.xml @@ -112,7 +112,7 @@ under the License. 10.23.0 true 2.42.0 - 3.55.1 + 4.0.0 1.5.32 none -Xdoclint:none From 0f0a58433b46004ff9869a2512beded177340bca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 06:17:22 +0200 Subject: [PATCH 212/271] MINOR: Bump io.grpc:grpc-bom from 1.79.0 to 1.80.0 (#1093) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [io.grpc:grpc-bom](https://github.com/grpc/grpc-java) from 1.79.0 to 1.80.0.
Release notes

Sourced from io.grpc:grpc-bom's releases.

V1.80.0

API Changes

  • core: Added PickResult.copyWithSubchannel() and PickResult.copyWithStreamTracerFactory() to simplify updating PickResult while preserving metadata. Load balancing policies should now ensure ForwardingSubchannel decorators are unwrapped before being returned in a pick result. (#12658) (eae16b251)

Bug Fixes

  • core: Fixed the retry backoff jitter range to [0.8, 1.2] to align with the gRPC A6 specification. Retries will now occur more consistently around the calculated backoff interval. (#12639) (024fdd0ea) core: Fixed a race condition in RetriableStream where inFlightSubStreams counting could become inconsistent during concurrent retry and deadline events. This ensures that client calls (such as blockingUnaryCall) do not hang indefinitely and correctly receive a close signal. (#12649) (73abb4854)

Improvements

  • api: Trigger R8's ServiceLoader optimization to reduce necessary configuration when using R8 Full Mode (470219f9c). This allows gRPC to avoid reflection, and the need to specify -keeps for various class’s constructors. Upgrade to protobuf 33.4 (#12615) (50c18f183)
  • cronet: Introduced CRONET_READ_BUFFER_SIZE_KEY to allow customizing the read buffer size per-stream via CallOptions. Increasing the buffer size from the 4KB default can significantly improve performance for large messages by reducing JNI and context-switching overhead. (31fdb6c22)
  • api: Moved FlagResetRule to api/testFixtures and updated ManagedChannelRegistry to honor the GRPC_ENABLE_RFC3986_URIS feature flag. This ensures that target parsing is consistent across the library when the new URI parser is enabled. (#12608)
  • api: Updated NameResolverRegistry to natively support io.grpc.Uri. This is a foundational change that allows gRPC's name resolution system to handle URIs parsed with the new RFC 3986-compliant parser, ensuring more robust target handling. (#12609) (990348876)
  • xds: Removed the GRPC_EXPERIMENTAL_XDS_SNI feature flag. SNI determination via xDS is now always enabled and follows gRFC A101, where SNI is derived from xDS configurations like auto_host_sni or UpstreamTlsContext.sni. This ensures that no SNI is sent if not explicitly configured, unless the legacy channel authority fallback is enabled. (#12625) (ac44e9681)

New Features

  • core: pick_first shuffling now a weighted shuffle and observes weights from EDS (34dd29042). This finishes the gRFC A113 pick_first: Weighted Random Shuffling support
  • netty: Added RFC 3986 support to the unix: name resolver. This enables proper parsing of Unix domain socket URIs, including correct handling of query and fragment components in both hierarchical (e.g., unix:///path) and opaque (e.g., unix:/path) formats. (#12659)

Thanks to

Commits
  • 6c231b4 Bump version to 1.80.0
  • daf7a6c Update README etc to reference 1.80.0
  • b7f9074 Revert "fix(xds): Allow and normalize trailing dot (FQDN) in matchHostName (#...
  • 09a6e2e Revert "netty: Preserve early server handshake failure cause in logs"
  • 31fdb6c Add CRONET_READ_BUFFER_SIZE_KEY API to CronetClientStream
  • 470219f Trigger R8's ServiceLoader optimization
  • 50ead96 netty: Preserve early server handshake failure cause in logs
  • eae16b2 unwrap ForwardingSubchannel during Picks (#12658)
  • d9320ee netty: Add RFC 3986 support to the 'unix:' name resolver.
  • d5536b3 netty: factor out some duplicated code into a helper method
  • Additional commits viewable in compare view

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0c5a00ded4..738aea03b4 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,7 @@ under the License. 2.0.17 33.6.0-jre 4.2.12.Final - 1.79.0 + 1.80.0 4.34.1 2.21.2 3.5.0 From e5482cd63d2043c0e418f1adc7bcd1e5192c5078 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Sun, 10 May 2026 08:15:56 +0200 Subject: [PATCH 213/271] MINOR: Fix Gandiva JNI build against Arrow C++ 24.0.0 (#1140) Arrow C++ 24.0.0 introduced two breaking changes for the JNI build: 1. `arrow::decimal()` was removed; replaced with `arrow::decimal128()` in the Gandiva JNI source. 2. xsimd >= 14.0.0 is now a required dependency. The Docker image's vcpkg registry only ships xsimd 13.2.0, so the Linux JNI CMake configuration was failing. Fixed by passing `xsimd_SOURCE=BUNDLED` to the Arrow C++ CMake configure step so Arrow downloads and uses xsimd 14.0.0 directly, and by passing the vcpkg toolchain file to the Arrow C++ configure step so other vcpkg-managed dependencies are still resolved correctly. --- .env | 2 +- compose.yaml | 3 ++- gandiva/src/main/cpp/jni_common.cc | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.env b/.env index 2bd7255476..51daa0406c 100644 --- a/.env +++ b/.env @@ -53,4 +53,4 @@ MAVEN=3.9.9 # Versions for various dependencies used to build artifacts # Keep in sync with apache/arrow ARROW_REPO_ROOT=./arrow -VCPKG="4334d8b4c8916018600212ab4dd4bbdc343065d1" # 2025.09.17 Release +VCPKG="66c0373dc7fca549e5803087b9487edfe3aca0a1" # 2026.01.16 Release diff --git a/compose.yaml b/compose.yaml index fb290b22fe..4fd825e5a5 100644 --- a/compose.yaml +++ b/compose.yaml @@ -109,5 +109,6 @@ services: ARROW_JAVA_CDATA: "ON" CCACHE_DIR: "/ccache" command: - ["git config --global --add safe.directory /arrow-java && \ + ["/bin/bash", "-c", + "git config --global --add safe.directory /arrow-java && /arrow-java/ci/scripts/jni_manylinux_build.sh /arrow-java /arrow /build/java /arrow-java/jni"] diff --git a/gandiva/src/main/cpp/jni_common.cc b/gandiva/src/main/cpp/jni_common.cc index 2851250072..ec4888a512 100644 --- a/gandiva/src/main/cpp/jni_common.cc +++ b/gandiva/src/main/cpp/jni_common.cc @@ -221,7 +221,7 @@ DataTypePtr ProtoTypeToDataType(const gandiva::types::ExtGandivaType& ext_type) return arrow::date64(); case gandiva::types::DECIMAL: // TODO: error handling - return arrow::decimal(ext_type.precision(), ext_type.scale()); + return arrow::decimal128(ext_type.precision(), ext_type.scale()); case gandiva::types::TIME32: return ProtoTypeToTime32(ext_type); case gandiva::types::TIME64: From 138e8521b17a2b83814b8714c7d8e0f94fe13656 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 May 2026 05:20:41 +0200 Subject: [PATCH 214/271] MINOR: Bump com.fasterxml.jackson:jackson-bom from 2.21.2 to 2.21.3 (#1133) Bumps [com.fasterxml.jackson:jackson-bom](https://github.com/FasterXML/jackson-bom) from 2.21.2 to 2.21.3.
Commits
  • 374fbd0 [maven-release-plugin] prepare release jackson-bom-2.21.3
  • 7059df7 Prep for 2.21.3 release
  • 2fd60bd Merge branch '2.20' into 2.21
  • b82a364 Merge branch '2.19' into 2.20
  • ef4e013 Merge branch '2.18' into 2.19
  • 536ae51 Post-release dep version bump
  • 536c533 [maven-release-plugin] prepare for next development iteration
  • 426b778 [maven-release-plugin] prepare release jackson-bom-2.18.7
  • a73cda9 Prep for 2.18.7 release
  • 76b4a05 Post-release dep version bump
  • Additional commits viewable in compare view

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 738aea03b4..a5058a6336 100644 --- a/pom.xml +++ b/pom.xml @@ -101,7 +101,7 @@ under the License. 4.2.12.Final 1.80.0 4.34.1 - 2.21.2 + 2.21.3 3.5.0 25.2.10 1.12.1 From 5dc3ea699a4b7f8e4799c8ee318b96c9a9cc4f31 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 May 2026 07:58:24 +0200 Subject: [PATCH 215/271] MINOR: Bump com.github.luben:zstd-jni from 1.5.7-7 to 1.5.7-8 (#1132) Bumps [com.github.luben:zstd-jni](https://github.com/luben/zstd-jni) from 1.5.7-7 to 1.5.7-8.
Commits

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- compression/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compression/pom.xml b/compression/pom.xml index 9014b6913a..f3de6fc248 100644 --- a/compression/pom.xml +++ b/compression/pom.xml @@ -55,7 +55,7 @@ under the License. com.github.luben zstd-jni - 1.5.7-7 + 1.5.7-8 From 5e2fb636514c708cf27bd367c8f452d72ab5fdef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 May 2026 15:45:59 +0200 Subject: [PATCH 216/271] MINOR: Bump com.google.api.grpc:proto-google-common-protos from 2.70.0 to 2.71.0 (#1144) Bumps [com.google.api.grpc:proto-google-common-protos](https://github.com/googleapis/sdk-platform-java) from 2.70.0 to 2.71.0.
Commits
  • 6473668 chore(main): release 2.63.0 (#3927)
  • 8015e7e chore: update googleapis commit at Fri Oct 3 02:28:22 UTC 2025 (#3923)
  • 48075a8 chore: Upper bound file deps change has chore type (#3949)
  • 1d74663 deps: update google auth library dependencies to v1.40.0 (#3945)
  • 7fb4f15 deps: Upgrade Google Http Java Client to v2.0.2 (#3946)
  • feabef3 feat(librariangen): add bazel package (#3940)
  • 8d6c1f9 deps: Bump Guava to v33.5.0 (#3943)
  • 180b9a0 build(deps): update dependency com.google.cloud:google-cloud-shared-config to...
  • 3f548fb deps: update upper bound dependencies file (#3947)
  • a1b5ba3 chore: Manage errorprone and j2objc versions in pom-parent (#3948)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.api.grpc:proto-google-common-protos&package-manager=maven&previous-version=2.70.0&new-version=2.71.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml index d199f70da4..15b870905d 100644 --- a/flight/flight-core/pom.xml +++ b/flight/flight-core/pom.xml @@ -134,7 +134,7 @@ under the License. com.google.api.grpc proto-google-common-protos - 2.70.0 + 2.71.0 test From 30d528fa9992c9ed16a01853a2ed285f0efcbd21 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 May 2026 15:52:35 +0200 Subject: [PATCH 217/271] MINOR: Bump checker.framework.version from 4.0.0 to 4.1.0 (#1131) Bumps `checker.framework.version` from 4.0.0 to 4.1.0. Updates `org.checkerframework:checker-qual` from 4.0.0 to 4.1.0
Release notes

Sourced from org.checkerframework:checker-qual's releases.

Checker Framework 4.1.0

Version 4.1.0 (2026-05-01)

User-visible changes

Removed deprecated script checker/bin-devel/build.sh; use ./gradlew assemble instead.

Removed deprecated names "builder", "object.construction", and "objectconstruction" for the Called Methods Checker.

Implementation details

New method annotation @DoesNotUnrefineReceiver.

In AnnotatedTypeFactory:

  • new method hasDoesNotUnrefineReceiver().
  • isAliasedTypeAnnotation() is now protected rather than public.

Closed issues

#6890, #7364, #7488.

Changelog

Sourced from org.checkerframework:checker-qual's changelog.

Version 4.1.0 (2026-05-01)

User-visible changes

Removed deprecated script checker/bin-devel/build.sh; use ./gradlew assemble instead.

Removed deprecated names "builder", "object.construction", and "objectconstruction" for the Called Methods Checker.

Implementation details

New method annotation @DoesNotUnrefineReceiver.

In AnnotatedTypeFactory:

  • new method hasDoesNotUnrefineReceiver().
  • isAliasedTypeAnnotation() is now protected rather than public.

Closed issues

#6890, #7364, #7488.

Commits

Updates `org.checkerframework:checker` from 4.0.0 to 4.1.0
Release notes

Sourced from org.checkerframework:checker's releases.

Checker Framework 4.1.0

Version 4.1.0 (2026-05-01)

User-visible changes

Removed deprecated script checker/bin-devel/build.sh; use ./gradlew assemble instead.

Removed deprecated names "builder", "object.construction", and "objectconstruction" for the Called Methods Checker.

Implementation details

New method annotation @DoesNotUnrefineReceiver.

In AnnotatedTypeFactory:

  • new method hasDoesNotUnrefineReceiver().
  • isAliasedTypeAnnotation() is now protected rather than public.

Closed issues

#6890, #7364, #7488.

Changelog

Sourced from org.checkerframework:checker's changelog.

Version 4.1.0 (2026-05-01)

User-visible changes

Removed deprecated script checker/bin-devel/build.sh; use ./gradlew assemble instead.

Removed deprecated names "builder", "object.construction", and "objectconstruction" for the Called Methods Checker.

Implementation details

New method annotation @DoesNotUnrefineReceiver.

In AnnotatedTypeFactory:

  • new method hasDoesNotUnrefineReceiver().
  • isAliasedTypeAnnotation() is now protected rather than public.

Closed issues

#6890, #7364, #7488.

Commits

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a5058a6336..e0754f7789 100644 --- a/pom.xml +++ b/pom.xml @@ -112,7 +112,7 @@ under the License. 10.23.0 true 2.42.0 - 4.0.0 + 4.1.0 1.5.32 none -Xdoclint:none From b068e28343d54604b1966e916a376ea4c1491f48 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 May 2026 15:52:55 +0200 Subject: [PATCH 218/271] MINOR: Bump com.github.ben-manes.caffeine:caffeine from 3.2.3 to 3.2.4 (#1130) Bumps [com.github.ben-manes.caffeine:caffeine](https://github.com/ben-manes/caffeine) from 3.2.3 to 3.2.4.
Release notes

Sourced from com.github.ben-manes.caffeine:caffeine's releases.

3.2.4

  • Improved access expiration's read performance by avoiding false sharing effects caused by the timestamp update
  • Fixed head-of-line blocking of expiration queues caused by in-flight async entries (#1954)
  • Fixed various minor issues found using AI audits
  • Added ObjectInputFilter support to JCache
Commits
  • 836b65c use a consistent expiration tolerance calculation
  • 0dc7daf resurrect in-flight async entries on expiration
  • 0bac8b5 handle head-of-line blocking of expiration queues (fixes #1954)
  • ff25836 test polish
  • f3a6176 Fix JCache close/createCache races and recursive teardown
  • 622fbe7 Fix removal in identity views and widen hill-climber counters
  • 8da5a7a defer weighing the entry until after the putIfAbsent hit fast-path
  • 94ad0ff Record eviction stats before notifying the removal listener consistently
  • f94c011 Auto-assert eviction stats alongside notifications.withCause.exclusively
  • 2e945e0 Skip timestamp writes within tolerance on the read path.
  • Additional commits viewable in compare view

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 1f9bc3e00e..a813b2e168 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -176,7 +176,7 @@ under the License. com.github.ben-manes.caffeine caffeine - 3.2.3 + 3.2.4 From e6d9248447ef90d2d9a1412bfdee7019377e0603 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 May 2026 15:53:10 +0200 Subject: [PATCH 219/271] MINOR: Bump com.gradle:common-custom-user-data-maven-extension from 2.1.0 to 2.2.0 (#1128) Bumps [com.gradle:common-custom-user-data-maven-extension](https://github.com/gradle/common-custom-user-data-maven-extension) from 2.1.0 to 2.2.0.
Release notes

Sourced from com.gradle:common-custom-user-data-maven-extension's releases.

2.2.0

  • [NEW] Add AI tag to the Build Scan when invoked by an AI Agent
  • [NEW] Add custom value to the Build Scan indicating which AI Agent invoked the build
  • [NEW] Add link in Build Scan to GitHub PR
  • [NEW] For GitHub PRs, capture GITHUB_BASE_REF as the value PR base branch
Commits
  • d594c60 [maven-release-plugin] prepare release v2.2.0
  • 0a48e7a Merge pull request #375 from gradle/cj/github-pr-base-branch
  • 02f6001 Capture GITHUB_BASE_REF as 'PR base branch' for GitHub PR builds
  • fc03fa9 Add recent feature additions to changes.md
  • 1501555 Merge pull request #370 from gradle/add-ai-agent-metadata
  • de773ae [Renovate Bot] Update dependency org.apache.maven:maven-core to v3.9.15 (#374)
  • 074832a [Renovate Bot] Update dependency maven to v3.9.15 (#373)
  • 9de6cf1 Merge pull request #372 from gradle/renovate/github-actions
  • ea1e3fc [Renovate Bot] Update actions/upload-artifact digest to 043fb46
  • 4384849 Match AI tags/values to CCUD Gradle
  • Additional commits viewable in compare view

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .mvn/extensions.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index f52dfafb4a..e909f05105 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -28,6 +28,6 @@ com.gradle common-custom-user-data-maven-extension - 2.1.0 + 2.2.0 From 331e9ee0968ab92499ed362ed276fa85e60b870d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 May 2026 15:53:29 +0200 Subject: [PATCH 220/271] MINOR: Bump commons-codec:commons-codec from 1.21.0 to 1.22.0 (#1127) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [commons-codec:commons-codec](https://github.com/apache/commons-codec) from 1.21.0 to 1.22.0.
Changelog

Sourced from commons-codec:commons-codec's changelog.

Apache Commons Codec 1.22.0 Release Notes

The Apache Commons Codec team is pleased to announce the release of Apache Commons Codec 1.22.0.

The Apache Commons Codec component contains encoders and decoders for formats such as Base16, Base32, Base64, digest, and Hexadecimal. In addition to these widely used encoders and decoders, the codec package also maintains a collection of phonetic encoding utilities.

This is a feature and maintenance release. Java 8 or later is required.

New features

  • CODEC-326: Add Base58 support. Thanks to Inkeet, Gary Gregory, Wolff Bock von Wuelfingen.
  •  Add
BaseNCodecInputStream.AbstracBuilder.setByteArray(byte[]). Thanks to
Gary Gregory.
  • CODEC-335: Add GitIdentifiers to compute Git blob and tree object identifiers. Thanks to Piotr P. Karwasz, Gary Gregory.

Fixed Bugs

  • CODEC-249: Fix Incorrect transform of CH digraph according Metaphone basic rules #423. Thanks to Shalu Jha, Andrey, Gary Gregory.
  • CODEC-317: ColognePhonetic can create duplicate consecutive codes in some cases. Thanks to DRUser123, Shalu Jha, Gary Gregory.
  •  Add boundary tests for BinaryCodec.fromAscii partial-bit
inputs [#425](https://github.com/apache/commons-codec/issues/425).
Thanks to fancying, Gary Gregory.
  • CODEC-336: Base64.Builder.setUrlSafe(boolean) Javadoc incorrectly states null is accepted for primitive boolean parameter. Thanks to Partha Paul, Gary Gregory.

Changes

  •  Bump org.apache.commons:commons-parent from 96 to 98. Thanks
to Gary Gregory.

For complete information on Apache Commons Codec, including instructions on how to submit bug reports, patches, or suggestions for improvement, see the Apache Commons Codec website:

https://commons.apache.org/proper/commons-codec/

Download page: https://commons.apache.org/proper/commons-codec/download_codec.cgi

Commits

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- vector/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vector/pom.xml b/vector/pom.xml index 4d247961c9..9b40e8820c 100644 --- a/vector/pom.xml +++ b/vector/pom.xml @@ -60,7 +60,7 @@ under the License. commons-codec commons-codec - 1.21.0 + 1.22.0 org.apache.arrow From 0f7665f2b78a5f6bcfb37924fbee05a45adfb5f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 May 2026 09:31:15 +0200 Subject: [PATCH 221/271] MINOR: Bump com.nimbusds:oauth2-oidc-sdk from 11.37 to 11.37.1 (#1143) Bumps [com.nimbusds:oauth2-oidc-sdk](https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions) from 11.37 to 11.37.1.
Changelog

Sourced from com.nimbusds:oauth2-oidc-sdk's changelog.

version 1.0 (2012-05-29) * First official release with authorisation endpoint, token endpoint, check ID endpoint and UserInfo endpoint support. * JSON Web Tokens (JWTs) support through the Nimbus-JWT library. * Language Tags (RFC 5646) support through the Nimbus-LangTag library. * JSON support through the JSON Smart library.

version 2.0 (2013-05-13) * Intermediary development release with Maven build, published to Maven Central.

version 2.1 (2013-06-06) * Updates the APIs to OpenID Connect Messages draft 20, OpenID Connect Standard draft 21, OpenID Connect Discovery draft 17 and OpenID Connect Registration draft 19. * Major refactoring of the APIs for greater simplicity. * Adds JUnit tests.

version 2.2 (2013-06-18) * Refactors dynamic OpenID Connect client registration. * Adds partial support of the OAuth 2.0 Dynamic Client Registration Protocol (draft-ietf-oauth-dyn-reg-12). * Optimises parsing of request parameters consisting of one or more tokens (scope, response type, etc).

version 2.3 (2013-06-19) * Renames OAuth 2.0 dynamic client registration package. * Adds ClientInformation.getClientMetadata() method. * Adds OIDCClientInformation class.

version 2.4 (2013-06-20) * Adds static OIDCClientInformation.parse(JSONObject) method.

version 2.5 (2013-06-22) * Adds support OAuth 2.0 dynamic client update. * Adds OpenID Connect dynamic client registration classes.

version 2.6 (2013-06-25) * Enforces order of preference of ACR values in OpenID Connect client metadata, as required by the specification. * Documentation and performance improvements.

version 2.7 (2013-06-26) * Switches Identifier generation to java.security.SecureRandom.

version 2.8 (2013-06-30) * Fixes serialisation and assignment bugs in ClientMetadata. * Switches Secret generation to java.security.SecureRandom.

version 2.9 (2013-09-17)

... (truncated)

Commits
  • 2a0f271 [maven-release-plugin] prepare for next development iteration
  • fac7277 Bumps Nimbus JOSE+JWT, BouncyCastle
  • 517deb7 [maven-release-plugin] prepare release 11.37.1
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.nimbusds:oauth2-oidc-sdk&package-manager=maven&previous-version=11.37&new-version=11.37.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index a813b2e168..ea33091101 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -182,7 +182,7 @@ under the License. com.nimbusds oauth2-oidc-sdk - 11.37 + 11.37.1 From 3bc34b041761081ac32a7cd3b167f9ab8b628677 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 May 2026 13:51:48 +0200 Subject: [PATCH 222/271] MINOR: Bump commons-io:commons-io from 2.21.0 to 2.22.0 (#1126) Bumps commons-io:commons-io from 2.21.0 to 2.22.0. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=commons-io:commons-io&package-manager=maven&previous-version=2.21.0&new-version=2.22.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dataset/pom.xml | 2 +- flight/flight-sql-jdbc-core/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dataset/pom.xml b/dataset/pom.xml index 7a0210ce95..cb889ecd7d 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -156,7 +156,7 @@ under the License. commons-io commons-io - 2.21.0 + 2.22.0 test diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index ea33091101..237b25d0e5 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -105,7 +105,7 @@ under the License. commons-io commons-io - 2.21.0 + 2.22.0 test From af86cd3a7c17237368ea8411ed32e21b262c5b7b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 May 2026 15:22:37 +0200 Subject: [PATCH 223/271] MINOR: Bump org.apache.calcite.avatica:avatica from 1.26.0 to 1.27.0 (#986) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.apache.calcite.avatica:avatica](https://github.com/apache/calcite-avatica) from 1.26.0 to 1.27.0.
Commits
  • 7754d94 [CALCITE-7200] Release Avatica 1.27.0
  • 1e05be2 [CALCITE-7171] Update Jackson from 2.15.4 to 2.18.4.1 and switch to using jac...
  • 9698a96 [CALCITE-7177] Update Guava from 33.4.0-jre to 33.4.8-jre in Avatica
  • 0aec625 Bump rexml from 3.4.1 to 3.4.2 in /site
  • 5954d1a [CALCITE-7165] Update OWASP plugin version to 12.1.3 for JDKs >= 11
  • 7b6f14c [CALCITE-7172] Update chekstyle version from 10.19.0 to 10.26.1 in Avatica
  • 3ee1fd7 [CALCITE-7169] Update protobuf from 3.25.5 to 3.25.8 in Avatica
  • 927dc10 [CALCITE-7168] Update httpcore5 from 5.3.1 to 5.3.5 in Avatica
  • 458189f [CALCITE-7167] Upgrade Jetty from 9.4.56.v20240826 to 9.4.58.v20250814 in Ava...
  • 592a39e [CALCITE-7166] Update Gradle from 8.7 to 8.14.3 in Avatica
  • Additional commits viewable in compare view

> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: JB Onofré --- flight/flight-sql-jdbc-core/pom.xml | 2 +- ...owFlightJdbcVectorSchemaRootResultSet.java | 2 +- .../accessor/ArrowFlightJdbcAccessor.java | 24 +++++++++++++++++++ flight/flight-sql-jdbc-driver/pom.xml | 1 + 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 237b25d0e5..96a11f2b9a 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -159,7 +159,7 @@ under the License. org.apache.calcite.avatica avatica - 1.26.0 + 1.27.0 diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java index 49334951de..5d02d6e843 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java @@ -128,7 +128,7 @@ public Object getObject(int columnIndex) throws SQLException { if (metaData.type.id == Types.TIMESTAMP_WITH_TIMEZONE) { return accessor.getTimestamp(localCalendar); } else { - return AvaticaSite.get(accessor, metaData.type.id, localCalendar); + return AvaticaSite.get(accessor, metaData.type.id, true, localCalendar); } } diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessor.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessor.java index f0fa55fa82..cd762fb1ac 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessor.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessor.java @@ -36,6 +36,10 @@ import java.util.Calendar; import java.util.Map; import java.util.function.IntSupplier; +import org.joou.UByte; +import org.joou.UInteger; +import org.joou.ULong; +import org.joou.UShort; /** Base Jdbc Accessor. */ public abstract class ArrowFlightJdbcAccessor implements Accessor { @@ -99,6 +103,26 @@ public long getLong() throws SQLException { throw getOperationNotSupported(this.getClass()); } + @Override + public UByte getUByte() throws SQLException { + throw getOperationNotSupported(this.getClass()); + } + + @Override + public UShort getUShort() throws SQLException { + throw getOperationNotSupported(this.getClass()); + } + + @Override + public UInteger getUInt() throws SQLException { + throw getOperationNotSupported(this.getClass()); + } + + @Override + public ULong getULong() throws SQLException { + throw getOperationNotSupported(this.getClass()); + } + @Override public float getFloat() throws SQLException { throw getOperationNotSupported(this.getClass()); diff --git a/flight/flight-sql-jdbc-driver/pom.xml b/flight/flight-sql-jdbc-driver/pom.xml index ff5763702b..801089d090 100644 --- a/flight/flight-sql-jdbc-driver/pom.xml +++ b/flight/flight-sql-jdbc-driver/pom.xml @@ -159,6 +159,7 @@ under the License. org.apache.calcite.avatica:* META-INF/services/java.sql.Driver + META-INF/README.txt From 8c36d02205974c0cb912b11383e26f141ee2971e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 May 2026 17:45:09 +0200 Subject: [PATCH 224/271] MINOR: Bump org.apache:apache from 37 to 38 (#1156) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.apache:apache](https://github.com/apache/maven-apache-parent) from 37 to 38.
Release notes

Sourced from org.apache:apache's releases.

Apache Parent POM version 38

👻 Maintenance

📦 Dependency updates

  • Bump org.apache.maven.plugins:maven-invoker-plugin from 3.10.0 to 3.10.1 (#578) @dependabot[bot]
  • Bump org.apache.maven.plugins:maven-invoker-plugin from 3.9.1 to 3.10.0 (#575) @dependabot[bot]
  • Bump org.apache.maven.plugins:maven-resources-plugin from 3.4.0 to 3.5.0 (#572) @dependabot[bot]
  • Bump org.apache.maven.plugins:maven-shade-plugin from 3.6.1 to 3.6.2 (#573) @dependabot[bot]
  • Bump org.apache.apache.resources:apache-source-release-assembly-descriptor from 1.7 to 1.8 (#571) @dependabot[bot]
  • Bump version.maven-surefire from 3.5.4 to 3.5.5 (#570) @dependabot[bot]
  • Bump org.apache.maven.plugins:maven-dependency-plugin from 3.9.0 to 3.10.0 (#568) @dependabot[bot]
  • Bump org.apache.maven.plugins:maven-compiler-plugin from 3.14.1 to 3.15.0 (#567) @dependabot[bot]
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache:apache&package-manager=maven&previous-version=37&new-version=38)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- bom/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index d97cc291c2..083ee1e0de 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache apache - 37 + 38 diff --git a/pom.xml b/pom.xml index e0754f7789..d6d36e5607 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache apache - 37 + 38 org.apache.arrow From c49d97625524a9ba3cb72380f88a2ee12f435695 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 May 2026 17:45:26 +0200 Subject: [PATCH 225/271] MINOR: Bump org.immutables:value-annotations from 2.12.1 to 2.12.2 (#1157) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.immutables:value-annotations](https://github.com/immutables/immutables) from 2.12.1 to 2.12.2.
Release notes

Sourced from org.immutables:value-annotations's releases.

2.12.2

Maintenance release

What's Changed

New Contributors

Full Changelog: https://github.com/immutables/immutables/compare/2.12.1...2.12.2

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.immutables:value-annotations&package-manager=maven&previous-version=2.12.1&new-version=2.12.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d6d36e5607..dabb02594a 100644 --- a/pom.xml +++ b/pom.xml @@ -183,7 +183,7 @@ under the License. org.immutables value-annotations - 2.12.1 + 2.12.2 provided From d88adb33b00e8a7c3b743312b11c05ed36d2bd37 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 May 2026 18:42:53 +0200 Subject: [PATCH 226/271] MINOR: Bump io.netty:netty-bom from 4.2.12.Final to 4.2.13.Final (#1155) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [io.netty:netty-bom](https://github.com/netty/netty) from 4.2.12.Final to 4.2.13.Final.
Release notes

Sourced from io.netty:netty-bom's releases.

netty-4.2.13.Final

CVEs Fixed

What's Changed

... (truncated)

Commits
  • b3844c8 [maven-release-plugin] prepare release netty-4.2.13.Final
  • 82f47fa Merge commit from fork
  • ada0999 Merge commit from fork
  • b4051e2 Fix BrotliDecoder not forwarding all decompressed chunks
  • 67207c1 Merge commit from fork
  • 541ca7c Merge commit from fork
  • 943edb3 Fix codec-dns tests
  • 6459a28 Merge commit from fork
  • b4ba61b Fix checkstyle in HttpObjectDecoder
  • 977661f Merge commit from fork
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=io.netty:netty-bom&package-manager=maven&previous-version=4.2.12.Final&new-version=4.2.13.Final)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index dabb02594a..df387a067d 100644 --- a/pom.xml +++ b/pom.xml @@ -98,7 +98,7 @@ under the License. 5.12.2 2.0.17 33.6.0-jre - 4.2.12.Final + 4.2.13.Final 1.80.0 4.34.1 2.21.3 From 28367478a6ae695e409368b6f68b9ea22730fd9c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 May 2026 15:47:49 +0200 Subject: [PATCH 227/271] MINOR: Bump io.grpc:grpc-bom from 1.80.0 to 1.81.0 (#1154) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [io.grpc:grpc-bom](https://github.com/grpc/grpc-java) from 1.80.0 to 1.81.0.
Release notes

Sourced from io.grpc:grpc-bom's releases.

V1.81.0

In this release we drop support for Android API level 22 or lower (Lollipop or earlier), following Google Play Service’s discontinued updates for Lollipop (API levels 21 & 22) and now requires a minimum of API level 23 (Android 6.0 Marshmallow).

API Changes

  • api: Deprecate LoadBalancer.handleResolvedAddresses(). Developers maintaining custom LoadBalancer implementations should transition to using LoadBalancer.acceptResolvedAddresses(). Unlike the deprecated method, acceptResolvedAddresses() returns a Status object, allowing the load balancer to explicitly report success or reject the update if the provided addresses or configuration are invalid. (#11623)

Behavior Changes

  • core: Enable dns "caching" on Android for 30 seconds to reduce CPU impact of a refresh loop with an LB policy (0675f70af). DnsNameResolver ignores re-resolution requests on OpenJDK-like platforms if it has been too soon since the last DNS query because InetAddress.getAllByName() has a cache with a fixed entry lifetime, but this logic was disabled for Android which does not have that style of cache. Android’s cache uses the result TTL, which will rarely be less than 30 seconds. This change would probably be most noticeable when 1) changing to a different network (e.g., from wifi to mobile), 2) the server has different addresses for different networks, and 3) the app is not using AndroidChannelBuilder with an android.context.Context. For reference, it seems Chrome caches for 1 minute

Bug Fixes

  • opentelemetry: Fix baggage propagation, the baggage propagation for opentelemetry introduced in #12389 was broken. The context is decided once and used for all recording for the call, thus guaranteeing all record()s have consistent information.
  • core: Address a race condition where ManagedChannelOrphanWrapper could incorrectly log a "not shutdown properly" warning during garbage collection when using directExecutor(). (#12705) (d459338d9)
  • xds: Fix xDS HTTP CONNECT's transport socket name bug which is now corrected to use typeUrl. (#12740) (eac9fe961)
  • xds: Fix an issue where subchannel metrics were dropping their association with the backend_service. This ensures xDS load balancing metrics are reported accurately. (#12735)

New Features

  • netty: Add tcp metrics, by implementing a few of the metrics defined in A80.
  • api: Add a CallOption for a custom label on per-RPC metrics (0e39b2967). This CallOption is copied by grpc-opentelemetry to the grpc.client.call.custom label as defined by gRFC A108. See also the gRPC OpenTelemetry Metrics guide (update in-progress)
  • xds: Add support for Weighted Round Robin (WRR) load balancing driven by custom backend metrics, implementing the behavior defined in gRFC A114. (#12645)
  • utils: Update AdvancedTlsX509KeyManager so that developers can now preserve and use key aliases when dynamically reloading TLS certificates. (#12686)

Documentation

  • Update the "Outgoing Flow Control" section in the Manual Flow Control example to say onNext() does not block, but rather queues the messages in memory and advises developers to use CallStreamObserver.isReady() to prevent this memory exhaustion (#12700) (a3a9ffcbe) (#12726) (65ae2efda)
  • examples: Clean up Health example, and document need for grpc-services (3ed732fc0)

Dependencies

  • Upgrade Dependencies (#12719) (16e17abba). Google-auth-library: 1.42.1, animal-sniffer: 1.27, assertj-core:3.27.7, error_prone_annotations:2.48.0, proto-google-common-protos:2.64.1, google-cloud-logging:3.23.10, jetty-http2-server:12.1.7, jetty-ee10-servlet:12.1.7, lincheck:3.4, opentelemetry-api:1.60.1, opentelemetry-exporter-prometheus:1.60.1-alpha, opentelemetry-gcp-resources:1.54.0-alpha, opentelemetry-sdk-extension-autoconfigure:1.60.1, opentelemetry-sdk-testing:1.60.1, robolectric:4.16.1, tomcat-embed-core:10.1.52, tomcat-embed-core9: 9.0.115,
  • Upgrade Netty to 4.1.132 and netty-tcnative to 2.0.75 (1528f809c)

Thanks to

Commits
  • 6951542 Bump version to 1.81.0
  • e94188e Update README etc to reference 1.81.0
  • 4813c6d core,xds: Fix backend_service plumbing for subchannel metrics (#12735)
  • 6737eb5 Revert "Replace javax ThreadSafe annotation with errorprone ThreadSafe (#1274...
  • ef35313 Replace javax ThreadSafe annotation with errorprone ThreadSafe (#12742)
  • 3ed732f examples: Clean up Health, and document need for grpc-services
  • eac9fe9 xds: fix xDS HTTP CONNECT's transport socket name bug (#12740)
  • 1528f80 Upgrade Netty to 4.1.132 and netty-tcnative to 2.0.75
  • d057a7e [xds] Implement A114: WRR support for custom backend metrics (#12645)
  • 842636f xds: Add configuration objects for ExtAuthz, GrpcService and Bootstrap change...
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=io.grpc:grpc-bom&package-manager=maven&previous-version=1.80.0&new-version=1.81.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: JB Onofré --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index df387a067d..a28d7167e1 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,7 @@ under the License. 2.0.17 33.6.0-jre 4.2.13.Final - 1.80.0 + 1.81.0 4.34.1 2.21.3 3.5.0 From 4899492a26f941d955f5567797aa697262e747dd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 May 2026 15:48:10 +0200 Subject: [PATCH 228/271] MINOR: Bump parquet.version from 1.17.0 to 1.17.1 (#1152) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps `parquet.version` from 1.17.0 to 1.17.1. Updates `org.apache.parquet:parquet-avro` from 1.17.0 to 1.17.1
Release notes

Sourced from org.apache.parquet:parquet-avro's releases.

Apache Parquet Java 1.17.1

What's Changed

Full Changelog: https://github.com/apache/parquet-java/compare/apache-parquet-1.17.0...apache-parquet-1.17.1

Apache Parquet Java 1.17.1 RC0

What's Changed

Full Changelog: https://github.com/apache/parquet-java/compare/apache-parquet-1.17.0...apache-parquet-1.17.1-rc0

Commits

Updates `org.apache.parquet:parquet-hadoop` from 1.17.0 to 1.17.1
Release notes

Sourced from org.apache.parquet:parquet-hadoop's releases.

Apache Parquet Java 1.17.1

What's Changed

Full Changelog: https://github.com/apache/parquet-java/compare/apache-parquet-1.17.0...apache-parquet-1.17.1

Apache Parquet Java 1.17.1 RC0

What's Changed

Full Changelog: https://github.com/apache/parquet-java/compare/apache-parquet-1.17.0...apache-parquet-1.17.1-rc0

Commits

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dataset/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataset/pom.xml b/dataset/pom.xml index cb889ecd7d..5acc837860 100644 --- a/dataset/pom.xml +++ b/dataset/pom.xml @@ -32,7 +32,7 @@ under the License. ../../../cpp/release-build/ - 1.17.0 + 1.17.1 1.12.1 From ef359de2cf33f3020eb7c41cd4a5eee564b99bf6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 May 2026 08:27:25 +0200 Subject: [PATCH 229/271] MINOR: Bump dep.slf4j.version from 2.0.17 to 2.0.18 (#1151) Bumps `dep.slf4j.version` from 2.0.17 to 2.0.18. Updates `org.slf4j:slf4j-api` from 2.0.17 to 2.0.18 Updates `org.slf4j:slf4j-jdk14` from 2.0.17 to 2.0.18 Updates `org.slf4j:jul-to-slf4j` from 2.0.17 to 2.0.18 Updates `org.slf4j:jcl-over-slf4j` from 2.0.17 to 2.0.18 Updates `org.slf4j:log4j-over-slf4j` from 2.0.17 to 2.0.18 Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a28d7167e1..2f55d591fd 100644 --- a/pom.xml +++ b/pom.xml @@ -96,7 +96,7 @@ under the License. ${project.build.directory}/generated-sources 1.9.0 5.12.2 - 2.0.17 + 2.0.18 33.6.0-jre 4.2.13.Final 1.81.0 From 4ecc92f817fc71fcf94e0df87d2c6c1bc9877847 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 May 2026 11:32:03 +0200 Subject: [PATCH 230/271] MINOR: Bump org.apache.parquet:parquet-variant from 1.17.0 to 1.17.1 (#1150) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.apache.parquet:parquet-variant](https://github.com/apache/parquet-mr) from 1.17.0 to 1.17.1.
Release notes

Sourced from org.apache.parquet:parquet-variant's releases.

Apache Parquet Java 1.17.1

What's Changed

Full Changelog: https://github.com/apache/parquet-java/compare/apache-parquet-1.17.0...apache-parquet-1.17.1

Apache Parquet Java 1.17.1 RC0

What's Changed

Full Changelog: https://github.com/apache/parquet-java/compare/apache-parquet-1.17.0...apache-parquet-1.17.1-rc0

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.parquet:parquet-variant&package-manager=maven&previous-version=1.17.0&new-version=1.17.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2f55d591fd..0ce71f74cc 100644 --- a/pom.xml +++ b/pom.xml @@ -105,7 +105,7 @@ under the License. 3.5.0 25.2.10 1.12.1 - 1.17.0 + 1.17.1 5.23.0 2 From 755ce550c4dfc1771c2b350a0159b5e7f903debb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 May 2026 10:32:04 +0200 Subject: [PATCH 231/271] MINOR: Bump org.immutables:value from 2.12.1 to 2.12.2 (#1168) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.immutables:value](https://github.com/immutables/immutables) from 2.12.1 to 2.12.2.
Release notes

Sourced from org.immutables:value's releases.

2.12.2

Maintenance release

What's Changed

New Contributors

Full Changelog: https://github.com/immutables/immutables/compare/2.12.1...2.12.2

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.immutables:value&package-manager=maven&previous-version=2.12.1&new-version=2.12.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0ce71f74cc..e69357c6af 100644 --- a/pom.xml +++ b/pom.xml @@ -314,7 +314,7 @@ under the License. org.immutables value - 2.12.1 + 2.12.2 From b58ce11516afd0aa2959673dac74ca0258ca2166 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 May 2026 10:54:46 +0200 Subject: [PATCH 232/271] MINOR: Bump com.google.protobuf:protobuf-bom from 4.34.1 to 4.35.0 (#1167) Bumps [com.google.protobuf:protobuf-bom](https://github.com/protocolbuffers/protobuf) from 4.34.1 to 4.35.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.protobuf:protobuf-bom&package-manager=maven&previous-version=4.34.1&new-version=4.35.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e69357c6af..2b629673ef 100644 --- a/pom.xml +++ b/pom.xml @@ -100,7 +100,7 @@ under the License. 33.6.0-jre 4.2.13.Final 1.81.0 - 4.34.1 + 4.35.0 2.21.3 3.5.0 25.2.10 From 7fc14a532492f00e0744d8316fa8fb2f6970d31e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 May 2026 11:15:13 +0200 Subject: [PATCH 233/271] MINOR: Bump io.netty:netty-bom from 4.2.13.Final to 4.2.14.Final (#1166) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [io.netty:netty-bom](https://github.com/netty/netty) from 4.2.13.Final to 4.2.14.Final.
Release notes

Sourced from io.netty:netty-bom's releases.

netty-4.2.14.Final

What's Changed

New Contributors

Full Changelog: https://github.com/netty/netty/compare/netty-4.2.13.Final...netty-4.2.14.Final

Commits
  • 0a60b75 [maven-release-plugin] prepare release netty-4.2.14.Final
  • 72df658 Fix MQTT decoder size check after variable header replay (#16787)
  • 7125dba MQTT: Allow MQTT 5 CONNECT with password only (#16833)
  • 9e19320 IoUring: Stop generic FileRegion drain loop when transferred() reaches count(...
  • 4ce9f17 Route synchronous onLookupComplete exceptions via fireExceptionCaught (#16794)
  • f7b1b7d Fix memoryAddress() for direct ByteBuffers wrapped by Unpooled without Unsafe...
  • 0ccb265 IpFilter: Fix ClassCastException caused by IpSubnetFilter if only ipv6 rules ...
  • a6aeb6d Resolve all localhost addresses without querying DNS servers (#16749)
  • c328ba2 Fix ResumptionController wrapping (#16815)
  • bc5862b HTTP2: Use 100 as default max concurrent streams setting (#16804)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=io.netty:netty-bom&package-manager=maven&previous-version=4.2.13.Final&new-version=4.2.14.Final)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2b629673ef..322e35f8b4 100644 --- a/pom.xml +++ b/pom.xml @@ -98,7 +98,7 @@ under the License. 5.12.2 2.0.18 33.6.0-jre - 4.2.13.Final + 4.2.14.Final 1.81.0 4.35.0 2.21.3 From 96339f6c2f35c3e6676e62fded200c0934f9d8d3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 May 2026 11:22:01 +0200 Subject: [PATCH 234/271] MINOR: Bump com.nimbusds:oauth2-oidc-sdk from 11.37.1 to 11.37.2 (#1164) Bumps [com.nimbusds:oauth2-oidc-sdk](https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions) from 11.37.1 to 11.37.2.
Changelog

Sourced from com.nimbusds:oauth2-oidc-sdk's changelog.

version 1.0 (2012-05-29) * First official release with authorisation endpoint, token endpoint, check ID endpoint and UserInfo endpoint support. * JSON Web Tokens (JWTs) support through the Nimbus-JWT library. * Language Tags (RFC 5646) support through the Nimbus-LangTag library. * JSON support through the JSON Smart library.

version 2.0 (2013-05-13) * Intermediary development release with Maven build, published to Maven Central.

version 2.1 (2013-06-06) * Updates the APIs to OpenID Connect Messages draft 20, OpenID Connect Standard draft 21, OpenID Connect Discovery draft 17 and OpenID Connect Registration draft 19. * Major refactoring of the APIs for greater simplicity. * Adds JUnit tests.

version 2.2 (2013-06-18) * Refactors dynamic OpenID Connect client registration. * Adds partial support of the OAuth 2.0 Dynamic Client Registration Protocol (draft-ietf-oauth-dyn-reg-12). * Optimises parsing of request parameters consisting of one or more tokens (scope, response type, etc).

version 2.3 (2013-06-19) * Renames OAuth 2.0 dynamic client registration package. * Adds ClientInformation.getClientMetadata() method. * Adds OIDCClientInformation class.

version 2.4 (2013-06-20) * Adds static OIDCClientInformation.parse(JSONObject) method.

version 2.5 (2013-06-22) * Adds support OAuth 2.0 dynamic client update. * Adds OpenID Connect dynamic client registration classes.

version 2.6 (2013-06-25) * Enforces order of preference of ACR values in OpenID Connect client metadata, as required by the specification. * Documentation and performance improvements.

version 2.7 (2013-06-26) * Switches Identifier generation to java.security.SecureRandom.

version 2.8 (2013-06-30) * Fixes serialisation and assignment bugs in ClientMetadata. * Switches Secret generation to java.security.SecureRandom.

version 2.9 (2013-09-17)

... (truncated)

Commits
  • fedf633 [maven-release-plugin] prepare for next development iteration
  • 29b77a0 Updates to JSON Smart 2.6.0
  • 6e53206 [maven-release-plugin] prepare release 11.37.2
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.nimbusds:oauth2-oidc-sdk&package-manager=maven&previous-version=11.37.1&new-version=11.37.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 96a11f2b9a..3741ee083c 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -182,7 +182,7 @@ under the License. com.nimbusds oauth2-oidc-sdk - 11.37.1 + 11.37.2 From 6a6b8c1c8f3364da125df743c4b00a212d91c449 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 May 2026 13:15:12 +0200 Subject: [PATCH 235/271] MINOR: Bump com.gradle:develocity-maven-extension from 2.4.0 to 2.4.1 (#1161) Bumps com.gradle:develocity-maven-extension from 2.4.0 to 2.4.1. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.gradle:develocity-maven-extension&package-manager=maven&previous-version=2.4.0&new-version=2.4.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .mvn/extensions.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index e909f05105..38b3c807b7 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -23,7 +23,7 @@ com.gradle develocity-maven-extension - 2.4.0 + 2.4.1 com.gradle From 315fd710cdf2a418e21cb567bedc8d7473e17ec7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 May 2026 11:20:28 +0200 Subject: [PATCH 236/271] MINOR: [CI] Bump docker/login-action from 4.1.0 to 4.2.0 (#1160) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [docker/login-action](https://github.com/docker/login-action) from 4.1.0 to 4.2.0.
Release notes

Sourced from docker/login-action's releases.

v4.2.0

Full Changelog: https://github.com/docker/login-action/compare/v4.1.0...v4.2.0

Commits
  • 650006c Merge pull request #960 from docker/dependabot/npm_and_yarn/aws-sdk-dependenc...
  • 99df1a3 chore: update generated content
  • 3ab375f build(deps): bump the aws-sdk-dependencies group across 1 directory with 2 up...
  • 39d8580 Merge pull request #970 from docker/dependabot/npm_and_yarn/docker/actions-to...
  • 4eefcd3 chore: update generated content
  • 56d092c build(deps): bump @​docker/actions-toolkit from 0.86.0 to 0.90.0
  • e2e31ca Merge pull request #976 from docker/dependabot/npm_and_yarn/actions/core-3.0.1
  • 0bced94 chore: update generated content
  • 3e75a0f build(deps): bump @​actions/core from 3.0.0 to 3.0.1
  • 365bebd Merge pull request #984 from docker/dependabot/github_actions/aws-actions/con...
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=docker/login-action&package-manager=github_actions&previous-version=4.1.0&new-version=4.2.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 2658f1da00..f0e99f1219 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -127,7 +127,7 @@ jobs: with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing - - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: registry: ghcr.io username: ${{ github.actor }} From da97828bfe2818e1233ef8c421e89e238fff8340 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Jun 2026 15:43:15 +0200 Subject: [PATCH 237/271] MINOR: Bump checker.framework.version from 4.1.0 to 4.2.0 (#1170) Bumps `checker.framework.version` from 4.1.0 to 4.2.0. Updates `org.checkerframework:checker-qual` from 4.1.0 to 4.2.0
Release notes

Sourced from org.checkerframework:checker-qual's releases.

Checker Framework 4.2.0

Version 4.2.0 (2026-06-01)

User-visible changes

Renamed error message key "createsmustcallfor.target.unparseable" to "createsmustcallfor.target.unparsable".

Implementation details

In AnnotatedTypeFactory:

  • new overload canonicalAnnotation(AnnotationMirror, TypeMirror).

In TypeHierarchy:

  • new methods equalsShallowEffective().

Closed issues

#7676, #7679, #7680, #7695, #7697, #7699, #7700, #7727.

Changelog

Sourced from org.checkerframework:checker-qual's changelog.

Version 4.2.0 (2026-06-01)

User-visible changes

Renamed error message key "createsmustcallfor.target.unparseable" to "createsmustcallfor.target.unparsable".

Implementation details

In AnnotatedTypeFactory:

  • new overload canonicalAnnotation(AnnotationMirror, TypeMirror).

In TypeHierarchy:

  • new methods equalsShallowEffective().

Closed issues

#7676, #7679, #7680, #7695, #7697, #7699, #7700, #7727.

Commits

Updates `org.checkerframework:checker` from 4.1.0 to 4.2.0
Release notes

Sourced from org.checkerframework:checker's releases.

Checker Framework 4.2.0

Version 4.2.0 (2026-06-01)

User-visible changes

Renamed error message key "createsmustcallfor.target.unparseable" to "createsmustcallfor.target.unparsable".

Implementation details

In AnnotatedTypeFactory:

  • new overload canonicalAnnotation(AnnotationMirror, TypeMirror).

In TypeHierarchy:

  • new methods equalsShallowEffective().

Closed issues

#7676, #7679, #7680, #7695, #7697, #7699, #7700, #7727.

Changelog

Sourced from org.checkerframework:checker's changelog.

Version 4.2.0 (2026-06-01)

User-visible changes

Renamed error message key "createsmustcallfor.target.unparseable" to "createsmustcallfor.target.unparsable".

Implementation details

In AnnotatedTypeFactory:

  • new overload canonicalAnnotation(AnnotationMirror, TypeMirror).

In TypeHierarchy:

  • new methods equalsShallowEffective().

Closed issues

#7676, #7679, #7680, #7695, #7697, #7699, #7700, #7727.

Commits

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 322e35f8b4..81da2a2866 100644 --- a/pom.xml +++ b/pom.xml @@ -112,7 +112,7 @@ under the License. 10.23.0 true 2.42.0 - 4.1.0 + 4.2.0 1.5.32 none -Xdoclint:none From 035d96b256cc5eb687add6af2f5326ac5523f272 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Jun 2026 15:57:55 +0200 Subject: [PATCH 238/271] MINOR: Bump dep.junit.jupiter.version from 5.12.2 to 6.1.0 (#1162) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps `dep.junit.jupiter.version` from 5.12.2 to 6.1.0. Updates `org.junit.jupiter:junit-jupiter-engine` from 5.12.2 to 6.1.0
Release notes

Sourced from org.junit.jupiter:junit-jupiter-engine's releases.

JUnit 6.1.0 = Platform 6.1.0 + Jupiter 6.1.0 + Vintage 6.1.0

See Release Notes.

New Contributors

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.0.3...r6.1.0

JUnit 6.1.0-RC1 = Platform 6.1.0-RC1 + Jupiter 6.1.0-RC1 + Vintage 6.1.0-RC1

See Release Notes.

New Contributors

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.1.0-M1...r6.1.0-RC1

JUnit 6.1.0-M1 = Platform 6.1.0-M1 + Jupiter 6.1.0-M1 + Vintage 6.1.0-M1

See Release Notes.

New Contributors

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.0.0...r6.1.0-M1

JUnit 6.0.3 = Platform 6.0.3 + Jupiter 6.0.3 + Vintage 6.0.3

See Release Notes.

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.0.2...r6.0.3

JUnit 6.0.2 = Platform 6.0.2 + Jupiter 6.0.2 + Vintage 6.0.2

See Release Notes.

... (truncated)

Commits

Updates `org.junit.jupiter:junit-jupiter-api` from 5.12.2 to 6.1.0
Release notes

Sourced from org.junit.jupiter:junit-jupiter-api's releases.

JUnit 6.1.0 = Platform 6.1.0 + Jupiter 6.1.0 + Vintage 6.1.0

See Release Notes.

New Contributors

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.0.3...r6.1.0

JUnit 6.1.0-RC1 = Platform 6.1.0-RC1 + Jupiter 6.1.0-RC1 + Vintage 6.1.0-RC1

See Release Notes.

New Contributors

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.1.0-M1...r6.1.0-RC1

JUnit 6.1.0-M1 = Platform 6.1.0-M1 + Jupiter 6.1.0-M1 + Vintage 6.1.0-M1

See Release Notes.

New Contributors

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.0.0...r6.1.0-M1

JUnit 6.0.3 = Platform 6.0.3 + Jupiter 6.0.3 + Vintage 6.0.3

See Release Notes.

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.0.2...r6.0.3

JUnit 6.0.2 = Platform 6.0.2 + Jupiter 6.0.2 + Vintage 6.0.2

See Release Notes.

... (truncated)

Commits

Updates `org.junit.jupiter:junit-jupiter-params` from 5.12.2 to 6.1.0
Release notes

Sourced from org.junit.jupiter:junit-jupiter-params's releases.

JUnit 6.1.0 = Platform 6.1.0 + Jupiter 6.1.0 + Vintage 6.1.0

See Release Notes.

New Contributors

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.0.3...r6.1.0

JUnit 6.1.0-RC1 = Platform 6.1.0-RC1 + Jupiter 6.1.0-RC1 + Vintage 6.1.0-RC1

See Release Notes.

New Contributors

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.1.0-M1...r6.1.0-RC1

JUnit 6.1.0-M1 = Platform 6.1.0-M1 + Jupiter 6.1.0-M1 + Vintage 6.1.0-M1

See Release Notes.

New Contributors

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.0.0...r6.1.0-M1

JUnit 6.0.3 = Platform 6.0.3 + Jupiter 6.0.3 + Vintage 6.0.3

See Release Notes.

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.0.2...r6.0.3

JUnit 6.0.2 = Platform 6.0.2 + Jupiter 6.0.2 + Vintage 6.0.2

See Release Notes.

... (truncated)

Commits

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 81da2a2866..2e9b793562 100644 --- a/pom.xml +++ b/pom.xml @@ -95,7 +95,7 @@ under the License. 1773644827 ${project.build.directory}/generated-sources 1.9.0 - 5.12.2 + 6.1.0 2.0.18 33.6.0-jre 4.2.14.Final From a993bf1727098cd21cd4973d6bcff2b20e5ebfd0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Jun 2026 09:19:57 +0200 Subject: [PATCH 239/271] MINOR: Bump com.google.api.grpc:proto-google-common-protos from 2.71.0 to 2.72.0 (#1178) Bumps [com.google.api.grpc:proto-google-common-protos](https://github.com/googleapis/sdk-platform-java) from 2.71.0 to 2.72.0.
Commits
  • 6e1c179 chore(main): release 2.64.0 (#3954)
  • 7a2f0b0 chore: update upper bound dependencies file (#3966)
  • 1e4a7e5 chore: update googleapis commit at Fri Oct 17 02:31:11 UTC 2025 (#3951)
  • ffb557c deps: Bump grpc-java to v1.76.0 (#3942)
  • 9ad8a4d chore: remove internal/librariangen following migration to librarian repo (#3...
  • 0a1bbea chore(librariangen): Generate to use languagecontainer.Run (#3968)
  • 452d703 feat(librariangen): generate grpc stubs and resource helpers (#3967)
  • 85057e8 ci: remove librarian skipping on matrix builds (#3969)
  • a26a6d9 chore(librariangen): languagecontainer package to parse release-init request ...
  • c86b4ea ci: exclude internal/librariangen/** using dorny/paths-filter (#3961)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.api.grpc:proto-google-common-protos&package-manager=maven&previous-version=2.71.0&new-version=2.72.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-core/pom.xml b/flight/flight-core/pom.xml index 15b870905d..9ae402cdc4 100644 --- a/flight/flight-core/pom.xml +++ b/flight/flight-core/pom.xml @@ -134,7 +134,7 @@ under the License. com.google.api.grpc proto-google-common-protos - 2.71.0 + 2.72.0 test From 485f813d0cec4987f5b597f6cd448d40f074ccce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Jun 2026 13:14:21 +0200 Subject: [PATCH 240/271] MINOR: Bump com.github.luben:zstd-jni from 1.5.7-8 to 1.5.7-10 (#1177) Bumps [com.github.luben:zstd-jni](https://github.com/luben/zstd-jni) from 1.5.7-8 to 1.5.7-10.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.github.luben:zstd-jni&package-manager=maven&previous-version=1.5.7-8&new-version=1.5.7-10)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- compression/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compression/pom.xml b/compression/pom.xml index f3de6fc248..41cdb03796 100644 --- a/compression/pom.xml +++ b/compression/pom.xml @@ -55,7 +55,7 @@ under the License. com.github.luben zstd-jni - 1.5.7-8 + 1.5.7-10 From c836fad712c8a2ca14ac3a4b5f117a3e9fbefdc7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Jun 2026 09:43:35 +0200 Subject: [PATCH 241/271] MINOR: Bump com.fasterxml.jackson:jackson-bom from 2.21.3 to 2.22.0 (#1173) Bumps [com.fasterxml.jackson:jackson-bom](https://github.com/FasterXML/jackson-bom) from 2.21.3 to 2.22.0.
Commits
  • 112e859 [maven-release-plugin] prepare release jackson-bom-2.22.0
  • 2cae2ce Prep for 2.22.0 release
  • 7955d21 Merge branch '2.21' into 2.x
  • 8922a05 Post-release dep version bump
  • 1fa9943 [maven-release-plugin] prepare for next development iteration
  • d1abd31 [maven-release-plugin] prepare release jackson-bom-2.21.4
  • 2aaea43 Prep for 2.21.4 release
  • 902ec69 Update Woodstox/stax2-api (to 7.2.0/4.3.0)
  • 2570647 Merge branch '2.21' into 2.x
  • 9d3a9d5 Post-release dep version bump
  • Additional commits viewable in compare view

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2e9b793562..a5ed136e95 100644 --- a/pom.xml +++ b/pom.xml @@ -101,7 +101,7 @@ under the License. 4.2.14.Final 1.81.0 4.35.0 - 2.21.3 + 2.22.0 3.5.0 25.2.10 1.12.1 From e525fbfe7f94301ea6b9725472a91206be0c7bd2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Jun 2026 14:31:56 +0200 Subject: [PATCH 242/271] MINOR: Bump com.diffplug.spotless:spotless-maven-plugin from 3.4.0 to 3.6.0 (#1172) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [com.diffplug.spotless:spotless-maven-plugin](https://github.com/diffplug/spotless) from 3.4.0 to 3.6.0.
Release notes

Sourced from com.diffplug.spotless:spotless-maven-plugin's releases.

Maven Plugin v3.6.0

Added

  • Add <cacheDirectory> to <eclipse>, <greclipse>, and <eclipseCdt> for the Equo/Solstice P2 cache. (#2944)
  • EclipseJdtFormtterStep now can conditionally set compiler source/compliance options. Allows for better parsing of AST Node for newer language features and more correct sorting; e.g. records or seal classes. (#2942)

Fixed

  • <versionCatalog> no longer splits long inline tables across multiple lines — Gradle's TOML 1.0 parser cannot read multi-line inline tables. The maxLineLength option has been removed. (#2948)
  • spotless:apply no longer aborts on the first file with lints; it now formats all files and reports a single aggregated lint failure across every file, matching the Gradle plugin's behavior. (#2937)
  • <greclipse> and <eclipseCdt> now default P2 data to the Maven local repository. (#2944)
  • forbidWildcardImports and forbidModuleImports now detect imports that have leading whitespace (indentation/tabs). (#2939)

Changes

  • Improved formatting performance by eliminating redundant per-step line-ending normalization in the core formatter loop. (#2934)

Maven Plugin v3.5.1

Fixed

  • <licenseHeader> with <yearMode>SET_FROM_GIT</yearMode> no longer runs git log through a shell, eliminating a shell-injection vector when formatting files whose names contain shell metacharacters.
  • Bump transitive plexus-utils 4.0.2 -> 4.0.3 to address CVE-2025-67030. (#2919)

Maven Plugin v3.5.0

Added

  • <scalafmt> now reads the version from the version field in the scalafmt config file when no <version> is explicitly set, falling back to the built-in default only if neither is available. (#2922)
  • Add <toml> format type with <versionCatalog> step for formatting and sorting Gradle version catalog files. (#2916)
  • Add <javaparserVersion> option to <cleanthat>, allowing users to override the JavaParser version pulled in transitively by Cleanthat. (#2903)
  • Add a expandWildcardImports API for java (#2829)

Fixed

  • Preserve case of JDBI named bind params that collide with SQL keywords (e.g. :limit, :offset) in the DBeaver SQL formatter. (#2899)
  • The -Dspotless.ratchetFrom=... user property now takes priority over <ratchetFrom> configured in the plugin or in individual formatters, instead of being overridden by them. (#2896, fixes #2842)
  • Fix non-idempotent formatting when importOrder() is combined with greclipse(): a single catch-all group no longer strips blank lines that greclipse() independently inserted between import groups. (#2914)

Changes

  • Fix expandWildcardImports failing on JDK XML types such as org.xml.sax.InputSource. (#2921)
  • Use Eclipse JDT's collator-based comparison when sorting Java members to better match Eclipse save actions. (#2920)
  • Bump default cleanthat version 2.24 -> 2.25. (#2903)
  • Bump default eclipse-jdt version from 4.35 to 4.39. (#2912)
Commits
  • 71a433c Published maven/3.6.0
  • 3a0f101 Published gradle/8.6.0
  • 007e9d8 Published lib/4.6.2
  • a074d53 Allow setting the local P2 cache dir in the Spotless Gradle plugin (#2944)
  • a266fc2 Merge branch 'main' into add-cache-directory-dsl
  • e0d466e Fix: sort members treats record declarations as types (#2942)
  • 3936b6f Merge branch 'main' into main
  • 278765f fix: expandWildcardImports support pom type dependency, fix #2839 (#2935)
  • a18ddec Remove maxLineLength from versionCatalog step (#2949)
  • b91ad87 Add changelog entries for versionCatalog maxLineLength removal
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.diffplug.spotless:spotless-maven-plugin&package-manager=maven&previous-version=3.4.0&new-version=3.6.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- bom/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index 083ee1e0de..3201761e81 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -208,7 +208,7 @@ under the License. com.diffplug.spotless spotless-maven-plugin - 3.4.0 + 3.6.0 org.codehaus.mojo diff --git a/pom.xml b/pom.xml index a5ed136e95..63128589a9 100644 --- a/pom.xml +++ b/pom.xml @@ -492,7 +492,7 @@ under the License. com.diffplug.spotless spotless-maven-plugin - 3.4.0 + 3.6.0 org.codehaus.mojo From cb24576e895c0bf8e1d062e2fc82f2b53bfa2eb8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Jun 2026 10:07:43 +0200 Subject: [PATCH 243/271] MINOR: Bump io.netty:netty-bom from 4.2.14.Final to 4.2.15.Final (#1175) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [io.netty:netty-bom](https://github.com/netty/netty) from 4.2.14.Final to 4.2.15.Final.
Release notes

Sourced from io.netty:netty-bom's releases.

netty-4.2.15.Final

Security fixes

  • CVE-2026-48059: memory exhaustion in io.netty:netty-codec-haproxy (high).
  • CVE-2026-47691: DNS cache poisoning in io.netty:netty-resolver-dns (high).
  • CVE-2026-50560: DDoS in io.netty:netty-codec-http2.
  • CVE-2026-50011: memory exhaustion in io.netty:netty-codec-redis (high).
  • CVE-2026-44250: memory exhaustion in io.netty:netty-codec-redis (high).
  • CVE-2026-44890: memory exhaustion in io.netty:netty-codec-redis (high).
  • CVE-2026-50009: information disclosure and denial of service in io.netty:netty-codec-classes-quic.
  • CVE-2026-44249: IPv6 subnet filter bypass in io.netty:netty-handler (high).
  • CVE-2026-50020: request smuggling in io.netty:netty-codec-http.
  • CVE-2026-44892: memory exhaustion in io.netty:netty-codec-http3 (high).
  • CVE-2026-44893: memory leak in io.netty:netty-codec-haproxy (high).
  • CVE-2026-44894: traffic amplification in io.netty:netty-codec-classes-quic (high).
  • CVE-2026-50010: TLS hostname verification accidentally disabled in io.netty:netty-handler (high).
  • CVE-2026-45673: DNS cache poisoning in io.netty:netty-resolver-dns.
  • CVE-2026-45416: excessive memory usage from SNIHandler in io.netty:netty-handler (high).
  • CVE-2026-45536: file descriptor leak in io.netty:netty-transport-native-epoll and io.netty:netty-transport-native-kqueue.
  • CVE-2026-45674: DNS cache poisoning in io.netty:netty-resolver-dns (high).
  • CVE-2026-46340: memory exhaustion in io.netty:netty-transport-sctp (high).
  • CVE-2026-47244: denial of service in io.netty:netty-codec-http2.
  • CVE-2026-48006: memory exhaustion in io.netty:netty-codec-redis (high).
  • CVE-2026-48748: memory exhaustion in io.netty:netty-codec-http3 (high).
  • CVE-2026-48043: memory exhaustion in io.netty:netty-codec-http2.

What's Changed

New Contributors

Full Changelog: https://github.com/netty/netty/compare/netty-4.2.14.Final...netty-4.2.15.Final

Commits
  • a41f7b2 [maven-release-plugin] prepare release netty-4.2.15.Final
  • 2394530 Auto-port 4.2: MQTT: Reject malformed no-payload packets with non-zero Remain...
  • 0bd1657 Add maxWindowLog parameter to ZstdDecoder to bound memory allocation (#16850)
  • 76291f5 Fix SCTP and Redis tests (#16893)
  • e067b6e Fix revapi warnings (#16885)
  • 5a52600 Pass maxAllocation to Brotli and Zstd decoders (#16844)
  • 541add0 Merge commit from fork
  • 270800e Merge commit from fork
  • 3d45a1e Merge commit from fork
  • 75127ca Merge commit from fork
  • Additional commits viewable in compare view

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 63128589a9..877a355860 100644 --- a/pom.xml +++ b/pom.xml @@ -98,7 +98,7 @@ under the License. 6.1.0 2.0.18 33.6.0-jre - 4.2.14.Final + 4.2.15.Final 1.81.0 4.35.0 2.22.0 From 5540c4d6358cd653fcc7e463e60aa3f21f3288ab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 26 Jun 2026 18:01:08 +0200 Subject: [PATCH 244/271] MINOR: Bump com.squareup.okhttp3:mockwebserver3-junit5 from 5.3.2 to 5.4.0 (#1185) Bumps [com.squareup.okhttp3:mockwebserver3-junit5](https://github.com/square/okhttp) from 5.3.2 to 5.4.0.
Changelog

Sourced from com.squareup.okhttp3:mockwebserver3-junit5's changelog.

Version 5.4.0

2026-06-08

  • New: Add superpowers to interceptors. Interceptors can now override anything settable on OkHttpClient.Builder, such as the cache, connection pool, socket factory, and DNS. We expect this will allow most users to use interceptors everywhere, insted of mixing and matching interceptors with custom Call.Factory wrappers.
  • Fix: Limit each HTTP/2 response to 256 KiB of total headers.
  • Upgrade: [kotlinx.coroutines 1.11.0][coroutines_1_11_0]. This is used by the optional okhttp-coroutines artifact.
  • Upgrade: [GraalVM 25.0.3][graalvm_25].
  • Upgrade: [Okio 3.17.0][okio_3_17_0].
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.squareup.okhttp3:mockwebserver3-junit5&package-manager=maven&previous-version=5.3.2&new-version=5.4.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 3741ee083c..9a8406cf31 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -129,7 +129,7 @@ under the License. com.squareup.okhttp3 mockwebserver3-junit5 - 5.3.2 + 5.4.0 test From 6e0b5f7905c436ef847eaba4a703564ee6faae7e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 26 Jun 2026 18:01:26 +0200 Subject: [PATCH 245/271] MINOR: Bump org.jacoco:jacoco-maven-plugin from 0.8.14 to 0.8.15 (#1176) Bumps [org.jacoco:jacoco-maven-plugin](https://github.com/jacoco/jacoco) from 0.8.14 to 0.8.15.
Release notes

Sourced from org.jacoco:jacoco-maven-plugin's releases.

0.8.15

New Features

  • JaCoCo now officially supports Java 26 (GitHub #2076).
  • Experimental support for Java 27 class files (GitHub #2004).
  • Compatibility methods generated by Kotlin compiler for functions defined in interfaces are filtered out during generation of report (GitHub #1905).
  • Compatibility methods generated by Kotlin compiler for exposed boxed inline value classes (JvmExposeBoxed annotation) are filtered out during generation of report (GitHub #1944).
  • Methods generated by the Kotlin compiler for functions with JvmStatic annotation are filtered out during generation of report (GitHub #2097).
  • Improved filtering of bytecode generated by Kotlin compiler for when expressions and statements with kotlin.String subject where first branch condition contains string with largest hash (GitHub #2098).
  • Part of bytecode that javac versions from 24 to 26 generate for switch statements and expressions with selector expression of type java.lang.String inside lambdas is filtered out during generation of report (GitHub #2023).
  • Improved performance of Kotlin files analysis by parsing SMAPs only once per class (GitHub #2114).
  • For better performance agent output methods tcpclient and tcpserver use BufferedOutputStream to write execution data to socket. Maven plugin, Ant tasks, CLI, API usage examples, and ExecDumpClient API use BufferedInputStream to read execution data from socket. Third-party integrations should do the same to benefit from this change in agent (GitHub #2089).

Fixed bugs

  • Fixed processing of Kotlin SMAP in synthetic classes (GitHub #1985).
  • Multiple JaCoCo runtimes within one JVM writing to the same output file should not cause data corruption when running on JDK versions from 6 to 10 affected by JDK-8166253 (GitHub #2065, #2074).
  • For better performance agent writes to output file via BufferedOutputStream, this fixes regression introduced in version 0.6.2 (GitHub #2073).
  • Fixed NullPointerException when JaCoCo agent is loaded by non system class loader, for example when loaded by JBoss Modules (GitHub #1651).

Non-functional Changes

  • JaCoCo now depends on ASM 9.10.1 (GitHub #2134).
Commits
  • 6c5260a Prepare release v0.8.15
  • 5c05141 Transfer of execution data through socket should use buffered stream (#2089)
  • ab5efa9 Remove from Azure Pipelines all builds except with JDK 5 and JDK EA (#2148)
  • 5f6ea38 Use Windows 2025 image in GitHub Actions (#2130)
  • 35a8af2 Use Renovate instead of Dependabot for updates of ASM (#2137)
  • 85b8ddf Upgrade ASM to 9.10.1 (#2134)
  • 2988647 AgentModule should use ClassLoader of agent instead of SystemClassLoader (#1651)
  • 75a4e31 Add filter for Kotlin @JvmExposeBoxed (#1944)
  • 691fa1d Use Renovate instead of Dependabot for updates of GitHub Actions (#2132)
  • 3e18f17 Require at least JDK 21 for build (#2128)
  • Additional commits viewable in compare view

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 877a355860..326b26ee97 100644 --- a/pom.xml +++ b/pom.xml @@ -352,7 +352,7 @@ under the License. org.jacoco jacoco-maven-plugin - 0.8.14 + 0.8.15

... (truncated)

Commits
  • 7b5e9ff Bump version to 1.82.1
  • 20768f1 Update README etc to reference 1.82.1
  • 5ab5eba kokoro: Remove extra / in architecture replacement
  • 6726caf buildscripts: add regional td config for psm-interop (v1.82.x backport) (#12864)
  • 022256f Bump version to 1.82.1-SNAPSHOT
  • 78fb519 Bump version to 1.82.0
  • b62b0fc Update README etc to reference 1.82.0
  • 8802dc3 build: downgrade multiarch to Ubuntu 20.04 and consolidate images (#12830)
  • be300bd kokoro: Avoid brew on Mac OS
  • 4111f6f core: throw IOException when ProxySelector returns null or empty list (#12793)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=io.grpc:grpc-bom&package-manager=maven&previous-version=1.81.0&new-version=1.82.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 326b26ee97..3aa3867b90 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,7 @@ under the License. 2.0.18 33.6.0-jre 4.2.15.Final - 1.81.0 + 1.82.1 4.35.0 2.22.0 3.5.0 From 2812cfaa4d455f202c9e11a1facdd83acb82530b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jun 2026 09:54:40 +0200 Subject: [PATCH 247/271] MINOR: Bump dep.junit.jupiter.version from 6.1.0 to 6.1.1 (#1200) Bumps `dep.junit.jupiter.version` from 6.1.0 to 6.1.1. Updates `org.junit.jupiter:junit-jupiter-engine` from 6.1.0 to 6.1.1
Release notes

Sourced from org.junit.jupiter:junit-jupiter-engine's releases.

JUnit 6.1.1 = Platform 6.1.1 + Jupiter 6.1.1 + Vintage 6.1.1

See Release Notes.

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.1.0...r6.1.1

Commits
  • 0d85889 Release 6.1.1
  • 0363eee Finalize 6.1.1 release notes
  • a6d540a Move entry to 6.1.1 release notes
  • 69339d5 Only pass timeout when publishing to avoid failure in nmcp plugin
  • dec2eb9 Allow excluding engines from memory cleanup mode (#5786)
  • a5f4270 Publish sha256/sha512 checksums again but filter out signature ones (#5796)
  • 8213012 Update plugin nmcp-settings to v1.6.0 (#5787)
  • d1bf847 Generate Javadoc for aggregator modules
  • d721de5 Pass --no-fonts to javadoc convention
  • d289ec6 Restore original SetSystemProperty values in a ParameterizedTest (#5720)
  • Additional commits viewable in compare view

Updates `org.junit.jupiter:junit-jupiter-api` from 6.1.0 to 6.1.1
Release notes

Sourced from org.junit.jupiter:junit-jupiter-api's releases.

JUnit 6.1.1 = Platform 6.1.1 + Jupiter 6.1.1 + Vintage 6.1.1

See Release Notes.

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.1.0...r6.1.1

Commits
  • 0d85889 Release 6.1.1
  • 0363eee Finalize 6.1.1 release notes
  • a6d540a Move entry to 6.1.1 release notes
  • 69339d5 Only pass timeout when publishing to avoid failure in nmcp plugin
  • dec2eb9 Allow excluding engines from memory cleanup mode (#5786)
  • a5f4270 Publish sha256/sha512 checksums again but filter out signature ones (#5796)
  • 8213012 Update plugin nmcp-settings to v1.6.0 (#5787)
  • d1bf847 Generate Javadoc for aggregator modules
  • d721de5 Pass --no-fonts to javadoc convention
  • d289ec6 Restore original SetSystemProperty values in a ParameterizedTest (#5720)
  • Additional commits viewable in compare view

Updates `org.junit.jupiter:junit-jupiter-params` from 6.1.0 to 6.1.1
Release notes

Sourced from org.junit.jupiter:junit-jupiter-params's releases.

JUnit 6.1.1 = Platform 6.1.1 + Jupiter 6.1.1 + Vintage 6.1.1

See Release Notes.

Full Changelog: https://github.com/junit-team/junit-framework/compare/r6.1.0...r6.1.1

Commits
  • 0d85889 Release 6.1.1
  • 0363eee Finalize 6.1.1 release notes
  • a6d540a Move entry to 6.1.1 release notes
  • 69339d5 Only pass timeout when publishing to avoid failure in nmcp plugin
  • dec2eb9 Allow excluding engines from memory cleanup mode (#5786)
  • a5f4270 Publish sha256/sha512 checksums again but filter out signature ones (#5796)
  • 8213012 Update plugin nmcp-settings to v1.6.0 (#5787)
  • d1bf847 Generate Javadoc for aggregator modules
  • d721de5 Pass --no-fonts to javadoc convention
  • d289ec6 Restore original SetSystemProperty values in a ParameterizedTest (#5720)
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3aa3867b90..fa493e066d 100644 --- a/pom.xml +++ b/pom.xml @@ -95,7 +95,7 @@ under the License. 1773644827 ${project.build.directory}/generated-sources 1.9.0 - 6.1.0 + 6.1.1 2.0.18 33.6.0-jre 4.2.15.Final From bf0b3445c5cac4f964665a5f28b4a80c4ad8d640 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jun 2026 16:57:05 +0900 Subject: [PATCH 248/271] MINOR: [CI] Bump actions/cache from 5 to 6 (#1197) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/cache](https://github.com/actions/cache) from 5 to 6.
Release notes

Sourced from actions/cache's releases.

v6.0.0

What's Changed

Full Changelog: https://github.com/actions/cache/compare/v5...v6.0.0

v5.1.0

What's Changed

Full Changelog: https://github.com/actions/cache/compare/v5...v5.1.0

v5.0.5

What's Changed

Full Changelog: https://github.com/actions/cache/compare/v5...v5.0.5

v5.0.4

What's Changed

New Contributors

Full Changelog: https://github.com/actions/cache/compare/v5...v5.0.4

v5.0.3

What's Changed

Full Changelog: https://github.com/actions/cache/compare/v5...v5.0.3

v.5.0.2

v5.0.2

What's Changed

... (truncated)

Commits
  • 55cc834 Merge pull request #1768 from jasongin/readonly-cache
  • d8cd72f Bump @​actions/cache to v6.1.0 - handle cache write error due to RO token
  • 2c8a9bd Merge pull request #1760 from actions/samirat/esm_migration_and_package_update
  • e9b91fd Prettier fixes
  • e4884b8 Rebuild dist
  • 10baf01 Fixed licenses
  • e39b386 Fix test mock return order
  • b692820 PR feedback
  • 6074912 Rebuild dist bundles as ESM to match type:module
  • 5a912e8 Fix lint and jest issues
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/cache&package-manager=github_actions&previous-version=5&new-version=6)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/dev.yml | 2 +- .github/workflows/rc.yml | 8 ++++---- .github/workflows/test.yml | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 2111f47254..f23c4e5e2f 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -42,7 +42,7 @@ jobs: with: python-version: '3.x' - name: pre-commit (cache) - uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cache/pre-commit key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index f0e99f1219..79c4476345 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -133,7 +133,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Cache - uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: .docker key: jni-linux-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} @@ -264,7 +264,7 @@ jobs: run: | echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} - name: Cache ccache - uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ccache key: jni-macos-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} @@ -340,7 +340,7 @@ jobs: run: | echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} - name: Cache ccache - uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ccache key: jni-windows-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} @@ -414,7 +414,7 @@ jobs: repository: apache/arrow-testing path: testing - name: Cache ~/.m2 - uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.m2 key: binaries-build-${{ hashFiles('**/*.java', '**/pom.xml') }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8c437a056d..1e4664237b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,7 +63,7 @@ jobs: fetch-depth: 0 submodules: recursive - name: Cache Docker Volumes - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: .docker key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('compose.yaml', '**/pom.xml', '**/*.java') }} @@ -190,7 +190,7 @@ jobs: run: | ci/scripts/util_free_space.sh - name: Cache Docker Volumes - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: .docker key: integration-conda-${{ hashFiles('cpp/**') }} From 365a5f46d1fe437d2337d705dd6a0043f2ac1703 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jun 2026 18:45:32 +0200 Subject: [PATCH 249/271] MINOR: [CI] Bump actions/checkout from 6 to 7 (#1193) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/checkout](https://github.com/actions/checkout) from 6 to 7.
Release notes

Sourced from actions/checkout's releases.

v7.0.0

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v6.0.3...v7.0.0

v6.0.3

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v6...v6.0.3

v6.0.2

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v6.0.1...v6.0.2

v6.0.1

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v6...v6.0.1

Changelog

Sourced from actions/checkout's changelog.

Changelog

v7.0.0

v6.0.3

v6.0.2

v6.0.1

v6.0.0

v5.0.1

v5.0.0

v4.3.1

v4.3.0

v4.2.2

v4.2.1

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/checkout&package-manager=github_actions&previous-version=6&new-version=7)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/dev.yml | 2 +- .github/workflows/dev_pr.yml | 2 +- .github/workflows/rc.yml | 20 ++++++++++---------- .github/workflows/release.yml | 2 +- .github/workflows/test.yml | 20 ++++++++++---------- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index f23c4e5e2f..25b08700fd 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -33,7 +33,7 @@ jobs: name: "pre-commit" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml index 20946c8185..ad000df88e 100644 --- a/.github/workflows/dev_pr.yml +++ b/.github/workflows/dev_pr.yml @@ -43,7 +43,7 @@ jobs: name: "Ensure PR format" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 79c4476345..fdfcd1cce3 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -38,7 +38,7 @@ jobs: timeout-minutes: 5 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: submodules: recursive - name: Prepare for tag @@ -113,17 +113,17 @@ jobs: ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ if: github.event_name == 'schedule' - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: apache/arrow path: arrow - name: Checkout apache/arrow-testing - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: apache/arrow-testing path: arrow/testing - name: Checkout apache/parquet-testing - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing @@ -180,17 +180,17 @@ jobs: ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ if: github.event_name == 'schedule' - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: apache/arrow path: arrow - name: Checkout apache/arrow-testing - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: apache/arrow-testing path: arrow/testing - name: Checkout apache/parquet-testing - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing @@ -310,7 +310,7 @@ jobs: ci/scripts/download_cpp.sh - name: Checkout Apache Arrow C++ if: github.event_name == 'schedule' - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: apache/arrow path: arrow @@ -409,7 +409,7 @@ jobs: test -f jni/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll test -f jni/arrow_orc_jni/x86_64/arrow_orc_jni.dll - name: Checkout apache/arrow-testing - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: apache/arrow-testing path: testing @@ -497,7 +497,7 @@ jobs: contents: write steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: path: site - name: Prepare branch diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a2c5a55544..7692eb6cbe 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -65,7 +65,7 @@ jobs: $artifact done - name: Checkout for publishing docs - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: path: site - name: Publish docs diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1e4664237b..ac8080d075 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,7 +58,7 @@ jobs: MAVEN: ${{ matrix.maven }} steps: - name: Checkout Arrow - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: fetch-depth: 0 submodules: recursive @@ -100,7 +100,7 @@ jobs: distribution: 'temurin' java-version: ${{ matrix.jdk }} - name: Checkout Arrow - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: fetch-depth: 0 submodules: recursive @@ -131,7 +131,7 @@ jobs: java-version: ${{ matrix.jdk }} distribution: 'temurin' - name: Checkout Arrow - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: fetch-depth: 0 submodules: recursive @@ -152,37 +152,37 @@ jobs: timeout-minutes: 60 steps: - name: Checkout Arrow - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: fetch-depth: 0 repository: apache/arrow submodules: recursive - name: Checkout Arrow Rust - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: apache/arrow-rs path: rust - name: Checkout Arrow nanoarrow - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: apache/arrow-nanoarrow path: nanoarrow - name: Checkout Arrow .NET - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: apache/arrow-dotnet path: dotnet - name: Checkout Arrow Go - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: apache/arrow-go path: go - name: Checkout Arrow Java - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: path: java - name: Checkout Arrow JavaScript - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: apache/arrow-js path: js From 9122af2ad87c6f51d8ed9082ba2f0a122980ea2a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jun 2026 18:45:52 +0200 Subject: [PATCH 250/271] MINOR: Bump com.google.protobuf:protobuf-bom from 4.35.0 to 4.35.1 (#1186) Bumps [com.google.protobuf:protobuf-bom](https://github.com/protocolbuffers/protobuf) from 4.35.0 to 4.35.1.
Commits

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fa493e066d..c39eafc209 100644 --- a/pom.xml +++ b/pom.xml @@ -100,7 +100,7 @@ under the License. 33.6.0-jre 4.2.15.Final 1.82.1 - 4.35.0 + 4.35.1 2.22.0 3.5.0 25.2.10 From b0e51af855ef5befac903b1a6de3f535258ccd1b Mon Sep 17 00:00:00 2001 From: YangJie Date: Tue, 30 Jun 2026 12:40:01 +0800 Subject: [PATCH 251/271] GH-1116: [Java] Fix compressed buffer prefix write and ZSTD dstCapacity (#1119) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Two fixes in the compression codec: 1. **`AbstractCompressionCodec.compress()`: capture `writerIndex()` once** The previous code read `uncompressedBuffer.writerIndex()` at multiple sites — for the size comparison and again after `doCompress()` to populate the 8-byte uncompressed-length prefix. Capture the value once at the top of `compress()` and reuse it for the empty-buffer check, the size comparison, and the prefix, so all three consumers see the same value. 2. **`ZstdCompressionCodec.doCompress()`: `dstCapacity` overstated by 8 bytes** `Zstd.compressUnsafe(dst, dstSize, ...)` expects `dstSize` to be the available space from `dst`. The code offsets `dst` by 8 bytes past the prefix but passed `8 + maxSize` instead of `maxSize`. The `compressBound()` headroom hides this in practice, but the parameter was semantically wrong. Pass `maxSize`. ## Tests Covered by the existing round-trip tests (`testEmptyBuffer`, `testReadWriteStream`, `testReadWriteFile`, etc.). I was not able to construct a minimal reproducer for the original `declaredUncompressed=0` symptom on the unfixed code, so both fixes are conservative correctness improvements derived from code inspection rather than failing-then-green regression tests. --- .../org/apache/arrow/compression/ZstdCompressionCodec.java | 2 +- .../arrow/vector/compression/AbstractCompressionCodec.java | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/compression/src/main/java/org/apache/arrow/compression/ZstdCompressionCodec.java b/compression/src/main/java/org/apache/arrow/compression/ZstdCompressionCodec.java index 290723608d..ed46fe81b4 100644 --- a/compression/src/main/java/org/apache/arrow/compression/ZstdCompressionCodec.java +++ b/compression/src/main/java/org/apache/arrow/compression/ZstdCompressionCodec.java @@ -44,7 +44,7 @@ protected ArrowBuf doCompress(BufferAllocator allocator, ArrowBuf uncompressedBu long bytesWritten = Zstd.compressUnsafe( compressedBuffer.memoryAddress() + CompressionUtil.SIZE_OF_UNCOMPRESSED_LENGTH, - dstSize, + maxSize, /*src*/ uncompressedBuffer.memoryAddress(), /* srcSize= */ uncompressedBuffer.writerIndex(), /* level= */ this.compressionLevel); diff --git a/vector/src/main/java/org/apache/arrow/vector/compression/AbstractCompressionCodec.java b/vector/src/main/java/org/apache/arrow/vector/compression/AbstractCompressionCodec.java index 58d9e4db9b..b108173c82 100644 --- a/vector/src/main/java/org/apache/arrow/vector/compression/AbstractCompressionCodec.java +++ b/vector/src/main/java/org/apache/arrow/vector/compression/AbstractCompressionCodec.java @@ -29,7 +29,11 @@ public abstract class AbstractCompressionCodec implements CompressionCodec { @Override public ArrowBuf compress(BufferAllocator allocator, ArrowBuf uncompressedBuffer) { - if (uncompressedBuffer.writerIndex() == 0L) { + // GH-1116: capture writerIndex() once so the empty-buffer check, size + // comparison, and uncompressed-length prefix all see the same value. + long uncompressedLength = uncompressedBuffer.writerIndex(); + + if (uncompressedLength == 0L) { // shortcut for empty buffer ArrowBuf compressedBuffer = allocator.buffer(CompressionUtil.SIZE_OF_UNCOMPRESSED_LENGTH); compressedBuffer.setLong(0, 0); @@ -41,7 +45,6 @@ public ArrowBuf compress(BufferAllocator allocator, ArrowBuf uncompressedBuffer) ArrowBuf compressedBuffer = doCompress(allocator, uncompressedBuffer); long compressedLength = compressedBuffer.writerIndex() - CompressionUtil.SIZE_OF_UNCOMPRESSED_LENGTH; - long uncompressedLength = uncompressedBuffer.writerIndex(); if (compressedLength > uncompressedLength) { // compressed buffer is larger, send the raw buffer From 301d3d8f889d3e4d1f45fc6e1707460ae5baaeb4 Mon Sep 17 00:00:00 2001 From: Pedro Matias Date: Tue, 30 Jun 2026 05:41:30 +0100 Subject: [PATCH 252/271] GH-1063: Add is_update field to ActionCreatePreparedStatementResult (#1064) ## What's Changed A new field, `optional bool is_update = 4;`, was added to `message ActionCreatePreparedStatementResult`. When this field is sent by the server, its value indicates whether the proper network flow to execute the query that the driver should follow uses `CommandPreparedStatementQuery` or `CommandPreparedStatementUpdate`. For outdated servers that don't send the field, the driver maintains its current behavior of using `CommandPreparedStatementQuery` when the `dataset_schema` is not empty, thus ensuring the backward compatibility of the new driver with old servers. This change was created with AI assistance (Augment Code and Claude code). All lines were manually reviewed by a human. The output is not copyrightable subject matter. - Closes #1063 --------- Co-authored-by: David Li --- arrow-format/FlightSql.proto | 5 ++ .../ArrowFlightJdbcFlightStreamResultSet.java | 2 +- ...owFlightJdbcVectorSchemaRootResultSet.java | 2 +- .../driver/jdbc/ArrowFlightMetaImpl.java | 23 +++++-- .../client/ArrowFlightSqlClientHandler.java | 19 ++++++ .../ArrowFlightPreparedStatementTest.java | 48 ++++++++++++++ .../jdbc/ArrowFlightStatementExecuteTest.java | 66 +++++++++++++++++++ .../jdbc/utils/MockFlightSqlProducer.java | 42 ++++++++++++ .../arrow/flight/sql/FlightSqlClient.java | 13 ++++ 9 files changed, 212 insertions(+), 8 deletions(-) diff --git a/arrow-format/FlightSql.proto b/arrow-format/FlightSql.proto index 566230c2a6..b1dc57b33b 100644 --- a/arrow-format/FlightSql.proto +++ b/arrow-format/FlightSql.proto @@ -1550,6 +1550,11 @@ message ActionCreatePreparedStatementResult { // If the query provided contained parameters, parameter_schema contains the // schema of the expected parameters. It should be an IPC-encapsulated Schema, as described in Schema.fbs. bytes parameter_schema = 3; + + // When set to true, the query should be executed with CommandPreparedStatementUpdate, + // when set to false, the query should be executed with CommandPreparedStatementQuery. + // If not set, the client can choose how to execute the query. + optional bool is_update = 4; } /* diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcFlightStreamResultSet.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcFlightStreamResultSet.java index 2885f7895b..376e5b11e7 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcFlightStreamResultSet.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcFlightStreamResultSet.java @@ -106,7 +106,7 @@ static ArrowFlightJdbcFlightStreamResultSet fromFlightInfo( final TimeZone timeZone = TimeZone.getDefault(); final QueryState state = new QueryState(); - final Meta.Signature signature = ArrowFlightMetaImpl.newSignature(null, null, null); + final Meta.Signature signature = ArrowFlightMetaImpl.newSignature(null, null, null, null); final AvaticaResultSetMetaData resultSetMetaData = new AvaticaResultSetMetaData(null, null, signature); diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java index 5d02d6e843..ad6670a001 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcVectorSchemaRootResultSet.java @@ -73,7 +73,7 @@ public static ArrowFlightJdbcVectorSchemaRootResultSet fromVectorSchemaRoot( final TimeZone timeZone = TimeZone.getDefault(); final QueryState state = new QueryState(); - final Meta.Signature signature = ArrowFlightMetaImpl.newSignature(null, null, null); + final Meta.Signature signature = ArrowFlightMetaImpl.newSignature(null, null, null, null); final AvaticaResultSetMetaData resultSetMetaData = new AvaticaResultSetMetaData(null, null, signature); diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java index 64529b50c8..0d85b5eddb 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java @@ -53,7 +53,8 @@ public ArrowFlightMetaImpl(final AvaticaConnection connection) { } /** Construct a signature. */ - static Signature newSignature(final String sql, Schema resultSetSchema, Schema parameterSchema) { + static Signature newSignature( + final String sql, Schema resultSetSchema, Schema parameterSchema, Boolean isUpdate) { List columnMetaData = resultSetSchema == null ? new ArrayList<>() @@ -62,10 +63,17 @@ static Signature newSignature(final String sql, Schema resultSetSchema, Schema p parameterSchema == null ? new ArrayList<>() : ConvertUtils.convertArrowFieldsToAvaticaParameters(parameterSchema.getFields()); - StatementType statementType = - resultSetSchema == null || resultSetSchema.getFields().isEmpty() - ? StatementType.IS_DML - : StatementType.SELECT; + // If the server provided the is_update field, use it to determine the statement type + StatementType statementType; + if (isUpdate != null) { + statementType = isUpdate ? StatementType.IS_DML : StatementType.SELECT; + } else { + // Fall back to the legacy logic: check if the result set schema is empty + statementType = + resultSetSchema == null || resultSetSchema.getFields().isEmpty() + ? StatementType.IS_DML + : StatementType.SELECT; + } return new Signature( columnMetaData, sql, @@ -178,7 +186,10 @@ private PreparedStatement prepareForHandle(final String query, StatementHandle h ((ArrowFlightConnection) connection).getClientHandler().prepare(query); handle.signature = newSignature( - query, preparedStatement.getDataSetSchema(), preparedStatement.getParameterSchema()); + query, + preparedStatement.getDataSetSchema(), + preparedStatement.getParameterSchema(), + preparedStatement.isUpdate()); statementHandlePreparedStatementMap.put(new StatementHandleKey(handle), preparedStatement); return preparedStatement; } diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java index f0ea284239..719cc38a2b 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/client/ArrowFlightSqlClientHandler.java @@ -388,6 +388,14 @@ public interface PreparedStatement extends AutoCloseable { */ Schema getParameterSchema(); + /** + * Gets whether this {@link PreparedStatement} is an update statement. + * + * @return {@code true} if this is an update statement, {@code false} if it's a query, or {@code + * null} if the server did not provide this information. + */ + @Nullable Boolean isUpdate(); + void setParameters(VectorSchemaRoot parameters); @Override @@ -456,6 +464,12 @@ public long executeUpdate() { @Override public StatementType getType() { + // If the server provided the is_update field, use it to determine the statement type + final Boolean isUpdate = preparedStatement.isUpdate(); + if (isUpdate != null) { + return isUpdate ? StatementType.UPDATE : StatementType.SELECT; + } + // Fall back to the legacy logic: check if the result set schema is empty final Schema schema = preparedStatement.getResultSetSchema(); return schema.getFields().isEmpty() ? StatementType.UPDATE : StatementType.SELECT; } @@ -475,6 +489,11 @@ public void setParameters(VectorSchemaRoot parameters) { preparedStatement.setParameters(parameters); } + @Override + public Boolean isUpdate() { + return preparedStatement.isUpdate(); + } + @Override public void close() { try { diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightPreparedStatementTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightPreparedStatementTest.java index 0369c3a162..078837adf3 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightPreparedStatementTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightPreparedStatementTest.java @@ -98,6 +98,40 @@ public void testSimpleQueryNoParameterBindingWithExecute() throws SQLException { } } + @Test + public void testSimpleQueryNoParameterBindingWithExecuteV2() throws SQLException { + final String query = "SELECT * FROM TEST_V2"; + final Schema schema = + new Schema(Collections.singletonList(Field.nullable("", Types.MinorType.INT.getType()))); + PRODUCER.addSelectQuery( + query, + schema, + Collections.singletonList( + listener -> { + try (final BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { + root.allocateNew(); + ((IntVector) root.getVector(0)).setSafe(0, 123); + root.setRowCount(1); + listener.start(root); + listener.putNext(); + } finally { + listener.completed(); + } + }), + false); + try (final PreparedStatement preparedStatement = connection.prepareStatement(query)) { + boolean isResultSet = preparedStatement.execute(); + assertTrue(isResultSet); + final ResultSet resultSet = preparedStatement.getResultSet(); + assertTrue(resultSet.next()); + assertEquals(123, resultSet.getInt(1)); + assertFalse(resultSet.next()); + assertFalse(preparedStatement.getMoreResults()); + assertEquals(-1, preparedStatement.getUpdateCount()); + } + } + @Test public void testQueryWithParameterBinding() throws SQLException { final String query = "Fake query with parameters"; @@ -203,6 +237,20 @@ public void testUpdateQueryWithExecute() throws SQLException { } } + @Test + public void testUpdateQueryWithExecuteV2() throws SQLException { + String query = "Fake update with execute V2"; + PRODUCER.addUpdateQuery(query, /*updatedRows*/ 99, true); + try (final PreparedStatement stmt = connection.prepareStatement(query)) { + boolean isResultSet = stmt.execute(); + assertFalse(isResultSet); + int updated = stmt.getUpdateCount(); + assertEquals(99, updated); + assertFalse(stmt.getMoreResults()); + assertEquals(-1, stmt.getUpdateCount()); + } + } + @Test public void testUpdateQueryWithParameters() throws SQLException { String query = "Fake update with parameters"; diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightStatementExecuteTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightStatementExecuteTest.java index 632cb0ba56..6acce9c2a6 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightStatementExecuteTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightStatementExecuteTest.java @@ -62,6 +62,9 @@ public class ArrowFlightStatementExecuteTest { private static final String SAMPLE_LARGE_UPDATE_QUERY = "UPDATE this_large_table SET this_large_field = that_large_field FROM this_large_test WHERE this_large_condition"; private static final long SAMPLE_LARGE_UPDATE_COUNT = Long.MAX_VALUE; + private static final String SAMPLE_QUERY_CMD_V2 = "SELECT * FROM this_test_v2"; + private static final String SAMPLE_LARGE_UPDATE_QUERY_V2 = + "UPDATE this_large_table_v2 SET this_large_field = that_large_field FROM this_large_test WHERE this_large_condition"; private static final MockFlightSqlProducer PRODUCER = new MockFlightSqlProducer(); @RegisterExtension @@ -96,6 +99,31 @@ public static void setUpBeforeClass() { })); PRODUCER.addUpdateQuery(SAMPLE_UPDATE_QUERY, SAMPLE_UPDATE_COUNT); PRODUCER.addUpdateQuery(SAMPLE_LARGE_UPDATE_QUERY, SAMPLE_LARGE_UPDATE_COUNT); + + // V2 queries with is_update field set + PRODUCER.addSelectQuery( + SAMPLE_QUERY_CMD_V2, + SAMPLE_QUERY_SCHEMA, + Collections.singletonList( + listener -> { + try (final BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + final VectorSchemaRoot root = + VectorSchemaRoot.create(SAMPLE_QUERY_SCHEMA, allocator)) { + final UInt1Vector vector = (UInt1Vector) root.getVector(VECTOR_NAME); + IntStream.range(0, SAMPLE_QUERY_ROWS) + .forEach(index -> vector.setSafe(index, index)); + vector.setValueCount(SAMPLE_QUERY_ROWS); + root.setRowCount(SAMPLE_QUERY_ROWS); + listener.start(root); + listener.putNext(); + } catch (final Throwable throwable) { + listener.error(throwable); + } finally { + listener.completed(); + } + }), + false); + PRODUCER.addUpdateQuery(SAMPLE_LARGE_UPDATE_QUERY_V2, SAMPLE_LARGE_UPDATE_COUNT, true); } @BeforeEach @@ -168,4 +196,42 @@ public void testUpdateCountShouldStartOnZero() throws SQLException { is(allOf(equalTo(statement.getLargeUpdateCount()), equalTo(0L)))); assertThat(statement.getResultSet(), is(nullValue())); } + + @Test + public void testExecuteShouldRunSelectQueryV2() throws SQLException { + assertThat(statement.execute(SAMPLE_QUERY_CMD_V2), is(true)); + final Set numbers = + IntStream.range(0, SAMPLE_QUERY_ROWS) + .boxed() + .map(Integer::byteValue) + .collect(Collectors.toCollection(HashSet::new)); + try (final ResultSet resultSet = statement.getResultSet()) { + final int columnCount = resultSet.getMetaData().getColumnCount(); + assertThat(columnCount, is(1)); + int rowCount = 0; + for (; resultSet.next(); rowCount++) { + assertThat(numbers.remove(resultSet.getByte(1)), is(true)); + } + assertThat(rowCount, is(equalTo(SAMPLE_QUERY_ROWS))); + } + assertThat(numbers, is(Collections.emptySet())); + assertThat( + (long) statement.getUpdateCount(), + is(allOf(equalTo(statement.getLargeUpdateCount()), equalTo(-1L)))); + } + + @Test + public void testExecuteShouldRunUpdateQueryForLargeUpdateV2() throws SQLException { + assertThat(statement.execute(SAMPLE_LARGE_UPDATE_QUERY_V2), is(false)); // UPDATE query. + final long updateCountSmall = statement.getUpdateCount(); + final long updateCountLarge = statement.getLargeUpdateCount(); + assertThat(updateCountLarge, is(equalTo(SAMPLE_LARGE_UPDATE_COUNT))); + assertThat( + updateCountSmall, + is( + allOf( + equalTo((long) AvaticaUtils.toSaturatedInt(updateCountLarge)), + not(equalTo(updateCountLarge))))); + assertThat(statement.getResultSet(), is(nullValue())); + } } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/MockFlightSqlProducer.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/MockFlightSqlProducer.java index 45c2a96404..6627d91ab6 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/MockFlightSqlProducer.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/MockFlightSqlProducer.java @@ -87,6 +87,7 @@ import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.JsonStringArrayList; import org.apache.calcite.avatica.Meta.StatementType; +import org.checkerframework.checker.nullness.qual.Nullable; /** An ad-hoc {@link FlightSqlProducer} for tests. */ public final class MockFlightSqlProducer implements FlightSqlProducer { @@ -101,6 +102,7 @@ public final class MockFlightSqlProducer implements FlightSqlProducer { private final SqlInfoBuilder sqlInfoBuilder = new SqlInfoBuilder(); private final Map parameterSchemas = new HashMap<>(); private final Map>> expectedParameterValues = new HashMap<>(); + private final Map isUpdateMap = new HashMap<>(); private final Map actionTypeCounter = new HashMap<>(); @@ -176,6 +178,40 @@ public void addUpdateQuery(final String sqlCommand, final long updatedRows) { }); } + /** + * Registers a new {@link StatementType#SELECT} SQL query, optionally setting the is_update field. + * + * @param sqlCommand the SQL command under which to register the new query. + * @param schema the schema to use for the query result. + * @param resultProviders the result provider for this query. + * @param isUpdate value to report for the is_update field, or {@code null} to leave it unset. + */ + public void addSelectQuery( + final String sqlCommand, + final Schema schema, + final List> resultProviders, + final @Nullable Boolean isUpdate) { + addSelectQuery(sqlCommand, schema, resultProviders); + if (isUpdate != null) { + isUpdateMap.put(sqlCommand, isUpdate); + } + } + + /** + * Registers a new {@link StatementType#UPDATE} SQL query, optionally setting the is_update field. + * + * @param sqlCommand the SQL command. + * @param updatedRows the number of rows affected. + * @param isUpdate value to report for the is_update field, or {@code null} to leave it unset. + */ + public void addUpdateQuery( + final String sqlCommand, final long updatedRows, final @Nullable Boolean isUpdate) { + addUpdateQuery(sqlCommand, updatedRows); + if (isUpdate != null) { + isUpdateMap.put(sqlCommand, isUpdate); + } + } + /** * Adds a catalog query to the results. * @@ -247,6 +283,12 @@ public void createPreparedStatement( resultBuilder.setParameterSchema(ByteString.copyFrom(outputStream.toByteArray())); } + // Set is_update field if present + final Boolean isUpdate = isUpdateMap.get(query); + if (isUpdate != null) { + resultBuilder.setIsUpdate(isUpdate); + } + listener.onNext(new Result(pack(resultBuilder.build()).toByteArray())); } catch (final Throwable t) { listener.onError(t); diff --git a/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlClient.java b/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlClient.java index 623f9311e8..0af09faee1 100644 --- a/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlClient.java +++ b/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlClient.java @@ -1284,6 +1284,19 @@ public Schema getParameterSchema() { return parameterSchema; } + /** + * Returns whether the server indicated this prepared statement is an update query. + * + * @return true if the server indicated this is an update query, false if the server indicated + * this is a select query, or null if the server did not provide this information. + */ + public Boolean isUpdate() { + if (preparedStatementResult.hasIsUpdate()) { + return preparedStatementResult.getIsUpdate(); + } + return null; + } + /** Get the schema of the result set (should be identical to {@link #getResultSetSchema()}). */ public SchemaResult fetchSchema(CallOption... options) { checkOpen(); From fcba1b0345faed30621863af85a102c6d2fc1362 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Jul 2026 07:51:48 +0200 Subject: [PATCH 253/271] MINOR: Bump org.cyclonedx:cyclonedx-maven-plugin from 2.9.1 to 2.9.2 (#1198) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.cyclonedx:cyclonedx-maven-plugin](https://github.com/CycloneDX/cyclonedx-maven-plugin) from 2.9.1 to 2.9.2.
Release notes

Sourced from org.cyclonedx:cyclonedx-maven-plugin's releases.

2.9.2

🚀 New features and improvements

  • chore: upgrade maven-dependency-analyzer/asm, support Java 25 (#630) @​shihyuho

📦 Dependency updates

🔧 Build

Commits
  • 0fe189d [maven-release-plugin] prepare release cyclonedx-maven-plugin-2.9.2
  • 96c218c update scm urls
  • 0fe08b4 Revert "Bump JamesIves/github-pages-deploy-action from 4.7.3 to 4.8.0"
  • 6779e48 Revert "Bump release-drafter/release-drafter from 6 to 7"
  • 955fead switch to Central Publishing Portal
  • 50dbac7 Bump release-drafter/release-drafter from 6 to 7
  • d50bc58 Bump org.apache.maven.plugins:maven-project-info-reports-plugin
  • 1034644 Bump plugin-tools.version from 3.15.0 to 3.15.2
  • 018ab8e Bump commons-codec:commons-codec from 1.17.1 to 1.22.0
  • e359705 Bump JamesIves/github-pages-deploy-action from 4.7.3 to 4.8.0
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.cyclonedx:cyclonedx-maven-plugin&package-manager=maven&previous-version=2.9.1&new-version=2.9.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c39eafc209..6a97ac4620 100644 --- a/pom.xml +++ b/pom.xml @@ -522,7 +522,7 @@ under the License. org.cyclonedx cyclonedx-maven-plugin - 2.9.1 + 2.9.2 org.apache.drill.tools From bd8cd52fc8426b485a8b3e64997fb178c84c9de2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Jul 2026 07:54:25 +0200 Subject: [PATCH 254/271] MINOR: Bump com.github.luben:zstd-jni from 1.5.7-10 to 1.5.7-11 (#1184) Bumps [com.github.luben:zstd-jni](https://github.com/luben/zstd-jni) from 1.5.7-10 to 1.5.7-11.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.github.luben:zstd-jni&package-manager=maven&previous-version=1.5.7-10&new-version=1.5.7-11)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- compression/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compression/pom.xml b/compression/pom.xml index 41cdb03796..aa7dee6f89 100644 --- a/compression/pom.xml +++ b/compression/pom.xml @@ -55,7 +55,7 @@ under the License. com.github.luben zstd-jni - 1.5.7-10 + 1.5.7-11 From 04471f4f8900e0a91efe0d6ae717c3211aa5efbd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Jul 2026 08:11:17 +0200 Subject: [PATCH 255/271] MINOR: Bump logback.version from 1.5.32 to 1.5.34 (#1171) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps `logback.version` from 1.5.32 to 1.5.34. Updates `ch.qos.logback:logback-classic` from 1.5.32 to 1.5.34
Release notes

Sourced from ch.qos.logback:logback-classic's releases.

Logback 1.5.34

2026-06-01 Release of logback version 1.5.34

• In case certain StackTraceElement values returned by the Throwable.getStackTrace method are null, StackTraceElementProxy substitutes a dummy instance instead of throwing an IllegalArgumentException. This resolves [issues #1040](qos-ch/logback#1040), reported by Naotsugu Kobayashi.

• HardenedObjectInputStream will now throw an InvalidClassException during deserialization attempts of Proxy classes. This change addresses potential deserialization whitelist bypass vulnerability reported by York Shen and registered as CVE-2026-10532.

• A bitwise identical binary of this version can be reproduced by building from source code at commit e62272ac152469aec1ede056c3c7d0d7314e7bfe associated with the tag v_1.5.34. This release was built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Logback 1.5.33

2026-05-27 Release of logback version 1.5.33

• PropertiesConfiguratorModelHandler now registers properties file URLs to the ConfigurationWatchList when scan is enabled (via local scan="true" attribute or top-level configuration scan), ensuring changes are detected and reconfiguration occurs. This problem was reported in issues/1034.

• When processing <conversionRule> elements and both class and converterClass attributes are specified, silently use the class attribute without issuing a warning. However, if the attribute values differ, a warning will be issued. This change was requested in issues/1031.

• HardenedModelInputStream will no longer accept to deserialize all classes located under the "java.lang" and "java.util" packages but a limited number of explicitly authorized classes in those packages. This potential deserialization whitelist bypass vulnerability was reported by York Shen and registered as CVE-2026-9828.

• SSL parameters for SSLSocketAppender now enable hostname verification by default. Moreover, the default protocol is now "TLSv1.2". This potential vulnerability was reported by York Shen.

• When printing the status message field, ViewStatusMessagesServletBase now escapes special characters such as "&" as character entities. This potential vulnerability was reported by York Shen.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit 124e8b49b55ac34d08743a0646bd463410192647 associated with the tag v_1.5.33. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits
  • e62272a prepare release 1.5.34
  • 1e9e926 add resolveProxyClassRejectsDynamicProxies unit test
  • 2de5cbe added StackTraceElementProxyTest, minor edits to AGENTS.md
  • 0e9b927 in case StackTraceElement is null use a substitute, fixing issues/1040
  • f7a0654 prevent resolveProxyClass bypass
  • 249b81f docs are no longer distributed
  • 1c3b26a start work on 1.5.34-SNAPSHOT
  • 124e8b4 prepare release 1.5.33
  • d8fd6f2 escapeTags in message field when printing status messages
  • 95edbeb hostnameVerification default to true in SSLParametersConfiguration, SSL.DEFAU...
  • Additional commits viewable in compare view

Updates `ch.qos.logback:logback-core` from 1.5.32 to 1.5.34
Release notes

Sourced from ch.qos.logback:logback-core's releases.

Logback 1.5.34

2026-06-01 Release of logback version 1.5.34

• In case certain StackTraceElement values returned by the Throwable.getStackTrace method are null, StackTraceElementProxy substitutes a dummy instance instead of throwing an IllegalArgumentException. This resolves [issues #1040](qos-ch/logback#1040), reported by Naotsugu Kobayashi.

• HardenedObjectInputStream will now throw an InvalidClassException during deserialization attempts of Proxy classes. This change addresses potential deserialization whitelist bypass vulnerability reported by York Shen and registered as CVE-2026-10532.

• A bitwise identical binary of this version can be reproduced by building from source code at commit e62272ac152469aec1ede056c3c7d0d7314e7bfe associated with the tag v_1.5.34. This release was built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Logback 1.5.33

2026-05-27 Release of logback version 1.5.33

• PropertiesConfiguratorModelHandler now registers properties file URLs to the ConfigurationWatchList when scan is enabled (via local scan="true" attribute or top-level configuration scan), ensuring changes are detected and reconfiguration occurs. This problem was reported in issues/1034.

• When processing <conversionRule> elements and both class and converterClass attributes are specified, silently use the class attribute without issuing a warning. However, if the attribute values differ, a warning will be issued. This change was requested in issues/1031.

• HardenedModelInputStream will no longer accept to deserialize all classes located under the "java.lang" and "java.util" packages but a limited number of explicitly authorized classes in those packages. This potential deserialization whitelist bypass vulnerability was reported by York Shen and registered as CVE-2026-9828.

• SSL parameters for SSLSocketAppender now enable hostname verification by default. Moreover, the default protocol is now "TLSv1.2". This potential vulnerability was reported by York Shen.

• When printing the status message field, ViewStatusMessagesServletBase now escapes special characters such as "&" as character entities. This potential vulnerability was reported by York Shen.

• A bit-wise identical binary of this version can be reproduced by building from source code at commit 124e8b49b55ac34d08743a0646bd463410192647 associated with the tag v_1.5.33. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits
  • e62272a prepare release 1.5.34
  • 1e9e926 add resolveProxyClassRejectsDynamicProxies unit test
  • 2de5cbe added StackTraceElementProxyTest, minor edits to AGENTS.md
  • 0e9b927 in case StackTraceElement is null use a substitute, fixing issues/1040
  • f7a0654 prevent resolveProxyClass bypass
  • 249b81f docs are no longer distributed
  • 1c3b26a start work on 1.5.34-SNAPSHOT
  • 124e8b4 prepare release 1.5.33
  • d8fd6f2 escapeTags in message field when printing status messages
  • 95edbeb hostnameVerification default to true in SSLParametersConfiguration, SSL.DEFAU...
  • Additional commits viewable in compare view

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6a97ac4620..231720eb44 100644 --- a/pom.xml +++ b/pom.xml @@ -113,7 +113,7 @@ under the License. true 2.42.0 4.2.0 - 1.5.32 + 1.5.34 none -Xdoclint:none From 8ce39730fe969b41c8620b02f3fa77c974d90784 Mon Sep 17 00:00:00 2001 From: Jordan Epstein <32082339+jordepic@users.noreply.github.com> Date: Thu, 2 Jul 2026 18:16:18 -0500 Subject: [PATCH 256/271] GH-1179: Correct the size of var-width vector with >0 start offset during vector append (#1180) ## What's Changed Fix VectorAppender data size computation for variable-width vectors with non-zero start offsets When appending a variable width offset vector in DataFusion comet I was receiving exceptions due to allocating too much memory. This is because Comet passes variable width arrays back to Java where the initial offset vector entry is greater than 0. Prior to this change, arrow-java determines how many bytes to copy by just looking at the last offset entry in the buffer, completely disregarding the value of the first. If first = 100 and last = 200, Java will still copy 200 bytes instead of 100. In this change we fix that. Closes #1179 --------- Co-authored-by: Jordan Epstein --- .../arrow/vector/util/VectorAppender.java | 92 ++++++--- .../arrow/vector/util/TestVectorAppender.java | 189 ++++++++++++++++++ 2 files changed, 257 insertions(+), 24 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java b/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java index e7c0d11cb9..2cfeb0a04d 100644 --- a/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java +++ b/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java @@ -125,10 +125,15 @@ public ValueVector visit(BaseVariableWidthVector deltaVector, Void value) { targetVector .getOffsetBuffer() .getInt((long) targetVector.getValueCount() * BaseVariableWidthVector.OFFSET_WIDTH); + // The delta vector's offset buffer need not start at zero (e.g. a vector imported through + // the C data interface from a sliced array), so the amount of data to append is the + // distance between its first and last offsets, not the last offset itself. + int deltaDataStart = deltaVector.getOffsetBuffer().getInt(0); int deltaDataSize = deltaVector - .getOffsetBuffer() - .getInt((long) deltaVector.getValueCount() * BaseVariableWidthVector.OFFSET_WIDTH); + .getOffsetBuffer() + .getInt((long) deltaVector.getValueCount() * BaseVariableWidthVector.OFFSET_WIDTH) + - deltaDataStart; int newValueCapacity = targetDataSize + deltaDataSize; // make sure there is enough capacity @@ -149,7 +154,7 @@ public ValueVector visit(BaseVariableWidthVector deltaVector, Void value) { // append data buffer MemoryUtil.copyMemory( - deltaVector.getDataBuffer().memoryAddress(), + deltaVector.getDataBuffer().memoryAddress() + deltaDataStart, targetVector.getDataBuffer().memoryAddress() + targetDataSize, deltaDataSize); @@ -160,7 +165,7 @@ public ValueVector visit(BaseVariableWidthVector deltaVector, Void value) { + (targetVector.getValueCount() + 1) * BaseVariableWidthVector.OFFSET_WIDTH, deltaVector.getValueCount() * BaseVariableWidthVector.OFFSET_WIDTH); - // increase each offset from the second buffer + // rebase each appended offset to the target's data, accounting for the delta's start offset for (int i = 0; i < deltaVector.getValueCount(); i++) { int oldOffset = targetVector @@ -172,7 +177,7 @@ public ValueVector visit(BaseVariableWidthVector deltaVector, Void value) { .getOffsetBuffer() .setInt( (long) (targetVector.getValueCount() + 1 + i) * BaseVariableWidthVector.OFFSET_WIDTH, - oldOffset + targetDataSize); + oldOffset - deltaDataStart + targetDataSize); } ((BaseVariableWidthVector) targetVector).setLastSet(newValueCount - 1); targetVector.setValueCount(newValueCount); @@ -196,11 +201,15 @@ public ValueVector visit(BaseLargeVariableWidthVector deltaVector, Void value) { .getOffsetBuffer() .getLong( (long) targetVector.getValueCount() * BaseLargeVariableWidthVector.OFFSET_WIDTH); + // see the corresponding comment in visit(BaseVariableWidthVector, Void): the delta's + // offset buffer need not start at zero + long deltaDataStart = deltaVector.getOffsetBuffer().getLong(0); long deltaDataSize = deltaVector - .getOffsetBuffer() - .getLong( - (long) deltaVector.getValueCount() * BaseLargeVariableWidthVector.OFFSET_WIDTH); + .getOffsetBuffer() + .getLong( + (long) deltaVector.getValueCount() * BaseLargeVariableWidthVector.OFFSET_WIDTH) + - deltaDataStart; long newValueCapacity = targetDataSize + deltaDataSize; // make sure there is enough capacity @@ -221,7 +230,7 @@ public ValueVector visit(BaseLargeVariableWidthVector deltaVector, Void value) { // append data buffer MemoryUtil.copyMemory( - deltaVector.getDataBuffer().memoryAddress(), + deltaVector.getDataBuffer().memoryAddress() + deltaDataStart, targetVector.getDataBuffer().memoryAddress() + targetDataSize, deltaDataSize); @@ -232,7 +241,7 @@ public ValueVector visit(BaseLargeVariableWidthVector deltaVector, Void value) { + (targetVector.getValueCount() + 1) * BaseLargeVariableWidthVector.OFFSET_WIDTH, deltaVector.getValueCount() * BaseLargeVariableWidthVector.OFFSET_WIDTH); - // increase each offset from the second buffer + // rebase each appended offset to the target's data, accounting for the delta's start offset for (int i = 0; i < deltaVector.getValueCount(); i++) { long oldOffset = targetVector @@ -245,7 +254,7 @@ public ValueVector visit(BaseLargeVariableWidthVector deltaVector, Void value) { .setLong( (long) (targetVector.getValueCount() + 1 + i) * BaseLargeVariableWidthVector.OFFSET_WIDTH, - oldOffset + targetDataSize); + oldOffset - deltaDataStart + targetDataSize); } ((BaseLargeVariableWidthVector) targetVector).setLastSet(newValueCount - 1); targetVector.setValueCount(newValueCount); @@ -331,16 +340,20 @@ public ValueVector visit(ListVector deltaVector, Void value) { targetVector .getOffsetBuffer() .getInt((long) targetVector.getValueCount() * ListVector.OFFSET_WIDTH); - int deltaListSize = + // see the corresponding comment in visit(BaseVariableWidthVector, Void): the delta's + // offset buffer need not start at zero + int deltaListStart = deltaVector.getOffsetBuffer().getInt(0); + int deltaListEnd = deltaVector .getOffsetBuffer() .getInt((long) deltaVector.getValueCount() * ListVector.OFFSET_WIDTH); + int deltaListSize = deltaListEnd - deltaListStart; ListVector targetListVector = (ListVector) targetVector; // make sure the underlying vector has value count set targetListVector.getDataVector().setValueCount(targetListSize); - deltaVector.getDataVector().setValueCount(deltaListSize); + deltaVector.getDataVector().setValueCount(deltaListEnd); // make sure there is enough capacity while (targetVector.getValueCapacity() < newValueCount) { @@ -372,13 +385,16 @@ public ValueVector visit(ListVector deltaVector, Void value) { .getOffsetBuffer() .setInt( (long) (targetVector.getValueCount() + 1 + i) * ListVector.OFFSET_WIDTH, - oldOffset + targetListSize); + oldOffset - deltaListStart + targetListSize); } targetListVector.setLastSet(newValueCount - 1); // append underlying vectors - VectorAppender innerAppender = new VectorAppender(targetListVector.getDataVector()); - deltaVector.getDataVector().accept(innerAppender, null); + appendDataVector( + targetListVector.getDataVector(), + deltaVector.getDataVector(), + deltaListStart, + deltaListSize); targetVector.setValueCount(newValueCount); return targetVector; @@ -400,17 +416,21 @@ public ValueVector visit(LargeListVector deltaVector, Void value) { targetVector .getOffsetBuffer() .getLong((long) targetVector.getValueCount() * LargeListVector.OFFSET_WIDTH); - long deltaListSize = + // see the corresponding comment in visit(BaseVariableWidthVector, Void): the delta's + // offset buffer need not start at zero + long deltaListStart = deltaVector.getOffsetBuffer().getLong(0); + long deltaListEnd = deltaVector .getOffsetBuffer() .getLong((long) deltaVector.getValueCount() * LargeListVector.OFFSET_WIDTH); + long deltaListSize = deltaListEnd - deltaListStart; - ListVector targetListVector = (ListVector) targetVector; + LargeListVector targetListVector = (LargeListVector) targetVector; // make sure the underlying vector has value count set // todo recheck these casts when int64 vectors are supported targetListVector.getDataVector().setValueCount(checkedCastToInt(targetListSize)); - deltaVector.getDataVector().setValueCount(checkedCastToInt(deltaListSize)); + deltaVector.getDataVector().setValueCount(checkedCastToInt(deltaListEnd)); // make sure there is enough capacity while (targetVector.getValueCapacity() < newValueCount) { @@ -427,10 +447,10 @@ public ValueVector visit(LargeListVector deltaVector, Void value) { // append offset buffer MemoryUtil.copyMemory( - deltaVector.getOffsetBuffer().memoryAddress() + ListVector.OFFSET_WIDTH, + deltaVector.getOffsetBuffer().memoryAddress() + LargeListVector.OFFSET_WIDTH, targetVector.getOffsetBuffer().memoryAddress() + (targetVector.getValueCount() + 1) * LargeListVector.OFFSET_WIDTH, - (long) deltaVector.getValueCount() * ListVector.OFFSET_WIDTH); + (long) deltaVector.getValueCount() * LargeListVector.OFFSET_WIDTH); // increase each offset from the second buffer for (int i = 0; i < deltaVector.getValueCount(); i++) { @@ -443,18 +463,42 @@ public ValueVector visit(LargeListVector deltaVector, Void value) { .getOffsetBuffer() .setLong( (long) (targetVector.getValueCount() + 1 + i) * LargeListVector.OFFSET_WIDTH, - oldOffset + targetListSize); + oldOffset - deltaListStart + targetListSize); } targetListVector.setLastSet(newValueCount - 1); // append underlying vectors - VectorAppender innerAppender = new VectorAppender(targetListVector.getDataVector()); - deltaVector.getDataVector().accept(innerAppender, null); + appendDataVector( + targetListVector.getDataVector(), + deltaVector.getDataVector(), + checkedCastToInt(deltaListStart), + checkedCastToInt(deltaListSize)); targetVector.setValueCount(newValueCount); return targetVector; } + /** + * Appends the range [start, start + length) of the delta vector's data vector to the target + * vector's data vector. The range may not cover the whole delta data vector when the delta's + * offset buffer does not start at zero. + */ + private static void appendDataVector( + ValueVector targetDataVector, ValueVector deltaDataVector, int start, int length) { + if (start == 0 && length == deltaDataVector.getValueCount()) { + VectorAppender innerAppender = new VectorAppender(targetDataVector); + deltaDataVector.accept(innerAppender, null); + return; + } + TransferPair transferPair = + deltaDataVector.getTransferPair(deltaDataVector.getField(), deltaDataVector.getAllocator()); + transferPair.splitAndTransfer(start, length); + try (ValueVector slicedDeltaDataVector = transferPair.getTo()) { + VectorAppender innerAppender = new VectorAppender(targetDataVector); + slicedDeltaDataVector.accept(innerAppender, null); + } + } + @Override public ValueVector visit(FixedSizeListVector deltaVector, Void value) { Preconditions.checkArgument( diff --git a/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java b/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java index df5521a1ad..9a8143f51b 100644 --- a/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java +++ b/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java @@ -26,10 +26,13 @@ import java.util.List; import java.util.stream.IntStream; import java.util.stream.Stream; +import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.memory.util.CommonUtil; +import org.apache.arrow.vector.BaseLargeVariableWidthVector; import org.apache.arrow.vector.BaseValueVector; +import org.apache.arrow.vector.BaseVariableWidthVector; import org.apache.arrow.vector.BaseVariableWidthViewVector; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; @@ -53,6 +56,7 @@ import org.apache.arrow.vector.holders.NullableBigIntHolder; import org.apache.arrow.vector.holders.NullableFloat4Holder; import org.apache.arrow.vector.holders.NullableIntHolder; +import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.vector.testing.ValueVectorDataPopulator; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -178,6 +182,82 @@ public void testAppendVariableWidthVector() { } } + @Test + public void testAppendVariableWidthVectorWithNonZeroStartOffset() { + try (VarCharVector target = new VarCharVector("", allocator); + VarCharVector delta = new VarCharVector("", allocator)) { + + target.allocateNew(64, 4); + ValueVectorDataPopulator.setVector(target, "a0", "a1"); + + // Build a delta vector whose offset buffer does not start at zero, as produced e.g. by + // importing a sliced array through the C data interface. The values are "BBBB" and + // "CCCC"; the data buffer additionally holds 4 bytes of unreferenced prefix ("AAAA"). + try (ArrowBuf validity = allocator.buffer(1); + ArrowBuf offsets = allocator.buffer(12); + ArrowBuf data = allocator.buffer(12)) { + validity.setByte(0, 0b11); + offsets.setInt(0, 4); + offsets.setInt(4, 8); + offsets.setInt(8, 12); + data.setBytes(0, "AAAABBBBCCCC".getBytes(StandardCharsets.UTF_8)); + delta.loadFieldBuffers(new ArrowFieldNode(2, 0), Arrays.asList(validity, offsets, data)); + } + + VectorAppender appender = new VectorAppender(target); + delta.accept(appender, null); + + // the unreferenced prefix must not be appended + assertEquals( + 4 + 8, + target + .getOffsetBuffer() + .getInt((long) target.getValueCount() * BaseVariableWidthVector.OFFSET_WIDTH)); + + try (VarCharVector expected = new VarCharVector("expected", allocator)) { + expected.allocateNew(); + ValueVectorDataPopulator.setVector(expected, "a0", "a1", "BBBB", "CCCC"); + assertVectorsEqual(expected, target); + } + } + } + + @Test + public void testAppendLargeVariableWidthVectorWithNonZeroStartOffset() { + try (LargeVarCharVector target = new LargeVarCharVector("", allocator); + LargeVarCharVector delta = new LargeVarCharVector("", allocator)) { + + target.allocateNew(64, 4); + ValueVectorDataPopulator.setVector(target, "a0", "a1"); + + try (ArrowBuf validity = allocator.buffer(1); + ArrowBuf offsets = allocator.buffer(24); + ArrowBuf data = allocator.buffer(12)) { + validity.setByte(0, 0b11); + offsets.setLong(0, 4); + offsets.setLong(8, 8); + offsets.setLong(16, 12); + data.setBytes(0, "AAAABBBBCCCC".getBytes(StandardCharsets.UTF_8)); + delta.loadFieldBuffers(new ArrowFieldNode(2, 0), Arrays.asList(validity, offsets, data)); + } + + VectorAppender appender = new VectorAppender(target); + delta.accept(appender, null); + + assertEquals( + 4 + 8, + target + .getOffsetBuffer() + .getLong((long) target.getValueCount() * BaseLargeVariableWidthVector.OFFSET_WIDTH)); + + try (LargeVarCharVector expected = new LargeVarCharVector("expected", allocator)) { + expected.allocateNew(); + ValueVectorDataPopulator.setVector(expected, "a0", "a1", "BBBB", "CCCC"); + assertVectorsEqual(expected, target); + } + } + } + @Test public void testAppendVariableWidthViewVector() { final int length1 = 10; @@ -431,6 +511,115 @@ public void testAppendListVector() { } } + @Test + public void testAppendListVectorWithNonZeroStartOffset() { + try (ListVector target = ListVector.empty("target", allocator); + ListVector delta = ListVector.empty("delta", allocator)) { + + target.allocateNew(); + ValueVectorDataPopulator.setVector(target, Arrays.asList(0, 1), Arrays.asList(2, 3)); + + // Build a delta vector whose offset buffer does not start at zero, as produced e.g. by + // importing a sliced array through the C data interface: lists [10, 11] and [12, 13], + // with one unreferenced prefix element (9) in the data vector. + delta.addOrGetVector(FieldType.nullable(Types.MinorType.INT.getType())); + IntVector deltaDataVector = (IntVector) delta.getDataVector(); + deltaDataVector.allocateNew(5); + for (int i = 0; i < 5; i++) { + deltaDataVector.set(i, 9 + i); + } + deltaDataVector.setValueCount(5); + try (ArrowBuf validity = allocator.buffer(1); + ArrowBuf offsets = allocator.buffer(12)) { + validity.setByte(0, 0b11); + offsets.setInt(0, 1); + offsets.setInt(4, 3); + offsets.setInt(8, 5); + delta.loadFieldBuffers(new ArrowFieldNode(2, 0), Arrays.asList(validity, offsets)); + } + assertEquals(Arrays.asList(10, 11), delta.getObject(0)); + + VectorAppender appender = new VectorAppender(target); + delta.accept(appender, null); + + assertEquals(4, target.getValueCount()); + // the unreferenced prefix element must not be appended + assertEquals( + 4 + 4, + target.getOffsetBuffer().getInt((long) target.getValueCount() * ListVector.OFFSET_WIDTH)); + assertEquals(Arrays.asList(0, 1), target.getObject(0)); + assertEquals(Arrays.asList(2, 3), target.getObject(1)); + assertEquals(Arrays.asList(10, 11), target.getObject(2)); + assertEquals(Arrays.asList(12, 13), target.getObject(3)); + } + } + + @Test + public void testAppendLargeListVector() { + try (LargeListVector target = LargeListVector.empty("target", allocator); + LargeListVector delta = LargeListVector.empty("delta", allocator)) { + + target.allocateNew(); + ValueVectorDataPopulator.setVector(target, Arrays.asList(0, 1), null, Arrays.asList(4, 5)); + + delta.allocateNew(); + ValueVectorDataPopulator.setVector(delta, Arrays.asList(10, 11, 12), Arrays.asList(13, 14)); + + VectorAppender appender = new VectorAppender(target); + delta.accept(appender, null); + + assertEquals(5, target.getValueCount()); + assertEquals(Arrays.asList(0, 1), target.getObject(0)); + assertTrue(target.isNull(1)); + assertEquals(Arrays.asList(4, 5), target.getObject(2)); + assertEquals(Arrays.asList(10, 11, 12), target.getObject(3)); + assertEquals(Arrays.asList(13, 14), target.getObject(4)); + } + } + + @Test + public void testAppendLargeListVectorWithNonZeroStartOffset() { + try (LargeListVector target = LargeListVector.empty("target", allocator); + LargeListVector delta = LargeListVector.empty("delta", allocator)) { + + target.allocateNew(); + ValueVectorDataPopulator.setVector(target, Arrays.asList(0, 1), Arrays.asList(2, 3)); + + // same as testAppendListVectorWithNonZeroStartOffset, with 8-byte offsets + delta.addOrGetVector(FieldType.nullable(Types.MinorType.INT.getType())); + IntVector deltaDataVector = (IntVector) delta.getDataVector(); + deltaDataVector.allocateNew(5); + for (int i = 0; i < 5; i++) { + deltaDataVector.set(i, 9 + i); + } + deltaDataVector.setValueCount(5); + try (ArrowBuf validity = allocator.buffer(1); + ArrowBuf offsets = allocator.buffer(24)) { + validity.setByte(0, 0b11); + offsets.setLong(0, 1); + offsets.setLong(8, 3); + offsets.setLong(16, 5); + delta.loadFieldBuffers(new ArrowFieldNode(2, 0), Arrays.asList(validity, offsets)); + } + assertEquals(Arrays.asList(10, 11), delta.getObject(0)); + + VectorAppender appender = new VectorAppender(target); + delta.accept(appender, null); + + assertEquals(4, target.getValueCount()); + // the unreferenced prefix element must not be appended + assertEquals( + 4 + 4, + target + .getOffsetBuffer() + .getLong((long) target.getValueCount() * LargeListVector.OFFSET_WIDTH)); + assertEquals(Arrays.asList(0, 1), target.getObject(0)); + assertEquals(Arrays.asList(2, 3), target.getObject(1)); + assertEquals(Arrays.asList(10, 11), target.getObject(2)); + assertEquals(Arrays.asList(12, 13), target.getObject(3)); + } + } + @Test public void testAppendEmptyListVector() { try (ListVector target = ListVector.empty("target", allocator); From 9e863319b1e3bedda45dbb12d5989f6b1330f664 Mon Sep 17 00:00:00 2001 From: Abdul Rawoof Khan Date: Mon, 6 Jul 2026 06:39:33 +0530 Subject: [PATCH 257/271] GH-1206: validate decompressed length in Lz4CompressionCodec (#1207) ## What's Changed `Lz4CompressionCodec.doDecompress` sizes the output buffer to the bytes it actually decompressed, but sets `writerIndex` to the length taken from the untrusted 8-byte prefix. A buffer whose prefix claims more than the real output leaves the returned `ArrowBuf` with a `writerIndex` past its capacity, and consumers then read off-heap memory beyond the allocation. This adds the actual-vs-claimed length check the ZSTD codec already does, so a mismatch throws instead of producing an over-long buffer. Closes #1206. --- .../compression/Lz4CompressionCodec.java | 7 ++++++ .../compression/TestCompressionCodec.java | 22 +++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/compression/src/main/java/org/apache/arrow/compression/Lz4CompressionCodec.java b/compression/src/main/java/org/apache/arrow/compression/Lz4CompressionCodec.java index 91cefc2a9e..f268e815fe 100644 --- a/compression/src/main/java/org/apache/arrow/compression/Lz4CompressionCodec.java +++ b/compression/src/main/java/org/apache/arrow/compression/Lz4CompressionCodec.java @@ -80,6 +80,13 @@ protected ArrowBuf doDecompress(BufferAllocator allocator, ArrowBuf compressedBu } byte[] outBytes = out.toByteArray(); + if (outBytes.length != decompressedLength) { + throw new RuntimeException( + "Expected != actual decompressed length: " + + decompressedLength + + " != " + + outBytes.length); + } ArrowBuf decompressedBuffer = allocator.buffer(outBytes.length); decompressedBuffer.setBytes(/* index= */ 0, outBytes); decompressedBuffer.writerIndex(decompressedLength); diff --git a/compression/src/test/java/org/apache/arrow/compression/TestCompressionCodec.java b/compression/src/test/java/org/apache/arrow/compression/TestCompressionCodec.java index b8fb4e28b9..d2d2921649 100644 --- a/compression/src/test/java/org/apache/arrow/compression/TestCompressionCodec.java +++ b/compression/src/test/java/org/apache/arrow/compression/TestCompressionCodec.java @@ -20,6 +20,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayOutputStream; @@ -59,6 +60,7 @@ import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -231,6 +233,26 @@ void testEmptyBuffer(int vectorLength, CompressionCodec codec) throws Exception AutoCloseables.close(decompressedBuffers); } + @Test + void testLz4DecompressRejectsWrongLength() { + byte[] data = new byte[512]; // all zeros, highly compressible + ArrowBuf orig = allocator.buffer(data.length); + orig.setBytes(0, data); + orig.writerIndex(data.length); + + CompressionCodec codec = new Lz4CompressionCodec(); + ArrowBuf compressed = codec.compress(allocator, orig); + + // tamper with the 8-byte uncompressed-length prefix so it no longer matches + // the real decompressed size + compressed.setLong(0, 1_000_000L); + + RuntimeException e = + assertThrows(RuntimeException.class, () -> codec.decompress(allocator, compressed)); + assertTrue(e.getMessage().contains("decompressed length")); + compressed.close(); + } + private static Stream codecTypes() { return Arrays.stream(CompressionUtil.CodecType.values()); } From c7e8e75c9978c60234dbcbd31311ac3ee2975fa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Pupier?= Date: Mon, 6 Jul 2026 08:50:26 +0200 Subject: [PATCH 258/271] MINOR: Restrict trigger push branch for GitHub Workflow (#1204) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Feature branches rarely need their own CI runs: the code is already tested when a pull request is opened against a release branch. If the push trigger has no branch restriction and pull_request is also configured, every push to a branch with an open PR runs the workflow twice: once for the push and once for the PR synchronisation. Always give the push trigger an explicit list of branches: this stops branches created from a release branch from inheriting its workflow runs. see https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=430408443#GitHubActionsRecommendedPractices-Restrictthepushtriggertospecificbranches ## What's Changed Please fill in a description of the changes here. **This contains breaking changes.** Closes #NNN. Note that I needed to recreate a PR as the previous one was closed https://github.com/apache/arrow-java/pull/1202#issuecomment-4864548916 Signed-off-by: Aurélien Pupier --- .github/workflows/dev.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 25b08700fd..7c590c749a 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -19,7 +19,9 @@ name: Dev on: pull_request: {} - push: {} + push: + branches-ignore: + - dependabot/** concurrency: group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} From 9e100a3b7f681a64f11eb4405e9e43431caf3647 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:21:40 +0200 Subject: [PATCH 259/271] MINOR: Bump com.squareup.okhttp3:mockwebserver3 from 5.3.2 to 5.4.0 (#1215) Bumps [com.squareup.okhttp3:mockwebserver3](https://github.com/square/okhttp) from 5.3.2 to 5.4.0.
Changelog

Sourced from com.squareup.okhttp3:mockwebserver3's changelog.

Version 5.4.0

2026-06-08

  • New: Add superpowers to interceptors. Interceptors can now override anything settable on OkHttpClient.Builder, such as the cache, connection pool, socket factory, and DNS. We expect this will allow most users to use interceptors everywhere, insted of mixing and matching interceptors with custom Call.Factory wrappers.
  • Fix: Limit each HTTP/2 response to 256 KiB of total headers.
  • Upgrade: [kotlinx.coroutines 1.11.0][coroutines_1_11_0]. This is used by the optional okhttp-coroutines artifact.
  • Upgrade: [GraalVM 25.0.3][graalvm_25].
  • Upgrade: [Okio 3.17.0][okio_3_17_0].
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.squareup.okhttp3:mockwebserver3&package-manager=maven&previous-version=5.3.2&new-version=5.4.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index 9a8406cf31..e25d8438af 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -123,7 +123,7 @@ under the License. com.squareup.okhttp3 mockwebserver3 - 5.3.2 + 5.4.0 test From e087824935d283d3636d8edb908a79df2de217c4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:25:41 +0200 Subject: [PATCH 260/271] MINOR: Bump com.diffplug.spotless:spotless-maven-plugin from 3.6.0 to 3.8.0 (#1214) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [com.diffplug.spotless:spotless-maven-plugin](https://github.com/diffplug/spotless) from 3.6.0 to 3.8.0.
Release notes

Sourced from com.diffplug.spotless:spotless-maven-plugin's releases.

Maven Plugin v3.8.0

Added

  • Add support for custom string format for license header copyright year via yearStringFormat(). (#2965)

Fixed

  • <expandWildcardImports> no longer triggers a full transitive dependency resolution on every build. Dependency resolution is now deferred until the step actually runs, so projects that do not use <expandWildcardImports> (or that use version ranges) are no longer penalized. (#2983)

Maven Plugin v3.7.0

Fixed

  • Parse standard git year output in LicenseHeaderStep. (#2940)
  • <toggleOffOn> no longer disables lint-only steps such as <forbidWildcardImports>. (#2962)
  • Fix StringIndexOutOfBoundsException in scenarios where copyright year is surrounded by whitespace. (#2973)

Added

  • Add support for AsciiDoc formatting via adocfmt. (#2960)
  • <flexmark> step now supports arbitrary formatter options via <formatterOptions>. (#2968)
Changelog

Sourced from com.diffplug.spotless:spotless-maven-plugin's changelog.

spotless-lib and spotless-lib-extra releases

If you are a Spotless user (as opposed to developer), then you are probably looking for:

This document is intended for Spotless developers.

We adhere to the keepachangelog format (starting after version 1.27.0).

[Unreleased]

[4.8.0] - 2026-06-29

Added

  • Add support for custom string format for license header copyright year via yearStringFormat(). (#2965)

[4.7.0] - 2026-06-16

Added

  • Add support for AsciiDoc formatting via adocfmt. (#2960)
  • flexmark step now supports arbitrary formatter options via a formatterOptions map. (#2968)

Fixed

  • FenceStep.preserveWithin now forwards lints from nested steps while still suppressing lints inside preserved blocks. (#2962)
  • Support ktfmt 0.63 and use its new builder API for formatting options to better avoid future breaking changes.
  • Parse standard git year output in LicenseHeaderStep. (#2940)
  • Fix StringIndexOutOfBoundsException in scenarios where copyright year is surrounded by whitespace. (#2973)

Changes

  • Bump default greclipse version to latest 4.35 -> 4.39. (#2924)

[4.6.2] - 2026-05-27

Fixed

  • P2Provisioner now passes cache directory overrides directly to Solstice. (#2944)
  • forbidWildcardImports and forbidModuleImports now detect imports that have leading whitespace (indentation/tabs). (#2939)
  • versionCatalog step no longer splits long inline tables across multiple lines — Gradle's TOML 1.0 parser cannot read multi-line inline tables. The maxLineLength option has been removed. (#2948)

Changes

  • EclipseJdtFormtterStep now can conditionally set compiler source/compliance options. Allows for better parsing of AST Node for newer language features and more correct sorting; e.g. records or seal classes. (#2942)
  • Formatter no longer recomputes line-ending normalization (LineEnding.toUnix) a second time for every formatter step that changes content, removing redundant O(n) work from the core formatting loop. (#2934)
  • expandWildcardImports support pom type dependency. (#2839)

[4.6.1] - 2026-05-15

Fixed

  • LicenseHeaderStep in SET_FROM_GIT year mode no longer invokes git log through bash -c / cmd /c, eliminating a shell-injection vector when processing repositories that contain files whose names include shell metacharacters.

[4.6.0] - 2026-05-14

Added

  • scalafmt() now reads the version from the version field in the scalafmt config file when no version is explicitly set in the plugin config, falling back to the built-in default only if neither is available. (#2922)
  • Add versionCatalog step for formatting and sorting Gradle version catalog (.toml) files. (#2916)
  • Add javaparserVersion option to the Cleanthat step, allowing callers to override the JavaParser version pulled in transitively by Cleanthat. (#2903)

Fixed

... (truncated)

Commits
  • 03d43ba Published maven/3.8.0
  • 8b80c13 Published gradle/8.8.0
  • 8ee6cf9 Published lib/4.8.0
  • 6c02c0b Add missing changelog entry.
  • 264f4cc Add regression test for forbidWildcardImports inside toggleOffOn (#2982)
  • 6abb064 fix #2983, expandWildcardImports triggers a full transitive reso… (#2984)
  • f4536d4 Update plugin spotbugs to v6.5.8 (#2987)
  • 873454a Update plugin spotbugs to v6.5.8
  • 000b8a8 Update dependency org.junit.jupiter:junit-jupiter to v6.1.1 (#2985)
  • 84ebcab Update dependency org.junit.jupiter:junit-jupiter to v6.1.1
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.diffplug.spotless:spotless-maven-plugin&package-manager=maven&previous-version=3.6.0&new-version=3.8.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- bom/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index 3201761e81..2d1085b160 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -208,7 +208,7 @@ under the License. com.diffplug.spotless spotless-maven-plugin - 3.6.0 + 3.8.0 org.codehaus.mojo diff --git a/pom.xml b/pom.xml index 231720eb44..017e3acb81 100644 --- a/pom.xml +++ b/pom.xml @@ -492,7 +492,7 @@ under the License. com.diffplug.spotless spotless-maven-plugin - 3.6.0 + 3.8.0 org.codehaus.mojo From 1801a8a0937fb4aaac3654c66ec9109a6892666c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:28:19 +0200 Subject: [PATCH 261/271] MINOR: Bump com.gradle:common-custom-user-data-maven-extension from 2.2.0 to 2.3.0 (#1213) Bumps [com.gradle:common-custom-user-data-maven-extension](https://github.com/gradle/common-custom-user-data-maven-extension) from 2.2.0 to 2.3.0.
Release notes

Sourced from com.gradle:common-custom-user-data-maven-extension's releases.

2.3.0

  • [NEW] Capture Cursor as an AI agent via the CURSOR_AGENT environment variable
Commits
  • 61a5a45 [maven-release-plugin] prepare release v2.3.0
  • 398a231 [Renovate Bot] Update actions/setup-java digest to 1bcf9fb (#389)
  • f66a5c6 Merge pull request #391 from gradle/erichaagdev/capture-cursor-ai-agent
  • 8311f58 Capture Cursor as an AI agent
  • 0236fc8 [Renovate Bot] Update dependency org.eclipse.sisu:org.eclipse.sisu.inject to ...
  • 0d8c2c5 [Renovate Bot] Update GitHub Actions to v7 (#386)
  • 21ef159 [Renovate Bot] Update Maven dependencies (#387)
  • d9240f0 [Renovate Bot] Update Maven dependencies to v0.11.0 (#384)
  • 83935cc Auto-merge GitHub Actions digest re-pins (#385)
  • 9d03ae6 [Renovate Bot] Update GitHub Actions to ad2b381 (#383)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.gradle:common-custom-user-data-maven-extension&package-manager=maven&previous-version=2.2.0&new-version=2.3.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .mvn/extensions.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index 38b3c807b7..eb06ecc9fb 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -28,6 +28,6 @@ com.gradle common-custom-user-data-maven-extension - 2.2.0 + 2.3.0 From afd688e024753c75854e707748fe5ad4411d3361 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:29:40 +0200 Subject: [PATCH 262/271] MINOR: Bump com.gradle:develocity-maven-extension from 2.4.1 to 2.5.0 (#1212) Bumps com.gradle:develocity-maven-extension from 2.4.1 to 2.5.0. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.gradle:develocity-maven-extension&package-manager=maven&previous-version=2.4.1&new-version=2.5.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .mvn/extensions.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index eb06ecc9fb..74482cb2c4 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -23,7 +23,7 @@ com.gradle develocity-maven-extension - 2.4.1 + 2.5.0 com.gradle From 59010e63bb58c702ccdbcf6f7e0fb047bf8c8f73 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:31:03 +0200 Subject: [PATCH 263/271] MINOR: Bump checker.framework.version from 4.2.0 to 4.2.1 (#1211) Bumps `checker.framework.version` from 4.2.0 to 4.2.1. Updates `org.checkerframework:checker-qual` from 4.2.0 to 4.2.1
Release notes

Sourced from org.checkerframework:checker-qual's releases.

Checker Framework 4.2.1

Version 4.2.1 (2026-07-01)

Closed issues

#7726.

Changelog

Sourced from org.checkerframework:checker-qual's changelog.

Version 4.2.1 (2026-07-01)

Closed issues

#7726.

Commits

Updates `org.checkerframework:checker` from 4.2.0 to 4.2.1
Release notes

Sourced from org.checkerframework:checker's releases.

Checker Framework 4.2.1

Version 4.2.1 (2026-07-01)

Closed issues

#7726.

Changelog

Sourced from org.checkerframework:checker's changelog.

Version 4.2.1 (2026-07-01)

Closed issues

#7726.

Commits

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 017e3acb81..d9f9d59c43 100644 --- a/pom.xml +++ b/pom.xml @@ -112,7 +112,7 @@ under the License. 10.23.0 true 2.42.0 - 4.2.0 + 4.2.1 1.5.34 none -Xdoclint:none From f2594c99cc98923ad5949713d4519c5d4f24f4f6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:32:02 +0200 Subject: [PATCH 264/271] MINOR: [CI] Bump docker/login-action from 4.2.0 to 4.4.0 (#1210) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [docker/login-action](https://github.com/docker/login-action) from 4.2.0 to 4.4.0.
Release notes

Sourced from docker/login-action's releases.

v4.4.0

Full Changelog: https://github.com/docker/login-action/compare/v4.3.0...v4.4.0

v4.3.0

Full Changelog: https://github.com/docker/login-action/compare/v4.2.0...v4.3.0

Commits
  • af1e73f Merge pull request #1034 from docker/dependabot/npm_and_yarn/aws-sdk-dependen...
  • da722bd [dependabot skip] chore: update generated content
  • 2916ad6 build(deps): bump the aws-sdk-dependencies group across 1 directory with 2 up...
  • ca0a662 Merge pull request #1035 from crazy-max/fix-registry-auth-empty-mask
  • c455755 chore: update generated content
  • 4835190 skip empty registry-auth secret mask
  • 992421c Merge pull request #1033 from docker/dependabot/github_actions/docker/bake-ac...
  • b249b43 Merge pull request #1032 from docker/dependabot/github_actions/docker/bake-ac...
  • 1b67977 build(deps): bump docker/bake-action from 7.2.0 to 7.3.0
  • 9d49d6a build(deps): bump docker/bake-action/subaction/matrix
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=docker/login-action&package-manager=github_actions&previous-version=4.2.0&new-version=4.4.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index fdfcd1cce3..5d2fb6683a 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -127,7 +127,7 @@ jobs: with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing - - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 with: registry: ghcr.io username: ${{ github.actor }} From 915428751b8871406268aeb27bd2f055dcfba6bf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 16:00:46 +0200 Subject: [PATCH 265/271] MINOR: Bump logback.version from 1.5.34 to 1.5.37 (#1209) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps `logback.version` from 1.5.34 to 1.5.37. Updates `ch.qos.logback:logback-classic` from 1.5.34 to 1.5.37
Release notes

Sourced from ch.qos.logback:logback-classic's releases.

Logback 1.5.37

2026-06-26 Release of logback version 1.5.37

  1. • Given the numerous vulnerabilities related to conditional configuration processing based on the evaluation of Java expressions using the Janino library, support for such expressions has been removed. Users are offered the an online migration service or the <condition> element introduced in version 1.5.20. See the relevant documentation for more details.

• A bitwise identical binary of this version can be reproduced by building from source code at commit c1df7f522e648eec7b4ef6a12c8758fec0f00048 associated with the tag v_1.5.37. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Logback 1.5.36

2026-06-25 Release of logback version 1.5.36

• The 'condition' attribute in <if> elements now reject certain references that are associated with ACE attacks. This issue was reported by "yulate" (yulate531@gmail.com.com) and registered as CVE-2026-13006. Please note that version 1.5.37 provides the full fix to this vulnerability.

• A bitwise identical binary of this version can be reproduced by building from source code at commit 9b94c37562bf25a6a944146701d42ee6c4eee888 associated with the tag v_1.5.36. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Logback 1.5.35

026-06-23 Release of logback version 1.5.35

• The 'condition' attribute in <if> elements now rejects unicode escape sequences (\u and \U). This closes a bypass of the existing prohibition on the new operator in Janino-evaluated conditions. This issue was reported by IcySun (icysun@qq.com) and registered as CVE-2026-13006. Please note that version 1.5.37 provides the full fix to this vulnerability.

• Added ConfiguratorRank.AUTHENTICATING (rank 100), the highest configurator rank, for certified/authenticating configurators discovered via the ServiceLoader mechanism. ContextInitializer now requires that at most one such configurator exist on the classpath; if more than one is found, initialization aborts with an error.

• ConsoleCharsetPropertyDefiner is no longer shipped. The Java 21 multi-release compilation of logback-core has been disabled, which removes this class from the published artifact. Configurations that referenced ch.qos.logback.core.property.ConsoleCharsetPropertyDefiner will need an alternative approach for console charset detection.

• The logback-examples module is now included in artifacts published to Maven Central.

• JoranConfigurator.makeAnotherInstance() and DefaultJoranConfigurator.performMultiStepConfigurationFileSearch() are now protected, allowing derived configurators to override these methods.

• A bitwise identical binary of this version can be reproduced by building from source code at commit 08bd1598d565d83444f72983935e7da4746783b7 associated with the tag v_1.5.35. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits

Updates `ch.qos.logback:logback-core` from 1.5.34 to 1.5.37
Release notes

Sourced from ch.qos.logback:logback-core's releases.

Logback 1.5.37

2026-06-26 Release of logback version 1.5.37

  1. • Given the numerous vulnerabilities related to conditional configuration processing based on the evaluation of Java expressions using the Janino library, support for such expressions has been removed. Users are offered the an online migration service or the <condition> element introduced in version 1.5.20. See the relevant documentation for more details.

• A bitwise identical binary of this version can be reproduced by building from source code at commit c1df7f522e648eec7b4ef6a12c8758fec0f00048 associated with the tag v_1.5.37. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Logback 1.5.36

2026-06-25 Release of logback version 1.5.36

• The 'condition' attribute in <if> elements now reject certain references that are associated with ACE attacks. This issue was reported by "yulate" (yulate531@gmail.com.com) and registered as CVE-2026-13006. Please note that version 1.5.37 provides the full fix to this vulnerability.

• A bitwise identical binary of this version can be reproduced by building from source code at commit 9b94c37562bf25a6a944146701d42ee6c4eee888 associated with the tag v_1.5.36. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Logback 1.5.35

026-06-23 Release of logback version 1.5.35

• The 'condition' attribute in <if> elements now rejects unicode escape sequences (\u and \U). This closes a bypass of the existing prohibition on the new operator in Janino-evaluated conditions. This issue was reported by IcySun (icysun@qq.com) and registered as CVE-2026-13006. Please note that version 1.5.37 provides the full fix to this vulnerability.

• Added ConfiguratorRank.AUTHENTICATING (rank 100), the highest configurator rank, for certified/authenticating configurators discovered via the ServiceLoader mechanism. ContextInitializer now requires that at most one such configurator exist on the classpath; if more than one is found, initialization aborts with an error.

• ConsoleCharsetPropertyDefiner is no longer shipped. The Java 21 multi-release compilation of logback-core has been disabled, which removes this class from the published artifact. Configurations that referenced ch.qos.logback.core.property.ConsoleCharsetPropertyDefiner will need an alternative approach for console charset detection.

• The logback-examples module is now included in artifacts published to Maven Central.

• JoranConfigurator.makeAnotherInstance() and DefaultJoranConfigurator.performMultiStepConfigurationFileSearch() are now protected, allowing derived configurators to override these methods.

• A bitwise identical binary of this version can be reproduced by building from source code at commit 08bd1598d565d83444f72983935e7da4746783b7 associated with the tag v_1.5.35. Release built using Java "21" 2023-10-17 LTS build 21.0.1.+12-LTS-29 under Linux Debian 11.6.

Commits

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: JB Onofré --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d9f9d59c43..78137eb4e9 100644 --- a/pom.xml +++ b/pom.xml @@ -113,7 +113,7 @@ under the License. true 2.42.0 4.2.1 - 1.5.34 + 1.5.37 none -Xdoclint:none From a9f0086090e649b83c89ea864c82ed7fcc84ab2f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 22:19:02 +0200 Subject: [PATCH 266/271] MINOR: Bump com.squareup.okhttp3:okhttp-jvm from 5.3.2 to 5.4.0 (#1208) Bumps [com.squareup.okhttp3:okhttp-jvm](https://github.com/square/okhttp) from 5.3.2 to 5.4.0.
Changelog

Sourced from com.squareup.okhttp3:okhttp-jvm's changelog.

Version 5.4.0

2026-06-08

  • New: Add superpowers to interceptors. Interceptors can now override anything settable on OkHttpClient.Builder, such as the cache, connection pool, socket factory, and DNS. We expect this will allow most users to use interceptors everywhere, insted of mixing and matching interceptors with custom Call.Factory wrappers.
  • Fix: Limit each HTTP/2 response to 256 KiB of total headers.
  • Upgrade: [kotlinx.coroutines 1.11.0][coroutines_1_11_0]. This is used by the optional okhttp-coroutines artifact.
  • Upgrade: [GraalVM 25.0.3][graalvm_25].
  • Upgrade: [Okio 3.17.0][okio_3_17_0].
Commits

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flight/flight-sql-jdbc-core/pom.xml b/flight/flight-sql-jdbc-core/pom.xml index e25d8438af..be2ee32868 100644 --- a/flight/flight-sql-jdbc-core/pom.xml +++ b/flight/flight-sql-jdbc-core/pom.xml @@ -135,7 +135,7 @@ under the License. com.squareup.okhttp3 okhttp-jvm - 5.3.2 + 5.4.0 test From 7f1f9f588af610b842ee1c3bccf6afbd528ae4db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Thu, 9 Jul 2026 14:42:25 +0100 Subject: [PATCH 267/271] GH-1188: Reduce test workflow waste (#1189) ## Summary This PR reduces wasted GitHub Actions time in the `Test` workflow by removing redundant rebuilds, improving caching, and skipping integration work when it is not relevant. ## What changed - Removed `clean` from `ci/scripts/test.sh` so the test phase reuses the classes compiled earlier in the job instead of deleting and recompiling them. - Narrowed the Maven/Docker cache key to POM changes, so source-only edits do not invalidate the dependency cache. - Enabled Maven dependency caching on the macOS and Windows test jobs. - Added `pull-requests: read` so the integration job can inspect changed files. Closes #1188 Note: The changes on this PR were highlighted and addressed with AI assistance --- .github/workflows/test.yml | 24 +++++++++++++----------- ci/scripts/test.sh | 7 ++++--- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ac8080d075..41a42c05ed 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -66,7 +66,7 @@ jobs: uses: actions/cache@v6 with: path: .docker - key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('compose.yaml', '**/pom.xml', '**/*.java') }} + key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} restore-keys: maven-${{ matrix.jdk }}-${{ matrix.maven }}- - name: Execute Docker Build env: @@ -94,16 +94,17 @@ jobs: jdk: 17 macos: latest steps: - - name: Set up Java - uses: actions/setup-java@v5 - with: - distribution: 'temurin' - java-version: ${{ matrix.jdk }} - name: Checkout Arrow uses: actions/checkout@v7 with: fetch-depth: 0 submodules: recursive + - name: Set up Java + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: ${{ matrix.jdk }} + cache: 'maven' - name: Build shell: bash env: @@ -125,16 +126,17 @@ jobs: matrix: jdk: [17] steps: - - name: Set up Java - uses: actions/setup-java@v5 - with: - java-version: ${{ matrix.jdk }} - distribution: 'temurin' - name: Checkout Arrow uses: actions/checkout@v7 with: fetch-depth: 0 submodules: recursive + - name: Set up Java + uses: actions/setup-java@v5 + with: + java-version: ${{ matrix.jdk }} + distribution: 'temurin' + cache: 'maven' - name: Build shell: bash env: diff --git a/ci/scripts/test.sh b/ci/scripts/test.sh index cacc20034e..8061ee455d 100755 --- a/ci/scripts/test.sh +++ b/ci/scripts/test.sh @@ -34,10 +34,11 @@ fi mvn="mvn -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" # Use `2 * ncores` threads mvn="${mvn} -T 2C" +mvn="${mvn} -Denforcer.skip=true" pushd "${build_dir}" -${mvn} -Darrow.test.dataRoot="${source_dir}/testing/data" clean test +${mvn} -Darrow.test.dataRoot="${source_dir}/testing/data" test projects=() if [ "${ARROW_JAVA_JNI}" = "ON" ]; then @@ -46,7 +47,7 @@ if [ "${ARROW_JAVA_JNI}" = "ON" ]; then projects+=(gandiva) fi if [ "${#projects[@]}" -gt 0 ]; then - ${mvn} clean test \ + ${mvn} test \ -Parrow-jni \ -pl "$( IFS=, @@ -56,7 +57,7 @@ if [ "${#projects[@]}" -gt 0 ]; then fi if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then - ${mvn} clean test -Parrow-c-data -pl c -Darrow.c.jni.dist.dir="${java_jni_dist_dir}" + ${mvn} test -Parrow-c-data -pl c -Darrow.c.jni.dist.dir="${java_jni_dist_dir}" fi popd From 21b6a05154b7713b12ace4c0ba3572ba5cc8fe66 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2026 10:19:32 +0900 Subject: [PATCH 268/271] MINOR: [CI] Bump actions/setup-python from 6 to 7 (#1235) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/setup-python](https://github.com/actions/setup-python) from 6 to 7.
Release notes

Sourced from actions/setup-python's releases.

v7.0.0

What's Changed

Enhancements

Bug Fix

Dependency Upgrade

New Contributors

Full Changelog: https://github.com/actions/setup-python/compare/v6...v7.0.0

v6.3.0

What's Changed

Enhancement

Dependency update

Documentation

New Contributors

Full Changelog: https://github.com/actions/setup-python/compare/v6.2.0...v6.3.0

v6.2.0

What's Changed

Dependency Upgrades

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/setup-python&package-manager=github_actions&previous-version=6&new-version=7)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/dev.yml | 2 +- .github/workflows/rc.yml | 4 ++-- .github/workflows/test.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 7c590c749a..f4dc9ad6f1 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -40,7 +40,7 @@ jobs: fetch-depth: 0 persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: python-version: '3.x' - name: pre-commit (cache) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 5d2fb6683a..18a721ac02 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -195,7 +195,7 @@ jobs: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: cache: 'pip' python-version: 3.12 @@ -446,7 +446,7 @@ jobs: contents: read packages: write steps: - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: cache: 'pip' - name: Download source archive diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 41a42c05ed..853b7cec09 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -198,7 +198,7 @@ jobs: key: integration-conda-${{ hashFiles('cpp/**') }} restore-keys: integration-conda- - name: Setup Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: 3.12 - name: Setup Archery From 06170242bde2f492e068235efdd2183a3cbd87d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Sat, 25 Jul 2026 14:35:09 +0100 Subject: [PATCH 269/271] GH-1244: Move integration tests to separate workflow (#1245) ## What's Changed Move the `integration` job from `test.yml` into its own `integration.yml` workflow. The new workflow uses top-level `paths` filters, so integration tests run only when changes affect relevant code or build inputs. This should also help to reduce the amount of runner time. Topic initially triggered in the [mailing list](https://lists.apache.org/thread/drf0o5kzg1zfmok7gc09k8qz8hh9ymvh) Closes #1244 --- .github/workflows/integration.yml | 127 ++++++++++++++++++++++++++++++ .github/workflows/test.yml | 69 ---------------- 2 files changed, 127 insertions(+), 69 deletions(-) create mode 100644 .github/workflows/integration.yml diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000000..3872d03d2f --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,127 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Integration + +on: + push: + branches: + - '**' + - '!dependabot/**' + tags: + - '**' + paths: + - '.github/workflows/integration.yml' + - '**/pom.xml' + - 'c/**' + - 'ci/scripts/**' + - 'compose.yaml' + - 'flight/**' + - 'format/**' + - 'testing/data/**' + - 'vector/**' + pull_request: + paths: + - '.github/workflows/integration.yml' + - '**/pom.xml' + - 'c/**' + - 'ci/scripts/**' + - 'compose.yaml' + - 'flight/**' + - 'format/**' + - 'testing/data/**' + - 'vector/**' + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +env: + DOCKER_VOLUME_PREFIX: ".docker/" + +jobs: + integration: + name: AMD64 integration + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout Arrow + uses: actions/checkout@v7 + with: + fetch-depth: 0 + repository: apache/arrow + submodules: recursive + - name: Checkout Arrow Rust + uses: actions/checkout@v7 + with: + repository: apache/arrow-rs + path: rust + - name: Checkout Arrow nanoarrow + uses: actions/checkout@v7 + with: + repository: apache/arrow-nanoarrow + path: nanoarrow + - name: Checkout Arrow .NET + uses: actions/checkout@v7 + with: + repository: apache/arrow-dotnet + path: dotnet + - name: Checkout Arrow Go + uses: actions/checkout@v7 + with: + repository: apache/arrow-go + path: go + - name: Checkout Arrow Java + uses: actions/checkout@v7 + with: + path: java + - name: Checkout Arrow JavaScript + uses: actions/checkout@v7 + with: + repository: apache/arrow-js + path: js + - name: Free up disk space + run: | + ci/scripts/util_free_space.sh + - name: Cache Docker Volumes + uses: actions/cache@v6 + with: + path: .docker + key: integration-conda-${{ hashFiles('cpp/**') }} + restore-keys: integration-conda- + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: 3.12 + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + run: | + source ci/scripts/util_enable_core_dumps.sh + archery docker run \ + -e ARCHERY_DEFAULT_BRANCH=main \ + -e ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS=java \ + -e ARCHERY_INTEGRATION_WITH_DOTNET=1 \ + -e ARCHERY_INTEGRATION_WITH_GO=1 \ + -e ARCHERY_INTEGRATION_WITH_JAVA=1 \ + -e ARCHERY_INTEGRATION_WITH_JS=1 \ + -e ARCHERY_INTEGRATION_WITH_NANOARROW=1 \ + -e ARCHERY_INTEGRATION_WITH_RUST=1 \ + conda-integration diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 853b7cec09..653b16fa32 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -147,72 +147,3 @@ jobs: env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} run: ci/scripts/test.sh . build jni - - integration: - name: AMD64 integration - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - fetch-depth: 0 - repository: apache/arrow - submodules: recursive - - name: Checkout Arrow Rust - uses: actions/checkout@v7 - with: - repository: apache/arrow-rs - path: rust - - name: Checkout Arrow nanoarrow - uses: actions/checkout@v7 - with: - repository: apache/arrow-nanoarrow - path: nanoarrow - - name: Checkout Arrow .NET - uses: actions/checkout@v7 - with: - repository: apache/arrow-dotnet - path: dotnet - - name: Checkout Arrow Go - uses: actions/checkout@v7 - with: - repository: apache/arrow-go - path: go - - name: Checkout Arrow Java - uses: actions/checkout@v7 - with: - path: java - - name: Checkout Arrow JavaScript - uses: actions/checkout@v7 - with: - repository: apache/arrow-js - path: js - - name: Free up disk space - run: | - ci/scripts/util_free_space.sh - - name: Cache Docker Volumes - uses: actions/cache@v6 - with: - path: .docker - key: integration-conda-${{ hashFiles('cpp/**') }} - restore-keys: integration-conda- - - name: Setup Python - uses: actions/setup-python@v7 - with: - python-version: 3.12 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: | - source ci/scripts/util_enable_core_dumps.sh - archery docker run \ - -e ARCHERY_DEFAULT_BRANCH=main \ - -e ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS=java \ - -e ARCHERY_INTEGRATION_WITH_DOTNET=1 \ - -e ARCHERY_INTEGRATION_WITH_GO=1 \ - -e ARCHERY_INTEGRATION_WITH_JAVA=1 \ - -e ARCHERY_INTEGRATION_WITH_JS=1 \ - -e ARCHERY_INTEGRATION_WITH_NANOARROW=1 \ - -e ARCHERY_INTEGRATION_WITH_RUST=1 \ - conda-integration From 85d7ef30ee5f623cf3442dd989df09c3c5754ad2 Mon Sep 17 00:00:00 2001 From: Sandesh Kumar Date: Thu, 6 Aug 2026 17:29:51 -0700 Subject: [PATCH 270/271] GH-1239: Fix memory leak when C Data import hits allocator limit mid-array (#1240) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ReferenceCountedArrowArray.unsafeAssociateAllocation` calls `retain()` before `wrapForeignAllocation`. If `wrapForeignAllocation` throws (allocator over its limit), the retain is never balanced, the reference count stays elevated, and the C Data release callback never fires — leaking the producer's native memory. **Fix:** call `retain()` after `wrapForeignAllocation` returns. Same behavior on the success path; on failure the existing `finally` in `ArrayImporter.importArray` drives the count to zero and fires the release callback. **Test:** `ImportOutOfMemoryTest` — exports a batch from a "producer" allocator, tries to import it into a too-small consumer, and asserts the producer drains to zero after the OOM. Fails on the original code, passes with the fix. Fixes: https://github.com/apache/arrow-java/issues/1239 --------- Signed-off-by: Sandesh Kumar Co-authored-by: Sandesh Kumar --- .../arrow/c/ReferenceCountedArrowArray.java | 19 ++- .../apache/arrow/c/ImportOutOfMemoryTest.java | 139 ++++++++++++++++++ 2 files changed, 151 insertions(+), 7 deletions(-) create mode 100644 c/src/test/java/org/apache/arrow/c/ImportOutOfMemoryTest.java diff --git a/c/src/main/java/org/apache/arrow/c/ReferenceCountedArrowArray.java b/c/src/main/java/org/apache/arrow/c/ReferenceCountedArrowArray.java index cf50f9417b..f51fb25105 100644 --- a/c/src/main/java/org/apache/arrow/c/ReferenceCountedArrowArray.java +++ b/c/src/main/java/org/apache/arrow/c/ReferenceCountedArrowArray.java @@ -64,13 +64,18 @@ void release() { */ ArrowBuf unsafeAssociateAllocation( BufferAllocator trackingAllocator, long capacity, long memoryAddress) { + // Retain only after wrapForeignAllocation succeeds. On the allocator-limit OOM path, + // wrapForeignAllocation throws before the ForeignAllocation is associated, so release0() + // is not called; retaining first would leave the count elevated with no matching release0(). + ArrowBuf buf = + trackingAllocator.wrapForeignAllocation( + new ForeignAllocation(capacity, memoryAddress) { + @Override + protected void release0() { + ReferenceCountedArrowArray.this.release(); + } + }); retain(); - return trackingAllocator.wrapForeignAllocation( - new ForeignAllocation(capacity, memoryAddress) { - @Override - protected void release0() { - ReferenceCountedArrowArray.this.release(); - } - }); + return buf; } } diff --git a/c/src/test/java/org/apache/arrow/c/ImportOutOfMemoryTest.java b/c/src/test/java/org/apache/arrow/c/ImportOutOfMemoryTest.java new file mode 100644 index 0000000000..7c099f2ef0 --- /dev/null +++ b/c/src/test/java/org/apache/arrow/c/ImportOutOfMemoryTest.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.c; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.List; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.OutOfMemoryException; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Regression test: a mid-import {@link OutOfMemoryException} must not leak the imported array. + * + *

A "producer" allocator owns the exported batch; if the C Data release callback fires, the + * producer drains to zero. A too-small consumer allocator forces an OOM part-way through the + * import. The test asserts the producer drains, confirming the release callback fired despite the + * failure. + */ +final class ImportOutOfMemoryTest { + private static final int ROWS = 1024; + private static final int VALUE_BYTES = 256; + private static final int COLUMNS = 4; + // Far smaller than the exported batch, so the import OOMs part-way through the buffers. + private static final long TINY_LIMIT = 16 * 1024; + + private RootAllocator root; + + @BeforeEach + public void setUp() { + root = new RootAllocator(Long.MAX_VALUE); + } + + @AfterEach + public void tearDown() { + root.close(); + } + + @Test + public void importOomDoesNotLeakExportedArray() { + // "producer" owns only the exported batch buffers; the C Data struct containers live on a + // separate allocator (they are consumed/closed by import, which would otherwise muddy the + // producer's balance). So producer draining to zero is an exact signal that the array's release + // callback fired. + try (BufferAllocator producer = root.newChildAllocator("producer", 0, Long.MAX_VALUE); + BufferAllocator structs = root.newChildAllocator("structs", 0, Long.MAX_VALUE)) { + try (ArrowArray array = ArrowArray.allocateNew(structs); + ArrowSchema schema = ArrowSchema.allocateNew(structs)) { + exportBatch(producer, array, schema); + assertTrue( + producer.getAllocatedMemory() > 0, "producer holds the exported batch before import"); + + // A consumer allocator far too small to hold the batch: the import throws part-way through. + try (BufferAllocator consumer = root.newChildAllocator("consumer", 0, TINY_LIMIT); + CDataDictionaryProvider provider = new CDataDictionaryProvider()) { + Schema importSchema = Data.importSchema(consumer, schema, provider); + try (VectorSchemaRoot importRoot = VectorSchemaRoot.create(importSchema, consumer)) { + Exception thrown = + assertThrows( + Exception.class, + () -> Data.importIntoVectorSchemaRoot(consumer, array, importRoot, provider)); + assertTrue( + hasOutOfMemoryCause(thrown), + "mid-import failure must be an allocator OOM: " + thrown); + } + } + + // The array's release callback must have fired despite the mid-import OOM, freeing the + // whole exported batch. On the unfixed retain-before-wrap code the batch is stranded. + assertEquals( + 0L, + producer.getAllocatedMemory(), + "import OOM leaked the exported batch (producer not drained)"); + } + } + } + + /** True if {@code t} is, or is caused by, an Arrow {@link OutOfMemoryException}. */ + private static boolean hasOutOfMemoryCause(Throwable t) { + for (Throwable cause = t; cause != null; cause = cause.getCause()) { + if (cause instanceof OutOfMemoryException) { + return true; + } + } + return false; + } + + /** + * Builds a wide multi-column VarChar batch on {@code alloc} and exports it into the C structs. + */ + private void exportBatch(BufferAllocator alloc, ArrowArray array, ArrowSchema schema) { + byte[] value = new byte[VALUE_BYTES]; + for (int i = 0; i < value.length; i++) { + value[i] = (byte) 'x'; + } + List vectors = new ArrayList<>(COLUMNS); + for (int c = 0; c < COLUMNS; c++) { + VarCharVector vector = new VarCharVector("col" + c, alloc); + vector.allocateNew((long) ROWS * VALUE_BYTES, ROWS); + for (int r = 0; r < ROWS; r++) { + vector.setSafe(r, value); + } + vector.setValueCount(ROWS); + vectors.add(vector); + } + try (VectorSchemaRoot source = new VectorSchemaRoot(vectors)) { + long total = 0; + for (FieldVector vector : source.getFieldVectors()) { + total += vector.getBufferSize(); + } + assertTrue(total > TINY_LIMIT, "test setup: batch must exceed the consumer limit"); + Data.exportVectorSchemaRoot(alloc, source, null, array, schema); + } + } +} From fa20039f39a7deacf5624ba8a6ee10e9d31e98ce Mon Sep 17 00:00:00 2001 From: David Li Date: Fri, 14 Aug 2026 10:57:43 +0900 Subject: [PATCH 271/271] GH-1241: Bump vcpkg version to stay in sync with apache/arrow (#1260) This fixes the CI for Linux. Closes #1241. --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 51daa0406c..ef0c5fb101 100644 --- a/.env +++ b/.env @@ -53,4 +53,4 @@ MAVEN=3.9.9 # Versions for various dependencies used to build artifacts # Keep in sync with apache/arrow ARROW_REPO_ROOT=./arrow -VCPKG="66c0373dc7fca549e5803087b9487edfe3aca0a1" # 2026.01.16 Release +VCPKG="9b965a116838c6cdcd36bca60d1b81b030c8ab8d" # 2026.05.27 (not release, upstream commit)