diff --git a/.env b/.env index 51daa0406c..ef0c5fb101 100644 --- a/.env +++ b/.env @@ -53,4 +53,4 @@ MAVEN=3.9.9 # Versions for various dependencies used to build artifacts # Keep in sync with apache/arrow ARROW_REPO_ROOT=./arrow -VCPKG="66c0373dc7fca549e5803087b9487edfe3aca0a1" # 2026.01.16 Release +VCPKG="9b965a116838c6cdcd36bca60d1b81b030c8ab8d" # 2026.05.27 (not release, upstream commit) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 7c590c749a..f4dc9ad6f1 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -40,7 +40,7 @@ jobs: fetch-depth: 0 persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: python-version: '3.x' - name: pre-commit (cache) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000000..3872d03d2f --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,127 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Integration + +on: + push: + branches: + - '**' + - '!dependabot/**' + tags: + - '**' + paths: + - '.github/workflows/integration.yml' + - '**/pom.xml' + - 'c/**' + - 'ci/scripts/**' + - 'compose.yaml' + - 'flight/**' + - 'format/**' + - 'testing/data/**' + - 'vector/**' + pull_request: + paths: + - '.github/workflows/integration.yml' + - '**/pom.xml' + - 'c/**' + - 'ci/scripts/**' + - 'compose.yaml' + - 'flight/**' + - 'format/**' + - 'testing/data/**' + - 'vector/**' + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +env: + DOCKER_VOLUME_PREFIX: ".docker/" + +jobs: + integration: + name: AMD64 integration + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout Arrow + uses: actions/checkout@v7 + with: + fetch-depth: 0 + repository: apache/arrow + submodules: recursive + - name: Checkout Arrow Rust + uses: actions/checkout@v7 + with: + repository: apache/arrow-rs + path: rust + - name: Checkout Arrow nanoarrow + uses: actions/checkout@v7 + with: + repository: apache/arrow-nanoarrow + path: nanoarrow + - name: Checkout Arrow .NET + uses: actions/checkout@v7 + with: + repository: apache/arrow-dotnet + path: dotnet + - name: Checkout Arrow Go + uses: actions/checkout@v7 + with: + repository: apache/arrow-go + path: go + - name: Checkout Arrow Java + uses: actions/checkout@v7 + with: + path: java + - name: Checkout Arrow JavaScript + uses: actions/checkout@v7 + with: + repository: apache/arrow-js + path: js + - name: Free up disk space + run: | + ci/scripts/util_free_space.sh + - name: Cache Docker Volumes + uses: actions/cache@v6 + with: + path: .docker + key: integration-conda-${{ hashFiles('cpp/**') }} + restore-keys: integration-conda- + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: 3.12 + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + run: | + source ci/scripts/util_enable_core_dumps.sh + archery docker run \ + -e ARCHERY_DEFAULT_BRANCH=main \ + -e ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS=java \ + -e ARCHERY_INTEGRATION_WITH_DOTNET=1 \ + -e ARCHERY_INTEGRATION_WITH_GO=1 \ + -e ARCHERY_INTEGRATION_WITH_JAVA=1 \ + -e ARCHERY_INTEGRATION_WITH_JS=1 \ + -e ARCHERY_INTEGRATION_WITH_NANOARROW=1 \ + -e ARCHERY_INTEGRATION_WITH_RUST=1 \ + conda-integration diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 5d2fb6683a..18a721ac02 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -195,7 +195,7 @@ jobs: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: cache: 'pip' python-version: 3.12 @@ -446,7 +446,7 @@ jobs: contents: read packages: write steps: - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: cache: 'pip' - name: Download source archive diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 41a42c05ed..653b16fa32 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -147,72 +147,3 @@ jobs: env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} run: ci/scripts/test.sh . build jni - - integration: - name: AMD64 integration - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - name: Checkout Arrow - uses: actions/checkout@v7 - with: - fetch-depth: 0 - repository: apache/arrow - submodules: recursive - - name: Checkout Arrow Rust - uses: actions/checkout@v7 - with: - repository: apache/arrow-rs - path: rust - - name: Checkout Arrow nanoarrow - uses: actions/checkout@v7 - with: - repository: apache/arrow-nanoarrow - path: nanoarrow - - name: Checkout Arrow .NET - uses: actions/checkout@v7 - with: - repository: apache/arrow-dotnet - path: dotnet - - name: Checkout Arrow Go - uses: actions/checkout@v7 - with: - repository: apache/arrow-go - path: go - - name: Checkout Arrow Java - uses: actions/checkout@v7 - with: - path: java - - name: Checkout Arrow JavaScript - uses: actions/checkout@v7 - with: - repository: apache/arrow-js - path: js - - name: Free up disk space - run: | - ci/scripts/util_free_space.sh - - name: Cache Docker Volumes - uses: actions/cache@v6 - with: - path: .docker - key: integration-conda-${{ hashFiles('cpp/**') }} - restore-keys: integration-conda- - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: 3.12 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: | - source ci/scripts/util_enable_core_dumps.sh - archery docker run \ - -e ARCHERY_DEFAULT_BRANCH=main \ - -e ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS=java \ - -e ARCHERY_INTEGRATION_WITH_DOTNET=1 \ - -e ARCHERY_INTEGRATION_WITH_GO=1 \ - -e ARCHERY_INTEGRATION_WITH_JAVA=1 \ - -e ARCHERY_INTEGRATION_WITH_JS=1 \ - -e ARCHERY_INTEGRATION_WITH_NANOARROW=1 \ - -e ARCHERY_INTEGRATION_WITH_RUST=1 \ - conda-integration diff --git a/c/src/main/java/org/apache/arrow/c/ReferenceCountedArrowArray.java b/c/src/main/java/org/apache/arrow/c/ReferenceCountedArrowArray.java index cf50f9417b..f51fb25105 100644 --- a/c/src/main/java/org/apache/arrow/c/ReferenceCountedArrowArray.java +++ b/c/src/main/java/org/apache/arrow/c/ReferenceCountedArrowArray.java @@ -64,13 +64,18 @@ void release() { */ ArrowBuf unsafeAssociateAllocation( BufferAllocator trackingAllocator, long capacity, long memoryAddress) { + // Retain only after wrapForeignAllocation succeeds. On the allocator-limit OOM path, + // wrapForeignAllocation throws before the ForeignAllocation is associated, so release0() + // is not called; retaining first would leave the count elevated with no matching release0(). + ArrowBuf buf = + trackingAllocator.wrapForeignAllocation( + new ForeignAllocation(capacity, memoryAddress) { + @Override + protected void release0() { + ReferenceCountedArrowArray.this.release(); + } + }); retain(); - return trackingAllocator.wrapForeignAllocation( - new ForeignAllocation(capacity, memoryAddress) { - @Override - protected void release0() { - ReferenceCountedArrowArray.this.release(); - } - }); + return buf; } } diff --git a/c/src/test/java/org/apache/arrow/c/ImportOutOfMemoryTest.java b/c/src/test/java/org/apache/arrow/c/ImportOutOfMemoryTest.java new file mode 100644 index 0000000000..7c099f2ef0 --- /dev/null +++ b/c/src/test/java/org/apache/arrow/c/ImportOutOfMemoryTest.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.c; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.List; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.OutOfMemoryException; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Regression test: a mid-import {@link OutOfMemoryException} must not leak the imported array. + * + *

A "producer" allocator owns the exported batch; if the C Data release callback fires, the + * producer drains to zero. A too-small consumer allocator forces an OOM part-way through the + * import. The test asserts the producer drains, confirming the release callback fired despite the + * failure. + */ +final class ImportOutOfMemoryTest { + private static final int ROWS = 1024; + private static final int VALUE_BYTES = 256; + private static final int COLUMNS = 4; + // Far smaller than the exported batch, so the import OOMs part-way through the buffers. + private static final long TINY_LIMIT = 16 * 1024; + + private RootAllocator root; + + @BeforeEach + public void setUp() { + root = new RootAllocator(Long.MAX_VALUE); + } + + @AfterEach + public void tearDown() { + root.close(); + } + + @Test + public void importOomDoesNotLeakExportedArray() { + // "producer" owns only the exported batch buffers; the C Data struct containers live on a + // separate allocator (they are consumed/closed by import, which would otherwise muddy the + // producer's balance). So producer draining to zero is an exact signal that the array's release + // callback fired. + try (BufferAllocator producer = root.newChildAllocator("producer", 0, Long.MAX_VALUE); + BufferAllocator structs = root.newChildAllocator("structs", 0, Long.MAX_VALUE)) { + try (ArrowArray array = ArrowArray.allocateNew(structs); + ArrowSchema schema = ArrowSchema.allocateNew(structs)) { + exportBatch(producer, array, schema); + assertTrue( + producer.getAllocatedMemory() > 0, "producer holds the exported batch before import"); + + // A consumer allocator far too small to hold the batch: the import throws part-way through. + try (BufferAllocator consumer = root.newChildAllocator("consumer", 0, TINY_LIMIT); + CDataDictionaryProvider provider = new CDataDictionaryProvider()) { + Schema importSchema = Data.importSchema(consumer, schema, provider); + try (VectorSchemaRoot importRoot = VectorSchemaRoot.create(importSchema, consumer)) { + Exception thrown = + assertThrows( + Exception.class, + () -> Data.importIntoVectorSchemaRoot(consumer, array, importRoot, provider)); + assertTrue( + hasOutOfMemoryCause(thrown), + "mid-import failure must be an allocator OOM: " + thrown); + } + } + + // The array's release callback must have fired despite the mid-import OOM, freeing the + // whole exported batch. On the unfixed retain-before-wrap code the batch is stranded. + assertEquals( + 0L, + producer.getAllocatedMemory(), + "import OOM leaked the exported batch (producer not drained)"); + } + } + } + + /** True if {@code t} is, or is caused by, an Arrow {@link OutOfMemoryException}. */ + private static boolean hasOutOfMemoryCause(Throwable t) { + for (Throwable cause = t; cause != null; cause = cause.getCause()) { + if (cause instanceof OutOfMemoryException) { + return true; + } + } + return false; + } + + /** + * Builds a wide multi-column VarChar batch on {@code alloc} and exports it into the C structs. + */ + private void exportBatch(BufferAllocator alloc, ArrowArray array, ArrowSchema schema) { + byte[] value = new byte[VALUE_BYTES]; + for (int i = 0; i < value.length; i++) { + value[i] = (byte) 'x'; + } + List vectors = new ArrayList<>(COLUMNS); + for (int c = 0; c < COLUMNS; c++) { + VarCharVector vector = new VarCharVector("col" + c, alloc); + vector.allocateNew((long) ROWS * VALUE_BYTES, ROWS); + for (int r = 0; r < ROWS; r++) { + vector.setSafe(r, value); + } + vector.setValueCount(ROWS); + vectors.add(vector); + } + try (VectorSchemaRoot source = new VectorSchemaRoot(vectors)) { + long total = 0; + for (FieldVector vector : source.getFieldVectors()) { + total += vector.getBufferSize(); + } + assertTrue(total > TINY_LIMIT, "test setup: batch must exceed the consumer limit"); + Data.exportVectorSchemaRoot(alloc, source, null, array, schema); + } + } +}