You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by li...@apache.org on 2026/08/22 10:03:07 UTC

(seatunnel) 02/02: [Fix][Connector-V2] Ensure Arrow allocator closes on reader failure

This is an automated email from the ASF dual-hosted git repository.

davidzollo pushed a commit to branch davidzollo_republish_10958_arrow_close_order
in repository https://gitbox.apache.org/repos/asf/seatunnel.git

commit 1388cd978e6c80ad90a21bc591358e58f34024de
Author: DanielLeens <Da...@users.noreply.github.com>
AuthorDate: Fri Jul 31 00:20:06 2026 +0800

    [Fix][Connector-V2] Ensure Arrow allocator closes on reader failure
    
    (cherry picked from commit c088af7c9f8c3d8cdb42e7d038fce771af4fc1a9)
---
 .../arrow/reader/ArrowToSeatunnelRowReader.java    | 37 ++++++++-
 .../reader/ArrowToSeatunnelRowReaderCloseTest.java | 93 ++++++++++++++++++++++
 2 files changed, 126 insertions(+), 4 deletions(-)

diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReader.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReader.java
index 2609a591ee..0f718862ae 100644
--- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReader.java
+++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReader.java
@@ -82,6 +82,22 @@ public class ArrowToSeatunnelRowReader implements AutoCloseable {
         initArrowReader(byteArray);
     }
 
+    /**
+     * Creates a reader with prebuilt Arrow resources.
+     *
+     * <p>The constructor keeps close-order regression tests focused on resource ownership while
+     * preserving the production byte-array constructor as the normal entry point.
+     */
+    ArrowToSeatunnelRowReader(
+            ArrowStreamReader arrowStreamReader,
+            RootAllocator rootAllocator,
+            SeaTunnelRowType seaTunnelRowType) {
+        this.seaTunnelDataTypes = seaTunnelRowType.getFieldTypes();
+        initFieldIndexMap(seaTunnelRowType);
+        this.arrowStreamReader = arrowStreamReader;
+        this.rootAllocator = rootAllocator;
+    }
+
     private void initFieldIndexMap(SeaTunnelRowType seaTunnelRowType) {
         for (int i = 0; i < seaTunnelRowType.getFieldNames().length; i++) {
             fieldIndexMap.put(seaTunnelRowType.getFieldNames()[i], i);
@@ -319,6 +335,7 @@ public class ArrowToSeatunnelRowReader implements AutoCloseable {
 
     @Override
     public void close() {
+        RuntimeException closeException = null;
         try {
             // ArrowStreamReader must be closed before RootAllocator.
             // ArrowStreamReader internally closes VectorSchemaRoot and releases all Arrow
@@ -328,11 +345,23 @@ public class ArrowToSeatunnelRowReader implements AutoCloseable {
             if (arrowStreamReader != null) {
                 arrowStreamReader.close();
             }
-            if (rootAllocator != null) {
-                rootAllocator.close();
-            }
         } catch (IOException e) {
-            throw new RuntimeException("failed to close arrow stream reader.", e);
+            closeException = new RuntimeException("failed to close arrow stream reader.", e);
+        } finally {
+            try {
+                if (rootAllocator != null) {
+                    rootAllocator.close();
+                }
+            } catch (RuntimeException e) {
+                if (closeException == null) {
+                    closeException = e;
+                } else {
+                    closeException.addSuppressed(e);
+                }
+            }
+        }
+        if (closeException != null) {
+            throw closeException;
         }
     }
 }
diff --git a/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReaderCloseTest.java b/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReaderCloseTest.java
new file mode 100644
index 0000000000..3d373563af
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReaderCloseTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.reader;
+
+import org.apache.seatunnel.shade.org.apache.arrow.memory.RootAllocator;
+import org.apache.seatunnel.shade.org.apache.arrow.vector.ipc.ArrowStreamReader;
+
+import org.apache.seatunnel.api.table.type.BasicType;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.mockito.InOrder;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+
+/**
+ * Verifies that Arrow reader cleanup releases the allocator even when stream reader close fails.
+ */
+public class ArrowToSeatunnelRowReaderCloseTest {
+
+    /**
+     * Covers the normal close path and preserves the required reader-before-allocator order.
+     *
+     * <p>This protects the Arrow allocator from being closed before the stream reader has released
+     * vector buffers.
+     */
+    @Test
+    public void testCloseReleasesAllocatorWhenStreamReaderCloseSucceeds() throws Exception {
+        ArrowStreamReader arrowStreamReader = Mockito.mock(ArrowStreamReader.class);
+        RootAllocator rootAllocator = Mockito.mock(RootAllocator.class);
+        ArrowToSeatunnelRowReader reader =
+                new ArrowToSeatunnelRowReader(
+                        arrowStreamReader, rootAllocator, createSeatunnelRowType());
+
+        reader.close();
+
+        InOrder inOrder = Mockito.inOrder(arrowStreamReader, rootAllocator);
+        inOrder.verify(arrowStreamReader).close();
+        inOrder.verify(rootAllocator).close();
+    }
+
+    /**
+     * Covers the regression where an ArrowStreamReader close failure skipped allocator cleanup.
+     *
+     * <p>The reader close failure is still propagated, but the allocator must always receive its
+     * close call.
+     */
+    @Test
+    public void testCloseReleasesAllocatorWhenStreamReaderCloseFails() throws Exception {
+        IOException closeFailure = new IOException("close failure");
+        ArrowStreamReader arrowStreamReader = Mockito.mock(ArrowStreamReader.class);
+        Mockito.doThrow(closeFailure).when(arrowStreamReader).close();
+        RootAllocator rootAllocator = Mockito.mock(RootAllocator.class);
+        ArrowToSeatunnelRowReader reader =
+                new ArrowToSeatunnelRowReader(
+                        arrowStreamReader, rootAllocator, createSeatunnelRowType());
+
+        RuntimeException exception = Assertions.assertThrows(RuntimeException.class, reader::close);
+
+        Assertions.assertSame(closeFailure, exception.getCause());
+        InOrder inOrder = Mockito.inOrder(arrowStreamReader, rootAllocator);
+        inOrder.verify(arrowStreamReader).close();
+        inOrder.verify(rootAllocator).close();
+    }
+
+    /**
+     * Builds the minimal SeaTunnel schema needed to initialize the reader resource owner.
+     *
+     * <p>The close tests only need a valid schema for constructor initialization.
+     */
+    private SeaTunnelRowType createSeatunnelRowType() {
+        return new SeaTunnelRowType(
+                new String[] {"id"}, new SeaTunnelDataType[] {BasicType.INT_TYPE});
+    }
+}