You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by li...@apache.org on 2026/08/22 10:03:07 UTC
(seatunnel) 02/02: [Fix][Connector-V2] Ensure Arrow allocator closes on reader failure
This is an automated email from the ASF dual-hosted git repository.
davidzollo pushed a commit to branch davidzollo_republish_10958_arrow_close_order
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit 1388cd978e6c80ad90a21bc591358e58f34024de
Author: DanielLeens <Da...@users.noreply.github.com>
AuthorDate: Fri Jul 31 00:20:06 2026 +0800
[Fix][Connector-V2] Ensure Arrow allocator closes on reader failure
(cherry picked from commit c088af7c9f8c3d8cdb42e7d038fce771af4fc1a9)
---
.../arrow/reader/ArrowToSeatunnelRowReader.java | 37 ++++++++-
.../reader/ArrowToSeatunnelRowReaderCloseTest.java | 93 ++++++++++++++++++++++
2 files changed, 126 insertions(+), 4 deletions(-)
diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReader.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReader.java
index 2609a591ee..0f718862ae 100644
--- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReader.java
+++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReader.java
@@ -82,6 +82,22 @@ public class ArrowToSeatunnelRowReader implements AutoCloseable {
initArrowReader(byteArray);
}
+ /**
+ * Creates a reader with prebuilt Arrow resources.
+ *
+ * <p>The constructor keeps close-order regression tests focused on resource ownership while
+ * preserving the production byte-array constructor as the normal entry point.
+ */
+ ArrowToSeatunnelRowReader(
+ ArrowStreamReader arrowStreamReader,
+ RootAllocator rootAllocator,
+ SeaTunnelRowType seaTunnelRowType) {
+ this.seaTunnelDataTypes = seaTunnelRowType.getFieldTypes();
+ initFieldIndexMap(seaTunnelRowType);
+ this.arrowStreamReader = arrowStreamReader;
+ this.rootAllocator = rootAllocator;
+ }
+
private void initFieldIndexMap(SeaTunnelRowType seaTunnelRowType) {
for (int i = 0; i < seaTunnelRowType.getFieldNames().length; i++) {
fieldIndexMap.put(seaTunnelRowType.getFieldNames()[i], i);
@@ -319,6 +335,7 @@ public class ArrowToSeatunnelRowReader implements AutoCloseable {
@Override
public void close() {
+ RuntimeException closeException = null;
try {
// ArrowStreamReader must be closed before RootAllocator.
// ArrowStreamReader internally closes VectorSchemaRoot and releases all Arrow
@@ -328,11 +345,23 @@ public class ArrowToSeatunnelRowReader implements AutoCloseable {
if (arrowStreamReader != null) {
arrowStreamReader.close();
}
- if (rootAllocator != null) {
- rootAllocator.close();
- }
} catch (IOException e) {
- throw new RuntimeException("failed to close arrow stream reader.", e);
+ closeException = new RuntimeException("failed to close arrow stream reader.", e);
+ } finally {
+ try {
+ if (rootAllocator != null) {
+ rootAllocator.close();
+ }
+ } catch (RuntimeException e) {
+ if (closeException == null) {
+ closeException = e;
+ } else {
+ closeException.addSuppressed(e);
+ }
+ }
+ }
+ if (closeException != null) {
+ throw closeException;
}
}
}
diff --git a/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReaderCloseTest.java b/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReaderCloseTest.java
new file mode 100644
index 0000000000..3d373563af
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReaderCloseTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.reader;
+
+import org.apache.seatunnel.shade.org.apache.arrow.memory.RootAllocator;
+import org.apache.seatunnel.shade.org.apache.arrow.vector.ipc.ArrowStreamReader;
+
+import org.apache.seatunnel.api.table.type.BasicType;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.mockito.InOrder;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+
+/**
+ * Verifies that Arrow reader cleanup releases the allocator even when stream reader close fails.
+ */
+public class ArrowToSeatunnelRowReaderCloseTest {
+
+ /**
+ * Covers the normal close path and preserves the required reader-before-allocator order.
+ *
+ * <p>This protects the Arrow allocator from being closed before the stream reader has released
+ * vector buffers.
+ */
+ @Test
+ public void testCloseReleasesAllocatorWhenStreamReaderCloseSucceeds() throws Exception {
+ ArrowStreamReader arrowStreamReader = Mockito.mock(ArrowStreamReader.class);
+ RootAllocator rootAllocator = Mockito.mock(RootAllocator.class);
+ ArrowToSeatunnelRowReader reader =
+ new ArrowToSeatunnelRowReader(
+ arrowStreamReader, rootAllocator, createSeatunnelRowType());
+
+ reader.close();
+
+ InOrder inOrder = Mockito.inOrder(arrowStreamReader, rootAllocator);
+ inOrder.verify(arrowStreamReader).close();
+ inOrder.verify(rootAllocator).close();
+ }
+
+ /**
+ * Covers the regression where an ArrowStreamReader close failure skipped allocator cleanup.
+ *
+ * <p>The reader close failure is still propagated, but the allocator must always receive its
+ * close call.
+ */
+ @Test
+ public void testCloseReleasesAllocatorWhenStreamReaderCloseFails() throws Exception {
+ IOException closeFailure = new IOException("close failure");
+ ArrowStreamReader arrowStreamReader = Mockito.mock(ArrowStreamReader.class);
+ Mockito.doThrow(closeFailure).when(arrowStreamReader).close();
+ RootAllocator rootAllocator = Mockito.mock(RootAllocator.class);
+ ArrowToSeatunnelRowReader reader =
+ new ArrowToSeatunnelRowReader(
+ arrowStreamReader, rootAllocator, createSeatunnelRowType());
+
+ RuntimeException exception = Assertions.assertThrows(RuntimeException.class, reader::close);
+
+ Assertions.assertSame(closeFailure, exception.getCause());
+ InOrder inOrder = Mockito.inOrder(arrowStreamReader, rootAllocator);
+ inOrder.verify(arrowStreamReader).close();
+ inOrder.verify(rootAllocator).close();
+ }
+
+ /**
+ * Builds the minimal SeaTunnel schema needed to initialize the reader resource owner.
+ *
+ * <p>The close tests only need a valid schema for constructor initialization.
+ */
+ private SeaTunnelRowType createSeatunnelRowType() {
+ return new SeaTunnelRowType(
+ new String[] {"id"}, new SeaTunnelDataType[] {BasicType.INT_TYPE});
+ }
+}