You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by ju...@apache.org on 2024/08/23 07:12:47 UTC

(paimon) branch master updated: [arrow] Pass arrow reader last value by serializer to binary row (#4042)

This is an automated email from the ASF dual-hosted git repository.

junhao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new 813cb6fa7 [arrow] Pass arrow reader last value by serializer to binary row (#4042)
813cb6fa7 is described below

commit 813cb6fa792d72b1a9587a6fc7db396129de2275
Author: YeJunHao <41...@users.noreply.github.com>
AuthorDate: Fri Aug 23 15:12:43 2024 +0800

    [arrow] Pass arrow reader last value by serializer to binary row (#4042)
---
 .../converter/Arrow2PaimonVectorConverter.java     |  9 ++-
 .../paimon/arrow/reader/ArrowBatchReader.java      |  9 ++-
 .../paimon/arrow/vector/ArrowFormatCWriter.java    | 81 ++++++++++++++++++++++
 .../paimon/arrow/vector/ArrowFormatWriter.java     | 18 ++---
 .../paimon/arrow/vector/ArrowFormatWriterTest.java | 35 ++++++++++
 5 files changed, 137 insertions(+), 15 deletions(-)

diff --git a/paimon-arrow/src/main/java/org/apache/paimon/arrow/converter/Arrow2PaimonVectorConverter.java b/paimon-arrow/src/main/java/org/apache/paimon/arrow/converter/Arrow2PaimonVectorConverter.java
index 04e9273e4..6536ae924 100644
--- a/paimon-arrow/src/main/java/org/apache/paimon/arrow/converter/Arrow2PaimonVectorConverter.java
+++ b/paimon-arrow/src/main/java/org/apache/paimon/arrow/converter/Arrow2PaimonVectorConverter.java
@@ -69,6 +69,7 @@ import org.apache.arrow.vector.BitVector;
 import org.apache.arrow.vector.DateDayVector;
 import org.apache.arrow.vector.DecimalVector;
 import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.FixedSizeBinaryVector;
 import org.apache.arrow.vector.Float4Vector;
 import org.apache.arrow.vector.Float8Vector;
 import org.apache.arrow.vector.IntVector;
@@ -112,7 +113,13 @@ public interface Arrow2PaimonVectorConverter {
 
                         @Override
                         public Bytes getBytes(int index) {
-                            byte[] bytes = ((VarCharVector) vector).get(index);
+                            byte[] bytes;
+                            if (vector instanceof FixedSizeBinaryVector) {
+                                bytes = ((FixedSizeBinaryVector) vector).get(index);
+                            } else {
+                                bytes = ((VarCharVector) vector).get(index);
+                            }
+
                             return new Bytes(bytes, 0, bytes.length) {
                                 @Override
                                 public byte[] getBytes() {
diff --git a/paimon-arrow/src/main/java/org/apache/paimon/arrow/reader/ArrowBatchReader.java b/paimon-arrow/src/main/java/org/apache/paimon/arrow/reader/ArrowBatchReader.java
index b8d32e57d..364bf4884 100644
--- a/paimon-arrow/src/main/java/org/apache/paimon/arrow/reader/ArrowBatchReader.java
+++ b/paimon-arrow/src/main/java/org/apache/paimon/arrow/reader/ArrowBatchReader.java
@@ -23,6 +23,7 @@ import org.apache.paimon.data.InternalRow;
 import org.apache.paimon.data.columnar.ColumnVector;
 import org.apache.paimon.data.columnar.ColumnarRow;
 import org.apache.paimon.data.columnar.VectorizedColumnBatch;
+import org.apache.paimon.data.serializer.InternalRowSerializer;
 import org.apache.paimon.types.DataField;
 import org.apache.paimon.types.RowType;
 
@@ -36,11 +37,13 @@ import java.util.List;
 /** Reader from a {@link VectorSchemaRoot} to paimon rows. */
 public class ArrowBatchReader {
 
+    private final InternalRowSerializer internalRowSerializer;
     private final VectorizedColumnBatch batch;
     private final Arrow2PaimonVectorConverter[] convertors;
     private final RowType projectedRowType;
 
     public ArrowBatchReader(RowType rowType) {
+        this.internalRowSerializer = new InternalRowSerializer(rowType);
         ColumnVector[] columnVectors = new ColumnVector[rowType.getFieldCount()];
         this.convertors = new Arrow2PaimonVectorConverter[rowType.getFieldCount()];
         this.batch = new VectorizedColumnBatch(columnVectors);
@@ -85,7 +88,11 @@ public class ArrowBatchReader {
                     public InternalRow next() {
                         columnarRow.setRowId(position);
                         position++;
-                        return columnarRow;
+                        // when pk merge, the last value will be referenced by maxKey. If reader
+                        // close, the maxKey will be useless. We must avoid this situation
+                        return position == rowCount
+                                ? internalRowSerializer.toBinaryRow(columnarRow)
+                                : columnarRow;
                     }
                 };
     }
diff --git a/paimon-arrow/src/main/java/org/apache/paimon/arrow/vector/ArrowFormatCWriter.java b/paimon-arrow/src/main/java/org/apache/paimon/arrow/vector/ArrowFormatCWriter.java
new file mode 100644
index 000000000..989019daf
--- /dev/null
+++ b/paimon-arrow/src/main/java/org/apache/paimon/arrow/vector/ArrowFormatCWriter.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.arrow.vector;
+
+import org.apache.paimon.arrow.ArrowUtils;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.types.RowType;
+
+import org.apache.arrow.c.ArrowArray;
+import org.apache.arrow.c.ArrowSchema;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
+
+/**
+ * This writer could flush to c struct, but you need to release it, except it has been released in c
+ * code.
+ */
+public class ArrowFormatCWriter implements AutoCloseable {
+
+    private final ArrowArray array;
+    private final ArrowSchema schema;
+    private final ArrowFormatWriter realWriter;
+
+    public ArrowFormatCWriter(RowType rowType, int writeBatchSize) {
+        this.realWriter = new ArrowFormatWriter(rowType, writeBatchSize);
+        RootAllocator allocator = realWriter.getAllocator();
+        array = ArrowArray.allocateNew(allocator);
+        schema = ArrowSchema.allocateNew(allocator);
+    }
+
+    public boolean write(InternalRow currentRow) {
+        return realWriter.write(currentRow);
+    }
+
+    public ArrowCStruct flush() {
+        realWriter.flush();
+        VectorSchemaRoot vectorSchemaRoot = realWriter.getVectorSchemaRoot();
+        return ArrowUtils.serializeToCStruct(vectorSchemaRoot, array, schema);
+    }
+
+    public boolean empty() {
+        return realWriter.empty();
+    }
+
+    // if c++ code release this, we don't need to release again (release twice may cause a problem)
+    public void release() {
+        array.release();
+        schema.release();
+    }
+
+    @Override
+    public void close() {
+        array.close();
+        schema.close();
+        realWriter.close();
+    }
+
+    public VectorSchemaRoot getVectorSchemaRoot() {
+        return realWriter.getVectorSchemaRoot();
+    }
+
+    public RootAllocator getAllocator() {
+        return realWriter.getAllocator();
+    }
+}
diff --git a/paimon-arrow/src/main/java/org/apache/paimon/arrow/vector/ArrowFormatWriter.java b/paimon-arrow/src/main/java/org/apache/paimon/arrow/vector/ArrowFormatWriter.java
index e5d7289c3..faa326557 100644
--- a/paimon-arrow/src/main/java/org/apache/paimon/arrow/vector/ArrowFormatWriter.java
+++ b/paimon-arrow/src/main/java/org/apache/paimon/arrow/vector/ArrowFormatWriter.java
@@ -24,8 +24,6 @@ import org.apache.paimon.arrow.writer.ArrowFieldWriterFactoryVisitor;
 import org.apache.paimon.data.InternalRow;
 import org.apache.paimon.types.RowType;
 
-import org.apache.arrow.c.ArrowArray;
-import org.apache.arrow.c.ArrowSchema;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.util.OversizedAllocationException;
@@ -35,8 +33,6 @@ public class ArrowFormatWriter implements AutoCloseable {
 
     private final VectorSchemaRoot vectorSchemaRoot;
     private final ArrowFieldWriter[] fieldWriters;
-    private final ArrowArray array;
-    private final ArrowSchema schema;
 
     private final int batchSize;
 
@@ -45,8 +41,6 @@ public class ArrowFormatWriter implements AutoCloseable {
 
     public ArrowFormatWriter(RowType rowType, int writeBatchSize) {
         allocator = new RootAllocator();
-        array = ArrowArray.allocateNew(allocator);
-        schema = ArrowSchema.allocateNew(allocator);
 
         vectorSchemaRoot = ArrowUtils.createVectorSchemaRoot(rowType, allocator, false);
 
@@ -64,11 +58,9 @@ public class ArrowFormatWriter implements AutoCloseable {
         this.batchSize = writeBatchSize;
     }
 
-    public ArrowCStruct flush() {
+    public void flush() {
         vectorSchemaRoot.setRowCount(rowId);
-        ArrowCStruct arrowCStruct = ArrowUtils.serializeToCStruct(vectorSchemaRoot, array, schema);
         rowId = 0;
-        return arrowCStruct;
     }
 
     public boolean write(InternalRow currentRow) {
@@ -94,10 +86,6 @@ public class ArrowFormatWriter implements AutoCloseable {
 
     @Override
     public void close() {
-        array.release();
-        schema.release();
-        array.close();
-        schema.close();
         vectorSchemaRoot.close();
         allocator.close();
     }
@@ -105,4 +93,8 @@ public class ArrowFormatWriter implements AutoCloseable {
     public VectorSchemaRoot getVectorSchemaRoot() {
         return vectorSchemaRoot;
     }
+
+    public RootAllocator getAllocator() {
+        return allocator;
+    }
 }
diff --git a/paimon-arrow/src/test/java/org/apache/paimon/arrow/vector/ArrowFormatWriterTest.java b/paimon-arrow/src/test/java/org/apache/paimon/arrow/vector/ArrowFormatWriterTest.java
index dc3b197a3..c27d167d3 100644
--- a/paimon-arrow/src/test/java/org/apache/paimon/arrow/vector/ArrowFormatWriterTest.java
+++ b/paimon-arrow/src/test/java/org/apache/paimon/arrow/vector/ArrowFormatWriterTest.java
@@ -157,6 +157,41 @@ public class ArrowFormatWriterTest {
         }
     }
 
+    @Test
+    public void testCWriter() {
+        try (ArrowFormatCWriter writer = new ArrowFormatCWriter(PRIMITIVE_TYPE, 4096)) {
+            List<InternalRow> list = new ArrayList<>();
+            List<InternalRow.FieldGetter> fieldGetters = new ArrayList<>();
+
+            for (int i = 0; i < PRIMITIVE_TYPE.getFieldCount(); i++) {
+                fieldGetters.add(InternalRow.createFieldGetter(PRIMITIVE_TYPE.getTypeAt(i), i));
+            }
+            for (int i = 0; i < 1000; i++) {
+                list.add(GenericRow.of(randomRowValues(null)));
+            }
+
+            list.forEach(writer::write);
+
+            writer.flush();
+            VectorSchemaRoot vectorSchemaRoot = writer.getVectorSchemaRoot();
+
+            ArrowBatchReader arrowBatchReader = new ArrowBatchReader(PRIMITIVE_TYPE);
+            Iterable<InternalRow> rows = arrowBatchReader.readBatch(vectorSchemaRoot);
+
+            Iterator<InternalRow> iterator = rows.iterator();
+            for (int i = 0; i < 1000; i++) {
+                InternalRow actual = iterator.next();
+                InternalRow expectec = list.get(i);
+
+                for (InternalRow.FieldGetter fieldGetter : fieldGetters) {
+                    Assertions.assertThat(fieldGetter.getFieldOrNull(actual))
+                            .isEqualTo(fieldGetter.getFieldOrNull(expectec));
+                }
+            }
+            writer.release();
+        }
+    }
+
     private Object[] randomRowValues(boolean[] nullable) {
         Object[] values = new Object[18];
         values[0] = BinaryString.fromString(StringUtils.getRandomString(RND, 10, 10));