You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by ju...@apache.org on 2024/08/23 07:12:47 UTC
(paimon) branch master updated: [arrow] Pass arrow reader last value by serializer to binary row (#4042)
This is an automated email from the ASF dual-hosted git repository.
junhao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 813cb6fa7 [arrow] Pass arrow reader last value by serializer to binary row (#4042)
813cb6fa7 is described below
commit 813cb6fa792d72b1a9587a6fc7db396129de2275
Author: YeJunHao <41...@users.noreply.github.com>
AuthorDate: Fri Aug 23 15:12:43 2024 +0800
[arrow] Pass arrow reader last value by serializer to binary row (#4042)
---
.../converter/Arrow2PaimonVectorConverter.java | 9 ++-
.../paimon/arrow/reader/ArrowBatchReader.java | 9 ++-
.../paimon/arrow/vector/ArrowFormatCWriter.java | 81 ++++++++++++++++++++++
.../paimon/arrow/vector/ArrowFormatWriter.java | 18 ++---
.../paimon/arrow/vector/ArrowFormatWriterTest.java | 35 ++++++++++
5 files changed, 137 insertions(+), 15 deletions(-)
diff --git a/paimon-arrow/src/main/java/org/apache/paimon/arrow/converter/Arrow2PaimonVectorConverter.java b/paimon-arrow/src/main/java/org/apache/paimon/arrow/converter/Arrow2PaimonVectorConverter.java
index 04e9273e4..6536ae924 100644
--- a/paimon-arrow/src/main/java/org/apache/paimon/arrow/converter/Arrow2PaimonVectorConverter.java
+++ b/paimon-arrow/src/main/java/org/apache/paimon/arrow/converter/Arrow2PaimonVectorConverter.java
@@ -69,6 +69,7 @@ import org.apache.arrow.vector.BitVector;
import org.apache.arrow.vector.DateDayVector;
import org.apache.arrow.vector.DecimalVector;
import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.FixedSizeBinaryVector;
import org.apache.arrow.vector.Float4Vector;
import org.apache.arrow.vector.Float8Vector;
import org.apache.arrow.vector.IntVector;
@@ -112,7 +113,13 @@ public interface Arrow2PaimonVectorConverter {
@Override
public Bytes getBytes(int index) {
- byte[] bytes = ((VarCharVector) vector).get(index);
+ byte[] bytes;
+ if (vector instanceof FixedSizeBinaryVector) {
+ bytes = ((FixedSizeBinaryVector) vector).get(index);
+ } else {
+ bytes = ((VarCharVector) vector).get(index);
+ }
+
return new Bytes(bytes, 0, bytes.length) {
@Override
public byte[] getBytes() {
diff --git a/paimon-arrow/src/main/java/org/apache/paimon/arrow/reader/ArrowBatchReader.java b/paimon-arrow/src/main/java/org/apache/paimon/arrow/reader/ArrowBatchReader.java
index b8d32e57d..364bf4884 100644
--- a/paimon-arrow/src/main/java/org/apache/paimon/arrow/reader/ArrowBatchReader.java
+++ b/paimon-arrow/src/main/java/org/apache/paimon/arrow/reader/ArrowBatchReader.java
@@ -23,6 +23,7 @@ import org.apache.paimon.data.InternalRow;
import org.apache.paimon.data.columnar.ColumnVector;
import org.apache.paimon.data.columnar.ColumnarRow;
import org.apache.paimon.data.columnar.VectorizedColumnBatch;
+import org.apache.paimon.data.serializer.InternalRowSerializer;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.RowType;
@@ -36,11 +37,13 @@ import java.util.List;
/** Reader from a {@link VectorSchemaRoot} to paimon rows. */
public class ArrowBatchReader {
+ private final InternalRowSerializer internalRowSerializer;
private final VectorizedColumnBatch batch;
private final Arrow2PaimonVectorConverter[] convertors;
private final RowType projectedRowType;
public ArrowBatchReader(RowType rowType) {
+ this.internalRowSerializer = new InternalRowSerializer(rowType);
ColumnVector[] columnVectors = new ColumnVector[rowType.getFieldCount()];
this.convertors = new Arrow2PaimonVectorConverter[rowType.getFieldCount()];
this.batch = new VectorizedColumnBatch(columnVectors);
@@ -85,7 +88,11 @@ public class ArrowBatchReader {
public InternalRow next() {
columnarRow.setRowId(position);
position++;
- return columnarRow;
+ // when pk merge, the last value will be referenced by maxKey. If reader
+ // close, the maxKey will be useless. We must avoid this situation
+ return position == rowCount
+ ? internalRowSerializer.toBinaryRow(columnarRow)
+ : columnarRow;
}
};
}
diff --git a/paimon-arrow/src/main/java/org/apache/paimon/arrow/vector/ArrowFormatCWriter.java b/paimon-arrow/src/main/java/org/apache/paimon/arrow/vector/ArrowFormatCWriter.java
new file mode 100644
index 000000000..989019daf
--- /dev/null
+++ b/paimon-arrow/src/main/java/org/apache/paimon/arrow/vector/ArrowFormatCWriter.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.arrow.vector;
+
+import org.apache.paimon.arrow.ArrowUtils;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.types.RowType;
+
+import org.apache.arrow.c.ArrowArray;
+import org.apache.arrow.c.ArrowSchema;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
+
+/**
+ * This writer could flush to c struct, but you need to release it, except it has been released in c
+ * code.
+ */
+public class ArrowFormatCWriter implements AutoCloseable {
+
+ private final ArrowArray array;
+ private final ArrowSchema schema;
+ private final ArrowFormatWriter realWriter;
+
+ public ArrowFormatCWriter(RowType rowType, int writeBatchSize) {
+ this.realWriter = new ArrowFormatWriter(rowType, writeBatchSize);
+ RootAllocator allocator = realWriter.getAllocator();
+ array = ArrowArray.allocateNew(allocator);
+ schema = ArrowSchema.allocateNew(allocator);
+ }
+
+ public boolean write(InternalRow currentRow) {
+ return realWriter.write(currentRow);
+ }
+
+ public ArrowCStruct flush() {
+ realWriter.flush();
+ VectorSchemaRoot vectorSchemaRoot = realWriter.getVectorSchemaRoot();
+ return ArrowUtils.serializeToCStruct(vectorSchemaRoot, array, schema);
+ }
+
+ public boolean empty() {
+ return realWriter.empty();
+ }
+
+ // if c++ code release this, we don't need to release again (release twice may cause a problem)
+ public void release() {
+ array.release();
+ schema.release();
+ }
+
+ @Override
+ public void close() {
+ array.close();
+ schema.close();
+ realWriter.close();
+ }
+
+ public VectorSchemaRoot getVectorSchemaRoot() {
+ return realWriter.getVectorSchemaRoot();
+ }
+
+ public RootAllocator getAllocator() {
+ return realWriter.getAllocator();
+ }
+}
diff --git a/paimon-arrow/src/main/java/org/apache/paimon/arrow/vector/ArrowFormatWriter.java b/paimon-arrow/src/main/java/org/apache/paimon/arrow/vector/ArrowFormatWriter.java
index e5d7289c3..faa326557 100644
--- a/paimon-arrow/src/main/java/org/apache/paimon/arrow/vector/ArrowFormatWriter.java
+++ b/paimon-arrow/src/main/java/org/apache/paimon/arrow/vector/ArrowFormatWriter.java
@@ -24,8 +24,6 @@ import org.apache.paimon.arrow.writer.ArrowFieldWriterFactoryVisitor;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.types.RowType;
-import org.apache.arrow.c.ArrowArray;
-import org.apache.arrow.c.ArrowSchema;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.util.OversizedAllocationException;
@@ -35,8 +33,6 @@ public class ArrowFormatWriter implements AutoCloseable {
private final VectorSchemaRoot vectorSchemaRoot;
private final ArrowFieldWriter[] fieldWriters;
- private final ArrowArray array;
- private final ArrowSchema schema;
private final int batchSize;
@@ -45,8 +41,6 @@ public class ArrowFormatWriter implements AutoCloseable {
public ArrowFormatWriter(RowType rowType, int writeBatchSize) {
allocator = new RootAllocator();
- array = ArrowArray.allocateNew(allocator);
- schema = ArrowSchema.allocateNew(allocator);
vectorSchemaRoot = ArrowUtils.createVectorSchemaRoot(rowType, allocator, false);
@@ -64,11 +58,9 @@ public class ArrowFormatWriter implements AutoCloseable {
this.batchSize = writeBatchSize;
}
- public ArrowCStruct flush() {
+ public void flush() {
vectorSchemaRoot.setRowCount(rowId);
- ArrowCStruct arrowCStruct = ArrowUtils.serializeToCStruct(vectorSchemaRoot, array, schema);
rowId = 0;
- return arrowCStruct;
}
public boolean write(InternalRow currentRow) {
@@ -94,10 +86,6 @@ public class ArrowFormatWriter implements AutoCloseable {
@Override
public void close() {
- array.release();
- schema.release();
- array.close();
- schema.close();
vectorSchemaRoot.close();
allocator.close();
}
@@ -105,4 +93,8 @@ public class ArrowFormatWriter implements AutoCloseable {
public VectorSchemaRoot getVectorSchemaRoot() {
return vectorSchemaRoot;
}
+
+ public RootAllocator getAllocator() {
+ return allocator;
+ }
}
diff --git a/paimon-arrow/src/test/java/org/apache/paimon/arrow/vector/ArrowFormatWriterTest.java b/paimon-arrow/src/test/java/org/apache/paimon/arrow/vector/ArrowFormatWriterTest.java
index dc3b197a3..c27d167d3 100644
--- a/paimon-arrow/src/test/java/org/apache/paimon/arrow/vector/ArrowFormatWriterTest.java
+++ b/paimon-arrow/src/test/java/org/apache/paimon/arrow/vector/ArrowFormatWriterTest.java
@@ -157,6 +157,41 @@ public class ArrowFormatWriterTest {
}
}
+ @Test
+ public void testCWriter() {
+ try (ArrowFormatCWriter writer = new ArrowFormatCWriter(PRIMITIVE_TYPE, 4096)) {
+ List<InternalRow> list = new ArrayList<>();
+ List<InternalRow.FieldGetter> fieldGetters = new ArrayList<>();
+
+ for (int i = 0; i < PRIMITIVE_TYPE.getFieldCount(); i++) {
+ fieldGetters.add(InternalRow.createFieldGetter(PRIMITIVE_TYPE.getTypeAt(i), i));
+ }
+ for (int i = 0; i < 1000; i++) {
+ list.add(GenericRow.of(randomRowValues(null)));
+ }
+
+ list.forEach(writer::write);
+
+ writer.flush();
+ VectorSchemaRoot vectorSchemaRoot = writer.getVectorSchemaRoot();
+
+ ArrowBatchReader arrowBatchReader = new ArrowBatchReader(PRIMITIVE_TYPE);
+ Iterable<InternalRow> rows = arrowBatchReader.readBatch(vectorSchemaRoot);
+
+ Iterator<InternalRow> iterator = rows.iterator();
+ for (int i = 0; i < 1000; i++) {
+ InternalRow actual = iterator.next();
+ InternalRow expectec = list.get(i);
+
+ for (InternalRow.FieldGetter fieldGetter : fieldGetters) {
+ Assertions.assertThat(fieldGetter.getFieldOrNull(actual))
+ .isEqualTo(fieldGetter.getFieldOrNull(expectec));
+ }
+ }
+ writer.release();
+ }
+ }
+
private Object[] randomRowValues(boolean[] nullable) {
Object[] values = new Object[18];
values[0] = BinaryString.fromString(StringUtils.getRandomString(RND, 10, 10));