You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by li...@apache.org on 2025/08/13 23:39:26 UTC
(arrow-java) branch main updated: GH-816: Presize JsonStringArrayList vector results (#817)
This is an automated email from the ASF dual-hosted git repository.
lidavidm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-java.git
The following commit(s) were added to refs/heads/main by this push:
new 52f7a86b GH-816: Presize JsonStringArrayList vector results (#817)
52f7a86b is described below
commit 52f7a86b0f315df8848dd68cb7457e13236b5fd5
Author: David Schlosnagle <da...@palantir.com>
AuthorDate: Wed Aug 13 19:39:22 2025 -0400
GH-816: Presize JsonStringArrayList vector results (#817)
## What's Changed
Presize `JsonStringArrayList`s when constructing them as part of
`ValueVector#getObject` conversions.
`FixedSizeListVector#getObject` already performs this optimization;
however, `ListVector`, `ListViewVector`, `LargeListVector`, and
`LargeListViewVector` do not yet presize the result
`JsonStringArrayList` requiring dynamic reallocations as elements are
converted & added. This can become a scalability bottleneck when using
these types.
Closes #816.
---
.../java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java | 4 ++--
.../apache/arrow/adapter/avro/AvroToArrowVectorIterator.java | 11 +++++++----
.../java/org/apache/arrow/vector/complex/LargeListVector.java | 4 +++-
.../org/apache/arrow/vector/complex/LargeListViewVector.java | 2 +-
.../main/java/org/apache/arrow/vector/complex/ListVector.java | 2 +-
.../java/org/apache/arrow/vector/complex/ListViewVector.java | 2 +-
6 files changed, 15 insertions(+), 10 deletions(-)
diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java
index aedef773..a6e77e40 100644
--- a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java
+++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java
@@ -1071,8 +1071,8 @@ public class AvroToArrowUtils {
}
private static String convertAliases(Set<String> aliases) {
- JsonStringArrayList jsonList = new JsonStringArrayList();
- aliases.stream().forEach(a -> jsonList.add(a));
+ JsonStringArrayList jsonList = new JsonStringArrayList(aliases.size());
+ jsonList.addAll(aliases);
return jsonList.toString();
}
}
diff --git a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowVectorIterator.java b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowVectorIterator.java
index 41233700..e82fdc36 100644
--- a/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowVectorIterator.java
+++ b/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowVectorIterator.java
@@ -17,13 +17,14 @@
package org.apache.arrow.adapter.avro;
import java.io.EOFException;
-import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.arrow.adapter.avro.consumers.CompositeAvroConsumer;
+import org.apache.arrow.adapter.avro.consumers.Consumer;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.ValueVectorUtility;
@@ -75,9 +76,11 @@ public class AvroToArrowVectorIterator implements Iterator<VectorSchemaRoot>, Au
private void initialize() {
// create consumers
compositeConsumer = AvroToArrowUtils.createCompositeConsumer(schema, config);
- List<FieldVector> vectors = new ArrayList<>();
- compositeConsumer.getConsumers().forEach(c -> vectors.add(c.getVector()));
- List<Field> fields = vectors.stream().map(t -> t.getField()).collect(Collectors.toList());
+ List<FieldVector> vectors =
+ compositeConsumer.getConsumers().stream()
+ .map(Consumer::getVector)
+ .collect(Collectors.toList());
+ List<Field> fields = vectors.stream().map(ValueVector::getField).collect(Collectors.toList());
VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, 0);
rootSchema = root.getSchema();
diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java
index 835d3468..997b5a8b 100644
--- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java
+++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java
@@ -31,6 +31,7 @@ import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.memory.util.ArrowBufPointer;
import org.apache.arrow.memory.util.ByteFunctionHelpers;
import org.apache.arrow.memory.util.CommonUtil;
+import org.apache.arrow.memory.util.LargeMemoryUtil;
import org.apache.arrow.memory.util.hash.ArrowBufHasher;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.AddOrGetResult;
@@ -861,10 +862,11 @@ public class LargeListVector extends BaseValueVector
if (isSet(index) == 0) {
return null;
}
- final List<Object> vals = new JsonStringArrayList<>();
final long start = offsetBuffer.getLong((long) index * OFFSET_WIDTH);
final long end = offsetBuffer.getLong(((long) index + 1L) * OFFSET_WIDTH);
final ValueVector vv = getDataVector();
+ final List<Object> vals =
+ new JsonStringArrayList<>(LargeMemoryUtil.checkedCastToInt(end - start));
for (long i = start; i < end; i++) {
vals.add(vv.getObject(checkedCastToInt(i)));
}
diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java
index 394c3c67..2da7eb05 100644
--- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java
+++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java
@@ -672,10 +672,10 @@ public class LargeListViewVector extends BaseLargeRepeatedValueViewVector
if (isSet(index) == 0) {
return null;
}
- final List<Object> vals = new JsonStringArrayList<>();
final int start = offsetBuffer.getInt(index * OFFSET_WIDTH);
final int end = start + sizeBuffer.getInt((index) * SIZE_WIDTH);
final ValueVector vv = getDataVector();
+ final List<Object> vals = new JsonStringArrayList<>(end - start);
for (int i = start; i < end; i++) {
vals.add(vv.getObject(checkedCastToInt(i)));
}
diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 2b281751..93a313ef 100644
--- a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -719,10 +719,10 @@ public class ListVector extends BaseRepeatedValueVector
if (isSet(index) == 0) {
return null;
}
- final List<Object> vals = new JsonStringArrayList<>();
final int start = offsetBuffer.getInt(index * OFFSET_WIDTH);
final int end = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH);
final ValueVector vv = getDataVector();
+ final List<Object> vals = new JsonStringArrayList<>(end - start);
for (int i = start; i < end; i++) {
vals.add(vv.getObject(i));
}
diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java
index 2b801019..8711db5e 100644
--- a/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java
+++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java
@@ -678,10 +678,10 @@ public class ListViewVector extends BaseRepeatedValueViewVector
if (isSet(index) == 0) {
return null;
}
- final List<Object> vals = new JsonStringArrayList<>();
final int start = offsetBuffer.getInt(index * OFFSET_WIDTH);
final int end = start + sizeBuffer.getInt((index) * SIZE_WIDTH);
final ValueVector vv = getDataVector();
+ final List<Object> vals = new JsonStringArrayList<>(end - start);
for (int i = start; i < end; i++) {
vals.add(vv.getObject(i));
}