You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by pv...@apache.org on 2026/09/16 12:52:05 UTC
(iceberg) branch main updated: Flink: Avoid per-row allocations in Parquet array and map writers (#16791)
This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 99826ae0be Flink: Avoid per-row allocations in Parquet array and map writers (#16791)
99826ae0be is described below
commit 99826ae0beba196703c04603ecf23745ce0b0c81
Author: Vova Kolmakov <wo...@gmail.com>
AuthorDate: Wed Sep 16 19:51:59 2026 +0700
Flink: Avoid per-row allocations in Parquet array and map writers (#16791)
Generated-by: Claude Code (claude-opus-5)
---
.../iceberg/flink/data/FlinkParquetWriters.java | 65 ++++++++++++----------
1 file changed, 37 insertions(+), 28 deletions(-)
diff --git a/flink/v2.3/flink/src/main/java/org/apache/iceberg/flink/data/FlinkParquetWriters.java b/flink/v2.3/flink/src/main/java/org/apache/iceberg/flink/data/FlinkParquetWriters.java
index d9a7e0dab9..4ff46fc3e8 100644
--- a/flink/v2.3/flink/src/main/java/org/apache/iceberg/flink/data/FlinkParquetWriters.java
+++ b/flink/v2.3/flink/src/main/java/org/apache/iceberg/flink/data/FlinkParquetWriters.java
@@ -512,7 +512,7 @@ public class FlinkParquetWriters {
}
private static class ArrayDataWriter<E> extends ParquetValueWriters.RepeatedWriter<ArrayData, E> {
- private final LogicalType elementType;
+ private final ElementIterator<E> elementIterator;
private ArrayDataWriter(
int definitionLevel,
@@ -520,25 +520,29 @@ public class FlinkParquetWriters {
ParquetValueWriter<E> writer,
LogicalType elementType) {
super(definitionLevel, repetitionLevel, writer);
- this.elementType = elementType;
+ this.elementIterator = new ElementIterator<>(ArrayData.createElementGetter(elementType));
}
@Override
protected Iterator<E> elements(ArrayData list) {
- return new ElementIterator<>(list);
+ elementIterator.reset(list);
+ return elementIterator;
}
- private class ElementIterator<E> implements Iterator<E> {
- private final int size;
- private final ArrayData list;
+ private static class ElementIterator<E> implements Iterator<E> {
private final ArrayData.ElementGetter getter;
+ private ArrayData list;
+ private int size;
private int index;
- private ElementIterator(ArrayData list) {
- this.list = list;
- size = list.size();
- getter = ArrayData.createElementGetter(elementType);
- index = 0;
+ private ElementIterator(ArrayData.ElementGetter getter) {
+ this.getter = getter;
+ }
+
+ private void reset(ArrayData newList) {
+ this.list = newList;
+ this.size = newList.size();
+ this.index = 0;
}
@Override
@@ -563,8 +567,7 @@ public class FlinkParquetWriters {
private static class MapDataWriter<K, V>
extends ParquetValueWriters.RepeatedKeyValueWriter<MapData, K, V> {
- private final LogicalType keyType;
- private final LogicalType valueType;
+ private final EntryIterator<K, V> entryIterator;
private MapDataWriter(
int definitionLevel,
@@ -574,32 +577,38 @@ public class FlinkParquetWriters {
LogicalType keyType,
LogicalType valueType) {
super(definitionLevel, repetitionLevel, keyWriter, valueWriter);
- this.keyType = keyType;
- this.valueType = valueType;
+ this.entryIterator =
+ new EntryIterator<>(
+ ArrayData.createElementGetter(keyType), ArrayData.createElementGetter(valueType));
}
@Override
protected Iterator<Map.Entry<K, V>> pairs(MapData map) {
- return new EntryIterator<>(map);
+ entryIterator.reset(map);
+ return entryIterator;
}
- private class EntryIterator<K, V> implements Iterator<Map.Entry<K, V>> {
- private final int size;
- private final ArrayData keys;
- private final ArrayData values;
+ private static class EntryIterator<K, V> implements Iterator<Map.Entry<K, V>> {
private final ParquetValueReaders.ReusableEntry<K, V> entry;
private final ArrayData.ElementGetter keyGetter;
private final ArrayData.ElementGetter valueGetter;
+ private ArrayData keys;
+ private ArrayData values;
+ private int size;
private int index;
- private EntryIterator(MapData map) {
- size = map.size();
- keys = map.keyArray();
- values = map.valueArray();
- entry = new ParquetValueReaders.ReusableEntry<>();
- keyGetter = ArrayData.createElementGetter(keyType);
- valueGetter = ArrayData.createElementGetter(valueType);
- index = 0;
+ private EntryIterator(
+ ArrayData.ElementGetter keyGetter, ArrayData.ElementGetter valueGetter) {
+ this.entry = new ParquetValueReaders.ReusableEntry<>();
+ this.keyGetter = keyGetter;
+ this.valueGetter = valueGetter;
+ }
+
+ private void reset(MapData map) {
+ this.keys = map.keyArray();
+ this.values = map.valueArray();
+ this.size = map.size();
+ this.index = 0;
}
@Override