You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by pv...@apache.org on 2026/09/16 12:52:05 UTC

(iceberg) branch main updated: Flink: Avoid per-row allocations in Parquet array and map writers (#16791)

This is an automated email from the ASF dual-hosted git repository.

pvary pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/main by this push:
     new 99826ae0be Flink: Avoid per-row allocations in Parquet array and map writers (#16791)
99826ae0be is described below

commit 99826ae0beba196703c04603ecf23745ce0b0c81
Author: Vova Kolmakov <wo...@gmail.com>
AuthorDate: Wed Sep 16 19:51:59 2026 +0700

    Flink: Avoid per-row allocations in Parquet array and map writers (#16791)
    
    Generated-by: Claude Code (claude-opus-5)
---
 .../iceberg/flink/data/FlinkParquetWriters.java    | 65 ++++++++++++----------
 1 file changed, 37 insertions(+), 28 deletions(-)

diff --git a/flink/v2.3/flink/src/main/java/org/apache/iceberg/flink/data/FlinkParquetWriters.java b/flink/v2.3/flink/src/main/java/org/apache/iceberg/flink/data/FlinkParquetWriters.java
index d9a7e0dab9..4ff46fc3e8 100644
--- a/flink/v2.3/flink/src/main/java/org/apache/iceberg/flink/data/FlinkParquetWriters.java
+++ b/flink/v2.3/flink/src/main/java/org/apache/iceberg/flink/data/FlinkParquetWriters.java
@@ -512,7 +512,7 @@ public class FlinkParquetWriters {
   }
 
   private static class ArrayDataWriter<E> extends ParquetValueWriters.RepeatedWriter<ArrayData, E> {
-    private final LogicalType elementType;
+    private final ElementIterator<E> elementIterator;
 
     private ArrayDataWriter(
         int definitionLevel,
@@ -520,25 +520,29 @@ public class FlinkParquetWriters {
         ParquetValueWriter<E> writer,
         LogicalType elementType) {
       super(definitionLevel, repetitionLevel, writer);
-      this.elementType = elementType;
+      this.elementIterator = new ElementIterator<>(ArrayData.createElementGetter(elementType));
     }
 
     @Override
     protected Iterator<E> elements(ArrayData list) {
-      return new ElementIterator<>(list);
+      elementIterator.reset(list);
+      return elementIterator;
     }
 
-    private class ElementIterator<E> implements Iterator<E> {
-      private final int size;
-      private final ArrayData list;
+    private static class ElementIterator<E> implements Iterator<E> {
       private final ArrayData.ElementGetter getter;
+      private ArrayData list;
+      private int size;
       private int index;
 
-      private ElementIterator(ArrayData list) {
-        this.list = list;
-        size = list.size();
-        getter = ArrayData.createElementGetter(elementType);
-        index = 0;
+      private ElementIterator(ArrayData.ElementGetter getter) {
+        this.getter = getter;
+      }
+
+      private void reset(ArrayData newList) {
+        this.list = newList;
+        this.size = newList.size();
+        this.index = 0;
       }
 
       @Override
@@ -563,8 +567,7 @@ public class FlinkParquetWriters {
 
   private static class MapDataWriter<K, V>
       extends ParquetValueWriters.RepeatedKeyValueWriter<MapData, K, V> {
-    private final LogicalType keyType;
-    private final LogicalType valueType;
+    private final EntryIterator<K, V> entryIterator;
 
     private MapDataWriter(
         int definitionLevel,
@@ -574,32 +577,38 @@ public class FlinkParquetWriters {
         LogicalType keyType,
         LogicalType valueType) {
       super(definitionLevel, repetitionLevel, keyWriter, valueWriter);
-      this.keyType = keyType;
-      this.valueType = valueType;
+      this.entryIterator =
+          new EntryIterator<>(
+              ArrayData.createElementGetter(keyType), ArrayData.createElementGetter(valueType));
     }
 
     @Override
     protected Iterator<Map.Entry<K, V>> pairs(MapData map) {
-      return new EntryIterator<>(map);
+      entryIterator.reset(map);
+      return entryIterator;
     }
 
-    private class EntryIterator<K, V> implements Iterator<Map.Entry<K, V>> {
-      private final int size;
-      private final ArrayData keys;
-      private final ArrayData values;
+    private static class EntryIterator<K, V> implements Iterator<Map.Entry<K, V>> {
       private final ParquetValueReaders.ReusableEntry<K, V> entry;
       private final ArrayData.ElementGetter keyGetter;
       private final ArrayData.ElementGetter valueGetter;
+      private ArrayData keys;
+      private ArrayData values;
+      private int size;
       private int index;
 
-      private EntryIterator(MapData map) {
-        size = map.size();
-        keys = map.keyArray();
-        values = map.valueArray();
-        entry = new ParquetValueReaders.ReusableEntry<>();
-        keyGetter = ArrayData.createElementGetter(keyType);
-        valueGetter = ArrayData.createElementGetter(valueType);
-        index = 0;
+      private EntryIterator(
+          ArrayData.ElementGetter keyGetter, ArrayData.ElementGetter valueGetter) {
+        this.entry = new ParquetValueReaders.ReusableEntry<>();
+        this.keyGetter = keyGetter;
+        this.valueGetter = valueGetter;
+      }
+
+      private void reset(MapData map) {
+        this.keys = map.keyArray();
+        this.values = map.valueArray();
+        this.size = map.size();
+        this.index = 0;
       }
 
       @Override