You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by hu...@apache.org on 2026/08/11 23:39:15 UTC

(iceberg) branch main updated: Core: Support metrics with nested field in AppendBenchmark and FileGenerationUtil (#17391)

This is an automated email from the ASF dual-hosted git repository.

huaxingao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/main by this push:
     new e31bda3c1a Core: Support metrics with nested field in AppendBenchmark and FileGenerationUtil (#17391)
e31bda3c1a is described below

commit e31bda3c1a04c3c0afab0cd3098f9a976300a4a2
Author: Hongyue/Steve Zhang <ho...@gmail.com>
AuthorDate: Tue Aug 11 16:39:07 2026 -0700

    Core: Support metrics with nested field in AppendBenchmark and FileGenerationUtil (#17391)
    
    * Core: Add nested field traversal in AppendBenchmark and FileGenerationUtil
    
    * expand testBoundsForNestedTypes to cover all metricsMode
    
    * Core: Resolve metrics mode by field id in FileGenerationUtil
    
    Use MetricsConfig.columnMode(int) instead of manually resolving the
    column name via Schema.findColumnName. The int overload performs the
    same id-to-name lookup internally and correctly handles column aliases,
    removing duplicated logic across the generator and its test.
    
    Co-Authored-By: Claude Opus 4.8 (1M context) <no...@anthropic.com>
    
    * fix schema used in UT
    
    ---------
    
    Co-authored-by: Claude Opus 4.8 (1M context) <no...@anthropic.com>
---
 .../java/org/apache/iceberg/AppendBenchmark.java   | 14 +++++-
 .../org/apache/iceberg/FileGenerationUtil.java     | 27 ++++++++---
 .../org/apache/iceberg/TestFileGenerationUtil.java | 53 +++++++++++++++++++---
 3 files changed, 80 insertions(+), 14 deletions(-)

diff --git a/core/src/jmh/java/org/apache/iceberg/AppendBenchmark.java b/core/src/jmh/java/org/apache/iceberg/AppendBenchmark.java
index 57a5c2d3a1..553d1a9430 100644
--- a/core/src/jmh/java/org/apache/iceberg/AppendBenchmark.java
+++ b/core/src/jmh/java/org/apache/iceberg/AppendBenchmark.java
@@ -71,7 +71,19 @@ public class AppendBenchmark {
           required(10, "str_col4", Types.StringType.get()),
           required(11, "str_col5", Types.StringType.get()),
           required(12, "str_col6", Types.StringType.get()),
-          required(13, "str_col7", Types.StringType.get()));
+          required(13, "str_col7", Types.StringType.get()),
+          required(14, "time_col", Types.TimeType.get()),
+          required(
+              15,
+              "struct_col",
+              Types.StructType.of(
+                  required(16, "nested_int", Types.IntegerType.get()),
+                  required(17, "nested_str", Types.StringType.get()))),
+          required(
+              18,
+              "map_col",
+              Types.MapType.ofRequired(19, 20, Types.StringType.get(), Types.LongType.get())),
+          required(21, "list_col", Types.ListType.ofRequired(22, Types.StringType.get())));
   private static final PartitionSpec SPEC = PartitionSpec.unpartitioned();
   private static final HadoopTables TABLES = new HadoopTables();
 
diff --git a/core/src/test/java/org/apache/iceberg/FileGenerationUtil.java b/core/src/test/java/org/apache/iceberg/FileGenerationUtil.java
index 7f8325eb64..a1fc8151e4 100644
--- a/core/src/test/java/org/apache/iceberg/FileGenerationUtil.java
+++ b/core/src/test/java/org/apache/iceberg/FileGenerationUtil.java
@@ -20,6 +20,7 @@ package org.apache.iceberg;
 
 import java.nio.ByteBuffer;
 import java.util.Comparator;
+import java.util.List;
 import java.util.Map;
 import java.util.Random;
 import java.util.UUID;
@@ -31,6 +32,7 @@ import org.apache.iceberg.MetricsModes.Truncate;
 import org.apache.iceberg.io.DeleteSchemaUtil;
 import org.apache.iceberg.io.LocationProvider;
 import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
 import org.apache.iceberg.relocated.com.google.common.collect.Maps;
 import org.apache.iceberg.transforms.Transform;
 import org.apache.iceberg.transforms.Transforms;
@@ -38,6 +40,7 @@ import org.apache.iceberg.types.Comparators;
 import org.apache.iceberg.types.Conversions;
 import org.apache.iceberg.types.Type;
 import org.apache.iceberg.types.Type.PrimitiveType;
+import org.apache.iceberg.types.TypeUtil;
 import org.apache.iceberg.types.Types;
 import org.apache.iceberg.util.Pair;
 import org.apache.iceberg.util.RandomUtil;
@@ -177,19 +180,20 @@ public class FileGenerationUtil {
     Map<Integer, ByteBuffer> upperBounds = Maps.newHashMap();
     Map<Integer, Type> originalTypes = Maps.newHashMap();
 
-    for (Types.NestedField column : schema.columns()) {
-      int fieldId = column.fieldId();
+    for (Types.NestedField field : leafFields(schema)) {
+      int fieldId = field.fieldId();
       columnSizes.put(fieldId, generateColumnSize());
       valueCounts.put(fieldId, generateValueCount());
       nullValueCounts.put(fieldId, (long) random().nextInt(5));
       nanValueCounts.put(fieldId, (long) random().nextInt(5));
-      originalTypes.put(fieldId, column.type());
+      originalTypes.put(fieldId, field.type());
+
       if (knownLowerBounds.containsKey(fieldId) && knownUpperBounds.containsKey(fieldId)) {
         lowerBounds.put(fieldId, knownLowerBounds.get(fieldId));
         upperBounds.put(fieldId, knownUpperBounds.get(fieldId));
-      } else if (column.type().isPrimitiveType()) {
-        PrimitiveType type = column.type().asPrimitiveType();
-        MetricsMode metricsMode = metricsConfig.columnMode(column.name());
+      } else {
+        PrimitiveType type = field.type().asPrimitiveType();
+        MetricsMode metricsMode = metricsConfig.columnMode(fieldId);
         Pair<ByteBuffer, ByteBuffer> bounds = generateBounds(type, metricsMode);
         lowerBounds.put(fieldId, bounds.first());
         upperBounds.put(fieldId, bounds.second());
@@ -207,6 +211,17 @@ public class FileGenerationUtil {
         originalTypes);
   }
 
+  // collects all primitive (leaf) fields, including those nested within structs, lists, and maps
+  static List<Types.NestedField> leafFields(Schema schema) {
+    List<Types.NestedField> leaves = Lists.newArrayList();
+    for (Types.NestedField field : TypeUtil.indexById(schema.asStruct()).values()) {
+      if (field.type().isPrimitiveType()) {
+        leaves.add(field);
+      }
+    }
+    return leaves;
+  }
+
   private static Metrics generatePositionDeleteMetrics(DataFile dataFile) {
     long rowCount = generateRowCount();
     Map<Integer, Long> columnSizes = Maps.newHashMap();
diff --git a/core/src/test/java/org/apache/iceberg/TestFileGenerationUtil.java b/core/src/test/java/org/apache/iceberg/TestFileGenerationUtil.java
index c167bb7039..590d4e1601 100644
--- a/core/src/test/java/org/apache/iceberg/TestFileGenerationUtil.java
+++ b/core/src/test/java/org/apache/iceberg/TestFileGenerationUtil.java
@@ -46,6 +46,21 @@ public class TestFileGenerationUtil {
           required(6, "timestamp_tz_col", Types.TimestampType.withZone()),
           required(7, "str_col", Types.StringType.get()));
 
+  public static final Schema NESTED_SCHEMA =
+      new Schema(
+          required(1, "int_col", Types.IntegerType.get()),
+          required(
+              2,
+              "struct_col",
+              Types.StructType.of(
+                  required(3, "nested_int", Types.IntegerType.get()),
+                  required(4, "nested_str", Types.StringType.get()))),
+          required(
+              5,
+              "map_col",
+              Types.MapType.ofRequired(6, 7, Types.StringType.get(), Types.LongType.get())),
+          required(8, "list_col", Types.ListType.ofRequired(9, Types.IntegerType.get())));
+
   @Test
   public void testBoundsWithDefaultMetricsConfig() {
     MetricsConfig metricsConfig = MetricsConfig.getDefault();
@@ -59,7 +74,7 @@ public class TestFileGenerationUtil {
     assertThat(metrics.lowerBounds()).hasSize(SCHEMA.columns().size());
     assertThat(metrics.upperBounds()).hasSize(SCHEMA.columns().size());
 
-    checkBounds(metrics, metricsConfig);
+    checkBounds(SCHEMA, metrics, metricsConfig);
   }
 
   @Test
@@ -79,7 +94,7 @@ public class TestFileGenerationUtil {
     assertThat(metrics.lowerBounds()).hasSize(SCHEMA.columns().size());
     assertThat(metrics.upperBounds()).hasSize(SCHEMA.columns().size());
 
-    checkBounds(metrics, metricsConfig);
+    checkBounds(SCHEMA, metrics, metricsConfig);
 
     ByteBuffer actualIntLower = metrics.lowerBounds().get(intField.fieldId());
     ByteBuffer actualIntUpper = metrics.upperBounds().get(intField.fieldId());
@@ -89,7 +104,6 @@ public class TestFileGenerationUtil {
 
   @ParameterizedTest
   @ValueSource(strings = {"none", "counts", "truncate(16)", "full"})
-  @SuppressWarnings("deprecation")
   void testBoundsForAllMetricsModes(String metricsMode) {
     MetricsConfig metricsConfig =
         MetricsConfig.from(
@@ -101,12 +115,37 @@ public class TestFileGenerationUtil {
             ImmutableMap.of() /* no lower bounds */,
             ImmutableMap.of() /* no upper bounds */);
 
-    checkBounds(metrics, metricsConfig);
+    checkBounds(SCHEMA, metrics, metricsConfig);
+  }
+
+  @ParameterizedTest
+  @ValueSource(strings = {"none", "counts", "truncate(16)", "full"})
+  public void testBoundsForNestedTypes(String metricsMode) {
+    MetricsConfig metricsConfig =
+        MetricsConfig.from(
+            ImmutableMap.of(TableProperties.DEFAULT_WRITE_METRICS_MODE, metricsMode),
+            NESTED_SCHEMA,
+            null);
+    Metrics metrics =
+        FileGenerationUtil.generateRandomMetrics(
+            NESTED_SCHEMA,
+            metricsConfig,
+            ImmutableMap.of() /* no known lower bounds */,
+            ImmutableMap.of() /* no known upper bounds */);
+
+    assertThat(metrics.lowerBounds())
+        .as("only leaf primitive fields have lower bound")
+        .containsOnlyKeys(1, 3, 4, 6, 7, 9);
+    assertThat(metrics.upperBounds())
+        .as("only leaf primitive fields have upper bound")
+        .containsOnlyKeys(1, 3, 4, 6, 7, 9);
+
+    checkBounds(NESTED_SCHEMA, metrics, metricsConfig);
   }
 
-  private void checkBounds(Metrics metrics, MetricsConfig metricsConfig) {
-    for (NestedField field : SCHEMA.columns()) {
-      MetricsMode mode = metricsConfig.columnMode(field.name());
+  private void checkBounds(Schema schema, Metrics metrics, MetricsConfig metricsConfig) {
+    for (NestedField field : FileGenerationUtil.leafFields(schema)) {
+      MetricsMode mode = metricsConfig.columnMode(field.fieldId());
       ByteBuffer lowerBuffer = metrics.lowerBounds().get(field.fieldId());
       ByteBuffer upperBuffer = metrics.upperBounds().get(field.fieldId());
       if (mode.equals(MetricsModes.None.get()) || mode.equals(MetricsModes.Counts.get())) {