You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by hu...@apache.org on 2026/08/11 23:39:15 UTC
(iceberg) branch main updated: Core: Support metrics with nested field in AppendBenchmark and FileGenerationUtil (#17391)
This is an automated email from the ASF dual-hosted git repository.
huaxingao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new e31bda3c1a Core: Support metrics with nested field in AppendBenchmark and FileGenerationUtil (#17391)
e31bda3c1a is described below
commit e31bda3c1a04c3c0afab0cd3098f9a976300a4a2
Author: Hongyue/Steve Zhang <ho...@gmail.com>
AuthorDate: Tue Aug 11 16:39:07 2026 -0700
Core: Support metrics with nested field in AppendBenchmark and FileGenerationUtil (#17391)
* Core: Add nested field traversal in AppendBenchmark and FileGenerationUtil
* expand testBoundsForNestedTypes to cover all metricsMode
* Core: Resolve metrics mode by field id in FileGenerationUtil
Use MetricsConfig.columnMode(int) instead of manually resolving the
column name via Schema.findColumnName. The int overload performs the
same id-to-name lookup internally and correctly handles column aliases,
removing duplicated logic across the generator and its test.
Co-Authored-By: Claude Opus 4.8 (1M context) <no...@anthropic.com>
* fix schema used in UT
---------
Co-authored-by: Claude Opus 4.8 (1M context) <no...@anthropic.com>
---
.../java/org/apache/iceberg/AppendBenchmark.java | 14 +++++-
.../org/apache/iceberg/FileGenerationUtil.java | 27 ++++++++---
.../org/apache/iceberg/TestFileGenerationUtil.java | 53 +++++++++++++++++++---
3 files changed, 80 insertions(+), 14 deletions(-)
diff --git a/core/src/jmh/java/org/apache/iceberg/AppendBenchmark.java b/core/src/jmh/java/org/apache/iceberg/AppendBenchmark.java
index 57a5c2d3a1..553d1a9430 100644
--- a/core/src/jmh/java/org/apache/iceberg/AppendBenchmark.java
+++ b/core/src/jmh/java/org/apache/iceberg/AppendBenchmark.java
@@ -71,7 +71,19 @@ public class AppendBenchmark {
required(10, "str_col4", Types.StringType.get()),
required(11, "str_col5", Types.StringType.get()),
required(12, "str_col6", Types.StringType.get()),
- required(13, "str_col7", Types.StringType.get()));
+ required(13, "str_col7", Types.StringType.get()),
+ required(14, "time_col", Types.TimeType.get()),
+ required(
+ 15,
+ "struct_col",
+ Types.StructType.of(
+ required(16, "nested_int", Types.IntegerType.get()),
+ required(17, "nested_str", Types.StringType.get()))),
+ required(
+ 18,
+ "map_col",
+ Types.MapType.ofRequired(19, 20, Types.StringType.get(), Types.LongType.get())),
+ required(21, "list_col", Types.ListType.ofRequired(22, Types.StringType.get())));
private static final PartitionSpec SPEC = PartitionSpec.unpartitioned();
private static final HadoopTables TABLES = new HadoopTables();
diff --git a/core/src/test/java/org/apache/iceberg/FileGenerationUtil.java b/core/src/test/java/org/apache/iceberg/FileGenerationUtil.java
index 7f8325eb64..a1fc8151e4 100644
--- a/core/src/test/java/org/apache/iceberg/FileGenerationUtil.java
+++ b/core/src/test/java/org/apache/iceberg/FileGenerationUtil.java
@@ -20,6 +20,7 @@ package org.apache.iceberg;
import java.nio.ByteBuffer;
import java.util.Comparator;
+import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.UUID;
@@ -31,6 +32,7 @@ import org.apache.iceberg.MetricsModes.Truncate;
import org.apache.iceberg.io.DeleteSchemaUtil;
import org.apache.iceberg.io.LocationProvider;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.apache.iceberg.transforms.Transform;
import org.apache.iceberg.transforms.Transforms;
@@ -38,6 +40,7 @@ import org.apache.iceberg.types.Comparators;
import org.apache.iceberg.types.Conversions;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Type.PrimitiveType;
+import org.apache.iceberg.types.TypeUtil;
import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.Pair;
import org.apache.iceberg.util.RandomUtil;
@@ -177,19 +180,20 @@ public class FileGenerationUtil {
Map<Integer, ByteBuffer> upperBounds = Maps.newHashMap();
Map<Integer, Type> originalTypes = Maps.newHashMap();
- for (Types.NestedField column : schema.columns()) {
- int fieldId = column.fieldId();
+ for (Types.NestedField field : leafFields(schema)) {
+ int fieldId = field.fieldId();
columnSizes.put(fieldId, generateColumnSize());
valueCounts.put(fieldId, generateValueCount());
nullValueCounts.put(fieldId, (long) random().nextInt(5));
nanValueCounts.put(fieldId, (long) random().nextInt(5));
- originalTypes.put(fieldId, column.type());
+ originalTypes.put(fieldId, field.type());
+
if (knownLowerBounds.containsKey(fieldId) && knownUpperBounds.containsKey(fieldId)) {
lowerBounds.put(fieldId, knownLowerBounds.get(fieldId));
upperBounds.put(fieldId, knownUpperBounds.get(fieldId));
- } else if (column.type().isPrimitiveType()) {
- PrimitiveType type = column.type().asPrimitiveType();
- MetricsMode metricsMode = metricsConfig.columnMode(column.name());
+ } else {
+ PrimitiveType type = field.type().asPrimitiveType();
+ MetricsMode metricsMode = metricsConfig.columnMode(fieldId);
Pair<ByteBuffer, ByteBuffer> bounds = generateBounds(type, metricsMode);
lowerBounds.put(fieldId, bounds.first());
upperBounds.put(fieldId, bounds.second());
@@ -207,6 +211,17 @@ public class FileGenerationUtil {
originalTypes);
}
+ // collects all primitive (leaf) fields, including those nested within structs, lists, and maps
+ static List<Types.NestedField> leafFields(Schema schema) {
+ List<Types.NestedField> leaves = Lists.newArrayList();
+ for (Types.NestedField field : TypeUtil.indexById(schema.asStruct()).values()) {
+ if (field.type().isPrimitiveType()) {
+ leaves.add(field);
+ }
+ }
+ return leaves;
+ }
+
private static Metrics generatePositionDeleteMetrics(DataFile dataFile) {
long rowCount = generateRowCount();
Map<Integer, Long> columnSizes = Maps.newHashMap();
diff --git a/core/src/test/java/org/apache/iceberg/TestFileGenerationUtil.java b/core/src/test/java/org/apache/iceberg/TestFileGenerationUtil.java
index c167bb7039..590d4e1601 100644
--- a/core/src/test/java/org/apache/iceberg/TestFileGenerationUtil.java
+++ b/core/src/test/java/org/apache/iceberg/TestFileGenerationUtil.java
@@ -46,6 +46,21 @@ public class TestFileGenerationUtil {
required(6, "timestamp_tz_col", Types.TimestampType.withZone()),
required(7, "str_col", Types.StringType.get()));
+ public static final Schema NESTED_SCHEMA =
+ new Schema(
+ required(1, "int_col", Types.IntegerType.get()),
+ required(
+ 2,
+ "struct_col",
+ Types.StructType.of(
+ required(3, "nested_int", Types.IntegerType.get()),
+ required(4, "nested_str", Types.StringType.get()))),
+ required(
+ 5,
+ "map_col",
+ Types.MapType.ofRequired(6, 7, Types.StringType.get(), Types.LongType.get())),
+ required(8, "list_col", Types.ListType.ofRequired(9, Types.IntegerType.get())));
+
@Test
public void testBoundsWithDefaultMetricsConfig() {
MetricsConfig metricsConfig = MetricsConfig.getDefault();
@@ -59,7 +74,7 @@ public class TestFileGenerationUtil {
assertThat(metrics.lowerBounds()).hasSize(SCHEMA.columns().size());
assertThat(metrics.upperBounds()).hasSize(SCHEMA.columns().size());
- checkBounds(metrics, metricsConfig);
+ checkBounds(SCHEMA, metrics, metricsConfig);
}
@Test
@@ -79,7 +94,7 @@ public class TestFileGenerationUtil {
assertThat(metrics.lowerBounds()).hasSize(SCHEMA.columns().size());
assertThat(metrics.upperBounds()).hasSize(SCHEMA.columns().size());
- checkBounds(metrics, metricsConfig);
+ checkBounds(SCHEMA, metrics, metricsConfig);
ByteBuffer actualIntLower = metrics.lowerBounds().get(intField.fieldId());
ByteBuffer actualIntUpper = metrics.upperBounds().get(intField.fieldId());
@@ -89,7 +104,6 @@ public class TestFileGenerationUtil {
@ParameterizedTest
@ValueSource(strings = {"none", "counts", "truncate(16)", "full"})
- @SuppressWarnings("deprecation")
void testBoundsForAllMetricsModes(String metricsMode) {
MetricsConfig metricsConfig =
MetricsConfig.from(
@@ -101,12 +115,37 @@ public class TestFileGenerationUtil {
ImmutableMap.of() /* no lower bounds */,
ImmutableMap.of() /* no upper bounds */);
- checkBounds(metrics, metricsConfig);
+ checkBounds(SCHEMA, metrics, metricsConfig);
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"none", "counts", "truncate(16)", "full"})
+ public void testBoundsForNestedTypes(String metricsMode) {
+ MetricsConfig metricsConfig =
+ MetricsConfig.from(
+ ImmutableMap.of(TableProperties.DEFAULT_WRITE_METRICS_MODE, metricsMode),
+ NESTED_SCHEMA,
+ null);
+ Metrics metrics =
+ FileGenerationUtil.generateRandomMetrics(
+ NESTED_SCHEMA,
+ metricsConfig,
+ ImmutableMap.of() /* no known lower bounds */,
+ ImmutableMap.of() /* no known upper bounds */);
+
+ assertThat(metrics.lowerBounds())
+ .as("only leaf primitive fields have lower bound")
+ .containsOnlyKeys(1, 3, 4, 6, 7, 9);
+ assertThat(metrics.upperBounds())
+ .as("only leaf primitive fields have upper bound")
+ .containsOnlyKeys(1, 3, 4, 6, 7, 9);
+
+ checkBounds(NESTED_SCHEMA, metrics, metricsConfig);
}
- private void checkBounds(Metrics metrics, MetricsConfig metricsConfig) {
- for (NestedField field : SCHEMA.columns()) {
- MetricsMode mode = metricsConfig.columnMode(field.name());
+ private void checkBounds(Schema schema, Metrics metrics, MetricsConfig metricsConfig) {
+ for (NestedField field : FileGenerationUtil.leafFields(schema)) {
+ MetricsMode mode = metricsConfig.columnMode(field.fieldId());
ByteBuffer lowerBuffer = metrics.lowerBounds().get(field.fieldId());
ByteBuffer upperBuffer = metrics.upperBounds().get(field.fieldId());
if (mode.equals(MetricsModes.None.get()) || mode.equals(MetricsModes.Counts.get())) {