You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2026/07/08 15:56:04 UTC
(iceberg) branch main updated: Parquet: Refactor ParquetMetrics to produce field stats (#17116)
This is an automated email from the ASF dual-hosted git repository.
rdblue pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 9bb14af0e3 Parquet: Refactor ParquetMetrics to produce field stats (#17116)
9bb14af0e3 is described below
commit 9bb14af0e3018170ce3ca2d88a3f8d980f44dbc8
Author: Ryan Blue <bl...@apache.org>
AuthorDate: Wed Jul 8 08:55:56 2026 -0700
Parquet: Refactor ParquetMetrics to produce field stats (#17116)
---
.../main/java/org/apache/iceberg/MetricsUtil.java | 60 ---
.../test/java/org/apache/iceberg/TestMetrics.java | 422 ++++++++-------------
.../org/apache/iceberg/orc/TestOrcMetrics.java | 8 +-
.../org/apache/iceberg/parquet/ParquetMetrics.java | 137 ++++---
4 files changed, 238 insertions(+), 389 deletions(-)
diff --git a/core/src/main/java/org/apache/iceberg/MetricsUtil.java b/core/src/main/java/org/apache/iceberg/MetricsUtil.java
index 052a13bb88..e6767db341 100644
--- a/core/src/main/java/org/apache/iceberg/MetricsUtil.java
+++ b/core/src/main/java/org/apache/iceberg/MetricsUtil.java
@@ -27,7 +27,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.BiFunction;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
@@ -554,63 +553,4 @@ public class MetricsUtil {
return result.isEmpty() ? null : Collections.unmodifiableMap(result);
}
-
- static ContentStats fromMetrics(Schema schema, Metrics metrics) {
- if (null == metrics) {
- return null;
- }
-
- BaseContentStats.Builder builder = BaseContentStats.builder().withTableSchema(schema);
- Map<Integer, BaseFieldStats.Builder<Object>> map = Maps.newHashMap();
- mergeCountMetric(map, metrics.valueCounts(), BaseFieldStats.Builder::valueCount);
- mergeCountMetric(map, metrics.nullValueCounts(), BaseFieldStats.Builder::nullValueCount);
- mergeCountMetric(map, metrics.nanValueCounts(), BaseFieldStats.Builder::nanValueCount);
- mergeBoundMetric(
- map, metrics.lowerBounds(), metrics.originalTypes(), BaseFieldStats.Builder::lowerBound);
- mergeBoundMetric(
- map, metrics.upperBounds(), metrics.originalTypes(), BaseFieldStats.Builder::upperBound);
-
- map.values().forEach(fieldStats -> builder.withFieldStats(fieldStats.build()));
-
- return builder.build();
- }
-
- private static void mergeCountMetric(
- Map<Integer, BaseFieldStats.Builder<Object>> fieldStatsById,
- Map<Integer, Long> counts,
- BiFunction<BaseFieldStats.Builder<Object>, Long, BaseFieldStats.Builder<Object>> setter) {
- if (counts == null) {
- return;
- }
-
- counts.forEach(
- (id, value) ->
- fieldStatsById.merge(
- id,
- setter.apply(BaseFieldStats.builder().fieldId(id), value),
- (oldVal, newVal) -> setter.apply(oldVal, value)));
- }
-
- private static void mergeBoundMetric(
- Map<Integer, BaseFieldStats.Builder<Object>> fieldStatsById,
- Map<Integer, ByteBuffer> bounds,
- Map<Integer, Type> originalTypes,
- BiFunction<BaseFieldStats.Builder<Object>, Object, BaseFieldStats.Builder<Object>> setter) {
- if (bounds == null || originalTypes == null) {
- return;
- }
-
- bounds.entrySet().stream()
- .filter(entry -> originalTypes.get(entry.getKey()) != null)
- .forEach(
- entry -> {
- Integer id = entry.getKey();
- Type type = originalTypes.get(id);
- Object boundValue = Conversions.fromByteBuffer(type, entry.getValue());
- fieldStatsById.merge(
- id,
- setter.apply(BaseFieldStats.builder().fieldId(id).type(type), boundValue),
- (oldVal, newVal) -> setter.apply(oldVal.type(type), boundValue));
- });
- }
}
diff --git a/core/src/test/java/org/apache/iceberg/TestMetrics.java b/core/src/test/java/org/apache/iceberg/TestMetrics.java
index 5dece00fc5..feffc18477 100644
--- a/core/src/test/java/org/apache/iceberg/TestMetrics.java
+++ b/core/src/test/java/org/apache/iceberg/TestMetrics.java
@@ -166,22 +166,20 @@ public abstract class TestMetrics {
record.setField("timestampColBelowEpoch", DateTimeUtil.timestampFromMicros(0L));
Metrics metrics = getMetrics(SIMPLE_SCHEMA, record, record);
- MetricsWithStats metricsWithStats =
- new MetricsWithStats(metrics, MetricsUtil.fromMetrics(SIMPLE_SCHEMA, metrics));
assertThat(metrics.recordCount()).isEqualTo(2L);
- assertCounts(1, 2L, 0L, metricsWithStats);
- assertCounts(2, 2L, 0L, metricsWithStats);
- assertCounts(3, 2L, 2L, metricsWithStats);
- assertCounts(4, 2L, 0L, 2L, metricsWithStats);
- assertCounts(5, 2L, 0L, 0L, metricsWithStats);
- assertCounts(6, 2L, 0L, metricsWithStats);
- assertCounts(7, 2L, 0L, metricsWithStats);
- assertCounts(8, 2L, 0L, metricsWithStats);
- assertCounts(9, 2L, 0L, metricsWithStats);
- assertCounts(10, 2L, 0L, metricsWithStats);
- assertCounts(11, 2L, 0L, metricsWithStats);
- assertCounts(12, 2L, 0L, metricsWithStats);
- assertCounts(13, 2L, 0L, metricsWithStats);
+ assertCounts(1, 2L, 0L, metrics);
+ assertCounts(2, 2L, 0L, metrics);
+ assertCounts(3, 2L, 2L, metrics);
+ assertCounts(4, 2L, 0L, 2L, metrics);
+ assertCounts(5, 2L, 0L, 0L, metrics);
+ assertCounts(6, 2L, 0L, metrics);
+ assertCounts(7, 2L, 0L, metrics);
+ assertCounts(8, 2L, 0L, metrics);
+ assertCounts(9, 2L, 0L, metrics);
+ assertCounts(10, 2L, 0L, metrics);
+ assertCounts(11, 2L, 0L, metrics);
+ assertCounts(12, 2L, 0L, metrics);
+ assertCounts(13, 2L, 0L, metrics);
}
@TestTemplate
@@ -216,54 +214,46 @@ public abstract class TestMetrics {
secondRecord.setField("timestampColBelowEpoch", DateTimeUtil.timestampFromMicros(-7_000L));
Metrics metrics = getMetrics(SIMPLE_SCHEMA, firstRecord, secondRecord);
- MetricsWithStats metricsWithStats =
- new MetricsWithStats(metrics, MetricsUtil.fromMetrics(SIMPLE_SCHEMA, metrics));
assertThat(metrics.recordCount()).isEqualTo(2L);
- assertCounts(1, 2L, 0L, metricsWithStats);
- assertBounds(1, BooleanType.get(), false, true, metricsWithStats);
- assertCounts(2, 2L, 0L, metricsWithStats);
- assertBounds(2, IntegerType.get(), Integer.MIN_VALUE, 3, metricsWithStats);
- assertCounts(3, 2L, 1L, metricsWithStats);
- assertBounds(3, LongType.get(), 5L, 5L, metricsWithStats);
- assertCounts(4, 2L, 0L, 0L, metricsWithStats);
- assertBounds(4, FloatType.get(), 1.0F, 2.0F, metricsWithStats);
- assertCounts(5, 2L, 1L, 0L, metricsWithStats);
- assertBounds(5, DoubleType.get(), 2.0D, 2.0D, metricsWithStats);
- assertCounts(6, 2L, 1L, metricsWithStats);
+ assertCounts(1, 2L, 0L, metrics);
+ assertBounds(1, BooleanType.get(), false, true, metrics);
+ assertCounts(2, 2L, 0L, metrics);
+ assertBounds(2, IntegerType.get(), Integer.MIN_VALUE, 3, metrics);
+ assertCounts(3, 2L, 1L, metrics);
+ assertBounds(3, LongType.get(), 5L, 5L, metrics);
+ assertCounts(4, 2L, 0L, 0L, metrics);
+ assertBounds(4, FloatType.get(), 1.0F, 2.0F, metrics);
+ assertCounts(5, 2L, 1L, 0L, metrics);
+ assertBounds(5, DoubleType.get(), 2.0D, 2.0D, metrics);
+ assertCounts(6, 2L, 1L, metrics);
+ assertBounds(6, DecimalType.of(10, 2), new BigDecimal("3.50"), new BigDecimal("3.50"), metrics);
+ assertCounts(7, 2L, 0L, metrics);
+ assertBounds(7, StringType.get(), CharBuffer.wrap("AAA"), CharBuffer.wrap("ZZZ"), metrics);
+ assertCounts(8, 2L, 1L, metrics);
+ assertBounds(8, DateType.get(), 1500, 1500, metrics);
+ assertCounts(9, 2L, 0L, metrics);
+ assertBounds(9, TimeType.get(), 2000L, 3000L, metrics);
+ assertCounts(10, 2L, 0L, metrics);
+ assertBounds(10, TimestampType.withoutZone(), 0L, 900L, metrics);
+ assertCounts(11, 2L, 0L, metrics);
assertBounds(
- 6, DecimalType.of(10, 2), new BigDecimal("3.50"), new BigDecimal("3.50"), metricsWithStats);
- assertCounts(7, 2L, 0L, metricsWithStats);
- assertBounds(
- 7, StringType.get(), CharBuffer.wrap("AAA"), CharBuffer.wrap("ZZZ"), metricsWithStats);
- assertCounts(8, 2L, 1L, metricsWithStats);
- assertBounds(8, DateType.get(), 1500, 1500, metricsWithStats);
- assertCounts(9, 2L, 0L, metricsWithStats);
- assertBounds(9, TimeType.get(), 2000L, 3000L, metricsWithStats);
- assertCounts(10, 2L, 0L, metricsWithStats);
- assertBounds(10, TimestampType.withoutZone(), 0L, 900L, metricsWithStats);
- assertCounts(11, 2L, 0L, metricsWithStats);
- assertBounds(
- 11,
- FixedType.ofLength(4),
- ByteBuffer.wrap(fixed),
- ByteBuffer.wrap(fixed),
- metricsWithStats);
- assertCounts(12, 2L, 0L, metricsWithStats);
+ 11, FixedType.ofLength(4), ByteBuffer.wrap(fixed), ByteBuffer.wrap(fixed), metrics);
+ assertCounts(12, 2L, 0L, metrics);
assertBounds(
12,
BinaryType.get(),
ByteBuffer.wrap("S".getBytes()),
ByteBuffer.wrap("W".getBytes()),
- metricsWithStats);
+ metrics);
if (fileFormat() == FileFormat.ORC) {
// TODO: The special condition for ORC can be removed when ORC-342 is fixed
// ORC-342: ORC writer creates inaccurate timestamp data and stats 1 sec below epoch
// Values in the range `[1969-12-31 23:59:59.000,1969-12-31 23:59:59.999]` will have 1 sec
// added to them
// So the upper bound value of -7_000 micros becomes 993_000 micros
- assertBounds(13, TimestampType.withoutZone(), -1_900_300L, 993_000L, metricsWithStats);
+ assertBounds(13, TimestampType.withoutZone(), -1_900_300L, 993_000L, metrics);
} else {
- assertBounds(13, TimestampType.withoutZone(), -1_900_300L, -7_000L, metricsWithStats);
+ assertBounds(13, TimestampType.withoutZone(), -1_900_300L, -7_000L, metrics);
}
}
@@ -281,41 +271,34 @@ public abstract class TestMetrics {
record.setField("decimalAsFixed", new BigDecimal("5.80"));
Metrics metrics = getMetrics(schema, record);
- MetricsWithStats metricsWithStats =
- new MetricsWithStats(metrics, MetricsUtil.fromMetrics(schema, metrics));
assertThat(metrics.recordCount()).isEqualTo(1);
- assertCounts(1, 1L, 0L, metricsWithStats);
- assertBounds(
- 1, DecimalType.of(4, 2), new BigDecimal("2.55"), new BigDecimal("2.55"), metricsWithStats);
- assertCounts(2, 1L, 0L, metricsWithStats);
- assertBounds(
- 2, DecimalType.of(14, 2), new BigDecimal("4.75"), new BigDecimal("4.75"), metricsWithStats);
- assertCounts(3, 1L, 0L, metricsWithStats);
- assertBounds(
- 3, DecimalType.of(22, 2), new BigDecimal("5.80"), new BigDecimal("5.80"), metricsWithStats);
+ assertCounts(1, 1L, 0L, metrics);
+ assertBounds(1, DecimalType.of(4, 2), new BigDecimal("2.55"), new BigDecimal("2.55"), metrics);
+ assertCounts(2, 1L, 0L, metrics);
+ assertBounds(2, DecimalType.of(14, 2), new BigDecimal("4.75"), new BigDecimal("4.75"), metrics);
+ assertCounts(3, 1L, 0L, metrics);
+ assertBounds(3, DecimalType.of(22, 2), new BigDecimal("5.80"), new BigDecimal("5.80"), metrics);
}
@TestTemplate
public void testMetricsForNestedStructFields() throws IOException {
Metrics metrics = getMetrics(NESTED_SCHEMA, buildNestedTestRecord());
- MetricsWithStats metricsWithStats =
- new MetricsWithStats(metrics, MetricsUtil.fromMetrics(NESTED_SCHEMA, metrics));
assertThat(metrics.recordCount()).isEqualTo(1L);
- assertCounts(1, 1L, 0L, metricsWithStats);
- assertBounds(1, IntegerType.get(), Integer.MAX_VALUE, Integer.MAX_VALUE, metricsWithStats);
- assertCounts(3, 1L, 0L, metricsWithStats);
- assertBounds(3, LongType.get(), 100L, 100L, metricsWithStats);
- assertCounts(5, 1L, 0L, metricsWithStats);
- assertBounds(5, LongType.get(), 20L, 20L, metricsWithStats);
- assertCounts(6, 1L, 0L, metricsWithStats);
+ assertCounts(1, 1L, 0L, metrics);
+ assertBounds(1, IntegerType.get(), Integer.MAX_VALUE, Integer.MAX_VALUE, metrics);
+ assertCounts(3, 1L, 0L, metrics);
+ assertBounds(3, LongType.get(), 100L, 100L, metrics);
+ assertCounts(5, 1L, 0L, metrics);
+ assertBounds(5, LongType.get(), 20L, 20L, metrics);
+ assertCounts(6, 1L, 0L, metrics);
assertBounds(
6,
BinaryType.get(),
ByteBuffer.wrap("A".getBytes()),
ByteBuffer.wrap("A".getBytes()),
- metricsWithStats);
- assertCounts(7, 1L, 0L, 1L, metricsWithStats);
- assertBounds(7, DoubleType.get(), null, null, metricsWithStats);
+ metrics);
+ assertCounts(7, 1L, 0L, 1L, metrics);
+ assertBounds(7, DoubleType.get(), null, null, metrics);
}
@TestTemplate
@@ -329,12 +312,10 @@ public abstract class TestMetrics {
MetricsConfig config = MetricsConfig.from(properties, NESTED_SCHEMA, null);
Metrics metrics = getMetrics(NESTED_SCHEMA, config, buildNestedTestRecord());
- MetricsWithStats metricsWithStats =
- new MetricsWithStats(metrics, MetricsUtil.fromMetrics(NESTED_SCHEMA, metrics));
assertThat(metrics.recordCount()).isEqualTo(1L);
assertThat(metrics.lowerBounds()).hasSize(1);
assertThat(metrics.upperBounds()).hasSize(1);
- assertBounds(3, LongType.get(), 100L, 100L, metricsWithStats);
+ assertBounds(3, LongType.get(), 100L, 100L, metrics);
}
private Record buildNestedTestRecord() {
@@ -373,18 +354,16 @@ public abstract class TestMetrics {
record.set(1, map);
Metrics metrics = getMetrics(schema, record);
- MetricsWithStats metricsWithStats =
- new MetricsWithStats(metrics, MetricsUtil.fromMetrics(schema, metrics));
assertThat(metrics.recordCount()).isEqualTo(1L);
- assertCounts(1, null, null, metricsWithStats);
- assertCounts(2, null, null, metricsWithStats);
- assertCounts(4, null, null, metricsWithStats);
- assertCounts(6, null, null, metricsWithStats);
- assertBounds(1, IntegerType.get(), null, null, metricsWithStats);
- assertBounds(2, StringType.get(), null, null, metricsWithStats);
- assertBounds(4, IntegerType.get(), null, null, metricsWithStats);
- assertBounds(6, StringType.get(), null, null, metricsWithStats);
- assertBounds(7, structType, null, null, metricsWithStats);
+ assertCounts(1, null, null, metrics);
+ assertCounts(2, null, null, metrics);
+ assertCounts(4, null, null, metrics);
+ assertCounts(6, null, null, metrics);
+ assertBounds(1, IntegerType.get(), null, null, metrics);
+ assertBounds(2, StringType.get(), null, null, metrics);
+ assertBounds(4, IntegerType.get(), null, null, metrics);
+ assertBounds(6, StringType.get(), null, null, metrics);
+ assertBounds(7, structType, null, null, metrics);
}
@TestTemplate
@@ -396,24 +375,20 @@ public abstract class TestMetrics {
secondRecord.setField("intCol", null);
Metrics metrics = getMetrics(schema, firstRecord, secondRecord);
- MetricsWithStats metricsWithStats =
- new MetricsWithStats(metrics, MetricsUtil.fromMetrics(schema, metrics));
assertThat(metrics.recordCount()).isEqualTo(2L);
- assertCounts(1, 2L, 2L, metricsWithStats);
- assertBounds(1, IntegerType.get(), null, null, metricsWithStats);
+ assertCounts(1, 2L, 2L, metrics);
+ assertBounds(1, IntegerType.get(), null, null, metrics);
}
@TestTemplate
public void testMetricsForNaNColumns() throws IOException {
Metrics metrics = getMetrics(FLOAT_DOUBLE_ONLY_SCHEMA, NAN_ONLY_RECORD, NAN_ONLY_RECORD);
- MetricsWithStats metricsWithStats =
- new MetricsWithStats(metrics, MetricsUtil.fromMetrics(FLOAT_DOUBLE_ONLY_SCHEMA, metrics));
assertThat(metrics.recordCount()).isEqualTo(2L);
- assertCounts(1, 2L, 0L, 2L, metricsWithStats);
- assertCounts(2, 2L, 0L, 2L, metricsWithStats);
+ assertCounts(1, 2L, 0L, 2L, metrics);
+ assertCounts(2, 2L, 0L, 2L, metrics);
- assertBounds(1, FloatType.get(), null, null, metricsWithStats);
- assertBounds(2, DoubleType.get(), null, null, metricsWithStats);
+ assertBounds(1, FloatType.get(), null, null, metrics);
+ assertBounds(2, DoubleType.get(), null, null, metrics);
}
@TestTemplate
@@ -424,14 +399,12 @@ public abstract class TestMetrics {
NAN_ONLY_RECORD,
FLOAT_DOUBLE_RECORD_1,
FLOAT_DOUBLE_RECORD_2);
- MetricsWithStats metricsWithStats =
- new MetricsWithStats(metrics, MetricsUtil.fromMetrics(FLOAT_DOUBLE_ONLY_SCHEMA, metrics));
assertThat(metrics.recordCount()).isEqualTo(3L);
- assertCounts(1, 3L, 0L, 1L, metricsWithStats);
- assertCounts(2, 3L, 0L, 1L, metricsWithStats);
+ assertCounts(1, 3L, 0L, 1L, metrics);
+ assertCounts(2, 3L, 0L, 1L, metrics);
- assertBounds(1, FloatType.get(), 1.2F, 5.6F, metricsWithStats);
- assertBounds(2, DoubleType.get(), 3.4D, 7.8D, metricsWithStats);
+ assertBounds(1, FloatType.get(), 1.2F, 5.6F, metrics);
+ assertBounds(2, DoubleType.get(), 3.4D, 7.8D, metrics);
}
@TestTemplate
@@ -442,14 +415,12 @@ public abstract class TestMetrics {
FLOAT_DOUBLE_RECORD_1,
NAN_ONLY_RECORD,
FLOAT_DOUBLE_RECORD_2);
- MetricsWithStats metricsWithStats =
- new MetricsWithStats(metrics, MetricsUtil.fromMetrics(FLOAT_DOUBLE_ONLY_SCHEMA, metrics));
assertThat(metrics.recordCount()).isEqualTo(3L);
- assertCounts(1, 3L, 0L, 1L, metricsWithStats);
- assertCounts(2, 3L, 0L, 1L, metricsWithStats);
+ assertCounts(1, 3L, 0L, 1L, metrics);
+ assertCounts(2, 3L, 0L, 1L, metrics);
- assertBounds(1, FloatType.get(), 1.2F, 5.6F, metricsWithStats);
- assertBounds(2, DoubleType.get(), 3.4D, 7.8D, metricsWithStats);
+ assertBounds(1, FloatType.get(), 1.2F, 5.6F, metrics);
+ assertBounds(2, DoubleType.get(), 3.4D, 7.8D, metrics);
}
@TestTemplate
@@ -460,14 +431,12 @@ public abstract class TestMetrics {
FLOAT_DOUBLE_RECORD_1,
FLOAT_DOUBLE_RECORD_2,
NAN_ONLY_RECORD);
- MetricsWithStats metricsWithStats =
- new MetricsWithStats(metrics, MetricsUtil.fromMetrics(FLOAT_DOUBLE_ONLY_SCHEMA, metrics));
assertThat(metrics.recordCount()).isEqualTo(3L);
- assertCounts(1, 3L, 0L, 1L, metricsWithStats);
- assertCounts(2, 3L, 0L, 1L, metricsWithStats);
+ assertCounts(1, 3L, 0L, 1L, metrics);
+ assertCounts(2, 3L, 0L, 1L, metrics);
- assertBounds(1, FloatType.get(), 1.2F, 5.6F, metricsWithStats);
- assertBounds(2, DoubleType.get(), 3.4D, 7.8D, metricsWithStats);
+ assertBounds(1, FloatType.get(), 1.2F, 5.6F, metrics);
+ assertBounds(2, DoubleType.get(), 3.4D, 7.8D, metrics);
}
@TestTemplate
@@ -504,8 +473,6 @@ public abstract class TestMetrics {
Metrics metrics =
getMetricsForRecordsWithSmallRowGroups(
SIMPLE_SCHEMA, outputFile, records.toArray(new Record[0]));
- MetricsWithStats metricsWithStats =
- new MetricsWithStats(metrics, MetricsUtil.fromMetrics(SIMPLE_SCHEMA, metrics));
InputFile recordsFile = outputFile.toInputFile();
assertThat(recordsFile).isNotNull();
@@ -513,22 +480,18 @@ public abstract class TestMetrics {
assertThat(splitCount(recordsFile)).isEqualTo(3);
assertThat(metrics.recordCount()).isEqualTo(201L);
- assertCounts(1, 201L, 0L, metricsWithStats);
- assertBounds(1, Types.BooleanType.get(), false, true, metricsWithStats);
- assertBounds(2, Types.IntegerType.get(), 1, 201, metricsWithStats);
- assertCounts(3, 201L, 1L, metricsWithStats);
- assertBounds(3, Types.LongType.get(), 2L, 201L, metricsWithStats);
- assertCounts(4, 201L, 0L, 0L, metricsWithStats);
- assertBounds(4, Types.FloatType.get(), 1.0F, 201.0F, metricsWithStats);
- assertCounts(5, 201L, 1L, 0L, metricsWithStats);
- assertBounds(5, Types.DoubleType.get(), 2.0D, 201.0D, metricsWithStats);
- assertCounts(6, 201L, 1L, metricsWithStats);
+ assertCounts(1, 201L, 0L, metrics);
+ assertBounds(1, Types.BooleanType.get(), false, true, metrics);
+ assertBounds(2, Types.IntegerType.get(), 1, 201, metrics);
+ assertCounts(3, 201L, 1L, metrics);
+ assertBounds(3, Types.LongType.get(), 2L, 201L, metrics);
+ assertCounts(4, 201L, 0L, 0L, metrics);
+ assertBounds(4, Types.FloatType.get(), 1.0F, 201.0F, metrics);
+ assertCounts(5, 201L, 1L, 0L, metrics);
+ assertBounds(5, Types.DoubleType.get(), 2.0D, 201.0D, metrics);
+ assertCounts(6, 201L, 1L, metrics);
assertBounds(
- 6,
- Types.DecimalType.of(10, 2),
- new BigDecimal("2.00"),
- new BigDecimal("201.00"),
- metricsWithStats);
+ 6, Types.DecimalType.of(10, 2), new BigDecimal("2.00"), new BigDecimal("201.00"), metrics);
}
@TestTemplate
@@ -559,8 +522,6 @@ public abstract class TestMetrics {
Metrics metrics =
getMetricsForRecordsWithSmallRowGroups(
NESTED_SCHEMA, outputFile, records.toArray(new Record[0]));
- MetricsWithStats metricsWithStats =
- new MetricsWithStats(metrics, MetricsUtil.fromMetrics(NESTED_SCHEMA, metrics));
InputFile recordsFile = outputFile.toInputFile();
assertThat(recordsFile).isNotNull();
@@ -568,21 +529,21 @@ public abstract class TestMetrics {
assertThat(splitCount(recordsFile)).isEqualTo(3);
assertThat(metrics.recordCount()).isEqualTo(201L);
- assertCounts(1, 201L, 0L, metricsWithStats);
- assertBounds(1, IntegerType.get(), 1, 201, metricsWithStats);
- assertCounts(3, 201L, 0L, metricsWithStats);
- assertBounds(3, LongType.get(), 1L, 201L, metricsWithStats);
- assertCounts(5, 201L, 0L, metricsWithStats);
- assertBounds(5, LongType.get(), 1L, 201L, metricsWithStats);
- assertCounts(6, 201L, 0L, metricsWithStats);
+ assertCounts(1, 201L, 0L, metrics);
+ assertBounds(1, IntegerType.get(), 1, 201, metrics);
+ assertCounts(3, 201L, 0L, metrics);
+ assertBounds(3, LongType.get(), 1L, 201L, metrics);
+ assertCounts(5, 201L, 0L, metrics);
+ assertBounds(5, LongType.get(), 1L, 201L, metrics);
+ assertCounts(6, 201L, 0L, metrics);
assertBounds(
6,
BinaryType.get(),
ByteBuffer.wrap("A".getBytes()),
ByteBuffer.wrap("A".getBytes()),
- metricsWithStats);
- assertCounts(7, 201L, 0L, 201L, metricsWithStats);
- assertBounds(7, DoubleType.get(), null, null, metricsWithStats);
+ metrics);
+ assertCounts(7, 201L, 0L, 201L, metrics);
+ assertBounds(7, DoubleType.get(), null, null, metrics);
}
@TestTemplate
@@ -593,20 +554,18 @@ public abstract class TestMetrics {
MetricsConfig.from(
ImmutableMap.of("write.metadata.metrics.default", "none"), NESTED_SCHEMA, null),
buildNestedTestRecord());
- MetricsWithStats metricsWithStats =
- new MetricsWithStats(metrics, MetricsUtil.fromMetrics(NESTED_SCHEMA, metrics));
assertThat(metrics.recordCount()).isEqualTo(1L);
assertThat(metrics.columnSizes()).isEmpty();
- assertCounts(1, null, null, metricsWithStats);
- assertBounds(1, Types.IntegerType.get(), null, null, metricsWithStats);
- assertCounts(3, null, null, metricsWithStats);
- assertBounds(3, Types.LongType.get(), null, null, metricsWithStats);
- assertCounts(5, null, null, metricsWithStats);
- assertBounds(5, Types.LongType.get(), null, null, metricsWithStats);
- assertCounts(6, null, null, metricsWithStats);
- assertBounds(6, Types.BinaryType.get(), null, null, metricsWithStats);
- assertCounts(7, null, null, metricsWithStats);
- assertBounds(7, Types.DoubleType.get(), null, null, metricsWithStats);
+ assertCounts(1, null, null, metrics);
+ assertBounds(1, Types.IntegerType.get(), null, null, metrics);
+ assertCounts(3, null, null, metrics);
+ assertBounds(3, Types.LongType.get(), null, null, metrics);
+ assertCounts(5, null, null, metrics);
+ assertBounds(5, Types.LongType.get(), null, null, metrics);
+ assertCounts(6, null, null, metrics);
+ assertBounds(6, Types.BinaryType.get(), null, null, metrics);
+ assertCounts(7, null, null, metrics);
+ assertBounds(7, Types.DoubleType.get(), null, null, metrics);
}
@TestTemplate
@@ -617,21 +576,19 @@ public abstract class TestMetrics {
MetricsConfig.from(
ImmutableMap.of("write.metadata.metrics.default", "counts"), NESTED_SCHEMA, null),
buildNestedTestRecord());
- MetricsWithStats metricsWithStats =
- new MetricsWithStats(metrics, MetricsUtil.fromMetrics(NESTED_SCHEMA, metrics));
assertThat(metrics.recordCount()).isEqualTo(1L);
assertThat(metrics.columnSizes()).doesNotContainValue(null);
assertThat(metrics.columnSizes()).isNotEmpty();
- assertCounts(1, 1L, 0L, metricsWithStats);
- assertBounds(1, Types.IntegerType.get(), null, null, metricsWithStats);
- assertCounts(3, 1L, 0L, metricsWithStats);
- assertBounds(3, Types.LongType.get(), null, null, metricsWithStats);
- assertCounts(5, 1L, 0L, metricsWithStats);
- assertBounds(5, Types.LongType.get(), null, null, metricsWithStats);
- assertCounts(6, 1L, 0L, metricsWithStats);
- assertBounds(6, Types.BinaryType.get(), null, null, metricsWithStats);
- assertCounts(7, 1L, 0L, 1L, metricsWithStats);
- assertBounds(7, Types.DoubleType.get(), null, null, metricsWithStats);
+ assertCounts(1, 1L, 0L, metrics);
+ assertBounds(1, Types.IntegerType.get(), null, null, metrics);
+ assertCounts(3, 1L, 0L, metrics);
+ assertBounds(3, Types.LongType.get(), null, null, metrics);
+ assertCounts(5, 1L, 0L, metrics);
+ assertBounds(5, Types.LongType.get(), null, null, metrics);
+ assertCounts(6, 1L, 0L, metrics);
+ assertBounds(6, Types.BinaryType.get(), null, null, metrics);
+ assertCounts(7, 1L, 0L, 1L, metrics);
+ assertBounds(7, Types.DoubleType.get(), null, null, metrics);
}
@TestTemplate
@@ -642,27 +599,24 @@ public abstract class TestMetrics {
MetricsConfig.from(
ImmutableMap.of("write.metadata.metrics.default", "full"), NESTED_SCHEMA, null),
buildNestedTestRecord());
- MetricsWithStats metricsWithStats =
- new MetricsWithStats(metrics, MetricsUtil.fromMetrics(NESTED_SCHEMA, metrics));
assertThat(metrics.recordCount()).isEqualTo(1L);
assertThat(metrics.columnSizes()).doesNotContainValue(null);
assertThat(metrics.columnSizes()).isNotEmpty();
- assertCounts(1, 1L, 0L, metricsWithStats);
- assertBounds(
- 1, Types.IntegerType.get(), Integer.MAX_VALUE, Integer.MAX_VALUE, metricsWithStats);
- assertCounts(3, 1L, 0L, metricsWithStats);
- assertBounds(3, Types.LongType.get(), 100L, 100L, metricsWithStats);
- assertCounts(5, 1L, 0L, metricsWithStats);
- assertBounds(5, Types.LongType.get(), 20L, 20L, metricsWithStats);
- assertCounts(6, 1L, 0L, metricsWithStats);
+ assertCounts(1, 1L, 0L, metrics);
+ assertBounds(1, Types.IntegerType.get(), Integer.MAX_VALUE, Integer.MAX_VALUE, metrics);
+ assertCounts(3, 1L, 0L, metrics);
+ assertBounds(3, Types.LongType.get(), 100L, 100L, metrics);
+ assertCounts(5, 1L, 0L, metrics);
+ assertBounds(5, Types.LongType.get(), 20L, 20L, metrics);
+ assertCounts(6, 1L, 0L, metrics);
assertBounds(
6,
Types.BinaryType.get(),
ByteBuffer.wrap("A".getBytes()),
ByteBuffer.wrap("A".getBytes()),
- metricsWithStats);
- assertCounts(7, 1L, 0L, 1L, metricsWithStats);
- assertBounds(7, Types.DoubleType.get(), null, null, metricsWithStats);
+ metrics);
+ assertCounts(7, 1L, 0L, 1L, metrics);
+ assertBounds(7, Types.DoubleType.get(), null, null, metrics);
}
@TestTemplate
@@ -682,16 +636,14 @@ public abstract class TestMetrics {
singleStringColSchema,
null),
record);
- MetricsWithStats metricsWithStats =
- new MetricsWithStats(metrics, MetricsUtil.fromMetrics(singleStringColSchema, metrics));
CharBuffer expectedMinBound = CharBuffer.wrap("Lorem ipsu");
CharBuffer expectedMaxBound = CharBuffer.wrap("Lorem ipsv");
assertThat(metrics.recordCount()).isEqualTo(1L);
assertThat(metrics.columnSizes()).doesNotContainValue(null);
assertThat(metrics.columnSizes()).isNotEmpty();
- assertCounts(1, 1L, 0L, metricsWithStats);
- assertBounds(1, Types.StringType.get(), expectedMinBound, expectedMaxBound, metricsWithStats);
+ assertCounts(1, 1L, 0L, metrics);
+ assertBounds(1, Types.StringType.get(), expectedMinBound, expectedMaxBound, metrics);
}
@TestTemplate
@@ -711,16 +663,14 @@ public abstract class TestMetrics {
singleBinaryColSchema,
null),
record);
- MetricsWithStats metricsWithStats =
- new MetricsWithStats(metrics, MetricsUtil.fromMetrics(singleBinaryColSchema, metrics));
ByteBuffer expectedMinBounds = ByteBuffer.wrap(new byte[] {0x1, 0x2, 0x3, 0x4, 0x5});
ByteBuffer expectedMaxBounds = ByteBuffer.wrap(new byte[] {0x1, 0x2, 0x3, 0x4, 0x6});
assertThat(metrics.recordCount()).isEqualTo(1L);
assertThat(metrics.columnSizes()).doesNotContainValue(null);
assertThat(metrics.columnSizes()).isNotEmpty();
- assertCounts(1, 1L, 0L, metricsWithStats);
- assertBounds(1, Types.BinaryType.get(), expectedMinBounds, expectedMaxBounds, metricsWithStats);
+ assertCounts(1, 1L, 0L, metrics);
+ assertBounds(1, Types.BinaryType.get(), expectedMinBounds, expectedMaxBounds, metrics);
}
@TestTemplate
@@ -772,22 +722,15 @@ public abstract class TestMetrics {
Metrics metrics =
getMetrics(SIMPLE_SCHEMA, MetricsConfig.forTable(table), firstRecord, secondRecord);
- MetricsWithStats metricsWithStats =
- new MetricsWithStats(metrics, MetricsUtil.fromMetrics(SIMPLE_SCHEMA, metrics));
assertThat(metrics.recordCount()).isEqualTo(2L);
- assertBounds(1, BooleanType.get(), false, true, metricsWithStats);
- assertBounds(2, IntegerType.get(), Integer.MIN_VALUE, Integer.MAX_VALUE, metricsWithStats);
- assertBounds(3, LongType.get(), Long.MIN_VALUE, Long.MAX_VALUE, metricsWithStats);
- assertBounds(
- 6,
- DecimalType.of(10, 2),
- new BigDecimal("0.00"),
- new BigDecimal("10.00"),
- metricsWithStats);
+ assertBounds(1, BooleanType.get(), false, true, metrics);
+ assertBounds(2, IntegerType.get(), Integer.MIN_VALUE, Integer.MAX_VALUE, metrics);
+ assertBounds(3, LongType.get(), Long.MIN_VALUE, Long.MAX_VALUE, metrics);
assertBounds(
- 7, StringType.get(), CharBuffer.wrap("AAA"), CharBuffer.wrap("ZZZ"), metricsWithStats);
- assertBounds(8, DateType.get(), 1500, 3000, metricsWithStats);
+ 6, DecimalType.of(10, 2), new BigDecimal("0.00"), new BigDecimal("10.00"), metrics);
+ assertBounds(7, StringType.get(), CharBuffer.wrap("AAA"), CharBuffer.wrap("ZZZ"), metrics);
+ assertBounds(8, DateType.get(), 1500, 3000, metrics);
}
@TestTemplate
@@ -813,51 +756,29 @@ public abstract class TestMetrics {
record.setField("nestedStructCol", nestedStruct);
Metrics metrics = getMetrics(NESTED_SCHEMA, MetricsConfig.forTable(table), record);
- MetricsWithStats metricsWithStats =
- new MetricsWithStats(metrics, MetricsUtil.fromMetrics(NESTED_SCHEMA, metrics));
- assertBounds(3, LongType.get(), Long.MAX_VALUE, Long.MAX_VALUE, metricsWithStats);
- assertBounds(5, LongType.get(), Long.MAX_VALUE, Long.MAX_VALUE, metricsWithStats);
+ assertBounds(3, LongType.get(), Long.MAX_VALUE, Long.MAX_VALUE, metrics);
+ assertBounds(5, LongType.get(), Long.MAX_VALUE, Long.MAX_VALUE, metrics);
}
- protected void assertCounts(
- int fieldId, Long valueCount, Long nullValueCount, MetricsWithStats metricsWithStats) {
- assertCounts(fieldId, valueCount, nullValueCount, null, metricsWithStats);
+ protected void assertCounts(int fieldId, Long valueCount, Long nullValueCount, Metrics metrics) {
+ assertCounts(fieldId, valueCount, nullValueCount, null, metrics);
}
protected void assertCounts(
- int fieldId,
- Long valueCount,
- Long nullValueCount,
- Long nanValueCount,
- MetricsWithStats metricsWithStats) {
- assertThat(metricsWithStats.metrics().valueCounts().get(fieldId)).isEqualTo(valueCount);
- assertThat(metricsWithStats.metrics().nullValueCounts().get(fieldId)).isEqualTo(nullValueCount);
- assertThat(metricsWithStats.metrics().nanValueCounts().get(fieldId)).isEqualTo(nanValueCount);
- if (null != metricsWithStats.stats()) {
- assertSerializable(metricsWithStats.stats());
- FieldStats<?> stat = metricsWithStats.stats().statsFor(fieldId);
- if (null == stat) {
- // stat is only null when metrics mode is set to none
- assertThat(valueCount).isNull();
- assertThat(nullValueCount).isNull();
- assertThat(nanValueCount).isNull();
- } else {
- assertThat(stat.valueCount()).isEqualTo(valueCount);
- assertThat(stat.nullValueCount()).isEqualTo(nullValueCount);
- assertThat(stat.nanValueCount()).isEqualTo(nanValueCount);
- }
- }
+ int fieldId, Long valueCount, Long nullValueCount, Long nanValueCount, Metrics metrics) {
+ assertThat(metrics.valueCounts().get(fieldId)).isEqualTo(valueCount);
+ assertThat(metrics.nullValueCounts().get(fieldId)).isEqualTo(nullValueCount);
+ assertThat(metrics.nanValueCounts().get(fieldId)).isEqualTo(nanValueCount);
}
- @SuppressWarnings("DataFlowIssue")
protected <T> void assertBounds(
- int fieldId, Type type, T lowerBound, T upperBound, MetricsWithStats metricsWithStats) {
- Map<Integer, ByteBuffer> lowerBounds = metricsWithStats.metrics().lowerBounds();
- Map<Integer, ByteBuffer> upperBounds = metricsWithStats.metrics().upperBounds();
+ int fieldId, Type type, T lowerBound, T upperBound, Metrics metrics) {
+ Map<Integer, ByteBuffer> lowerBounds = metrics.lowerBounds();
+ Map<Integer, ByteBuffer> upperBounds = metrics.upperBounds();
if (null != lowerBound || null != upperBound) {
// if there's an expected lower/upper bound, then the original type should be available
- assertThat(metricsWithStats.metrics().originalTypes().get(fieldId)).isEqualTo(type);
+ assertThat(metrics.originalTypes().get(fieldId)).isEqualTo(type);
}
if (lowerBounds.containsKey(fieldId)) {
@@ -871,36 +792,5 @@ public abstract class TestMetrics {
} else {
assertThat(upperBound).isNull();
}
-
- if (null != metricsWithStats.stats()) {
- assertSerializable(metricsWithStats.stats());
- FieldStats<?> stat = metricsWithStats.stats().statsFor(fieldId);
- if (null == stat) {
- // stat is only null when metrics mode is set to none
- assertThat(lowerBound).isNull();
- assertThat(upperBound).isNull();
- } else {
- assertThat(stat).isNotNull();
- if (lowerBound instanceof CharBuffer || upperBound instanceof CharBuffer) {
- // CharBuffer is stored as String in content stats
- assertThat(stat.lowerBound()).isEqualTo(lowerBound.toString());
- assertThat(stat.upperBound()).isEqualTo(upperBound.toString());
- } else {
- assertThat(stat.lowerBound()).isEqualTo(lowerBound);
- assertThat(stat.upperBound()).isEqualTo(upperBound);
- }
- }
- }
- }
-
- protected record MetricsWithStats(Metrics metrics, ContentStats stats) {}
-
- private void assertSerializable(ContentStats stats) {
- try {
- ContentStats serialized = TestHelpers.roundTripSerialize(stats);
- assertThat(serialized).isEqualTo(stats);
- } catch (IOException | ClassNotFoundException e) {
- throw new RuntimeException(e);
- }
}
}
diff --git a/data/src/test/java/org/apache/iceberg/orc/TestOrcMetrics.java b/data/src/test/java/org/apache/iceberg/orc/TestOrcMetrics.java
index c1647bb716..cde3bf97d1 100644
--- a/data/src/test/java/org/apache/iceberg/orc/TestOrcMetrics.java
+++ b/data/src/test/java/org/apache/iceberg/orc/TestOrcMetrics.java
@@ -144,12 +144,12 @@ public class TestOrcMetrics extends TestMetrics {
@Override
protected <T> void assertBounds(
- int fieldId, Type type, T lowerBound, T upperBound, MetricsWithStats metricsWithStats) {
+ int fieldId, Type type, T lowerBound, T upperBound, Metrics metrics) {
if (isBinaryType(type)) {
- assertThat(metricsWithStats.metrics().lowerBounds()).doesNotContainKey(fieldId);
- assertThat(metricsWithStats.metrics().upperBounds()).doesNotContainKey(fieldId);
+ assertThat(metrics.lowerBounds()).doesNotContainKey(fieldId);
+ assertThat(metrics.upperBounds()).doesNotContainKey(fieldId);
return;
}
- super.assertBounds(fieldId, type, lowerBound, upperBound, metricsWithStats);
+ super.assertBounds(fieldId, type, lowerBound, upperBound, metrics);
}
}
diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java
index 42950bc073..67d6a0bc73 100644
--- a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java
+++ b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java
@@ -50,6 +50,7 @@ import org.apache.iceberg.util.NaNUtil;
import org.apache.iceberg.util.UnicodeUtil;
import org.apache.iceberg.variants.PhysicalType;
import org.apache.iceberg.variants.ShreddedObject;
+import org.apache.iceberg.variants.Variant;
import org.apache.iceberg.variants.VariantMetadata;
import org.apache.iceberg.variants.VariantValue;
import org.apache.iceberg.variants.Variants;
@@ -66,43 +67,64 @@ import org.apache.parquet.schema.Type;
class ParquetMetrics {
private ParquetMetrics() {}
- static Metrics metrics(
+ static Iterable<FieldMetrics<?>> fieldMetrics(
Schema schema,
MessageType type,
MetricsConfig metricsConfig,
ParquetMetadata metadata,
Stream<FieldMetrics<?>> fields) {
- long rowCount = 0L;
- Map<Integer, Long> columnSizes = Maps.newHashMap();
Multimap<ColumnPath, ColumnChunkMetaData> columns =
Multimaps.newMultimap(Maps.newHashMap(), Lists::newArrayList);
for (BlockMetaData block : metadata.getBlocks()) {
- rowCount += block.getRowCount();
for (ColumnChunkMetaData column : block.getColumns()) {
columns.put(column.getPath(), column);
+ }
+ }
+ Map<Integer, FieldMetrics<?>> metricsById =
+ fields.collect(Collectors.toMap(FieldMetrics::id, Function.identity()));
+
+ return TypeWithSchemaVisitor.visit(
+ schema.asStruct(), type, new MetricsVisitor(schema, metricsConfig, metricsById, columns));
+ }
+
+ private static long rowCount(ParquetMetadata metadata) {
+ long rowCount = 0L;
+ for (BlockMetaData block : metadata.getBlocks()) {
+ rowCount += block.getRowCount();
+ }
+
+ return rowCount;
+ }
+
+ private static Map<Integer, Long> columnSizes(
+ Schema schema, MessageType type, ParquetMetadata metadata, MetricsConfig metricsConfig) {
+ Map<Integer, Long> columnSizes = Maps.newHashMap();
+ for (BlockMetaData block : metadata.getBlocks()) {
+ for (ColumnChunkMetaData column : block.getColumns()) {
Type.ID id =
type.getColumnDescription(column.getPath().toArray()).getPrimitiveType().getId();
- if (null == id) {
- continue;
- }
-
- int fieldId = id.intValue();
- MetricsModes.MetricsMode mode = MetricsUtil.metricsMode(schema, metricsConfig, fieldId);
- if (mode != MetricsModes.None.get()) {
- columnSizes.put(fieldId, columnSizes.getOrDefault(fieldId, 0L) + column.getTotalSize());
+ if (id != null) {
+ int fieldId = id.intValue();
+ MetricsModes.MetricsMode mode = MetricsUtil.metricsMode(schema, metricsConfig, fieldId);
+ if (mode != MetricsModes.None.get()) {
+ columnSizes.put(fieldId, columnSizes.getOrDefault(fieldId, 0L) + column.getTotalSize());
+ }
}
}
}
- Map<Integer, FieldMetrics<?>> metricsById =
- fields.collect(Collectors.toMap(FieldMetrics::id, Function.identity()));
+ return columnSizes;
+ }
- Iterable<FieldMetrics<ByteBuffer>> results =
- TypeWithSchemaVisitor.visit(
- schema.asStruct(),
- type,
- new MetricsVisitor(schema, metricsConfig, metricsById, columns));
+ static Metrics metrics(
+ Schema schema,
+ MessageType type,
+ MetricsConfig metricsConfig,
+ ParquetMetadata metadata,
+ Stream<FieldMetrics<?>> fields) {
+ long rowCount = rowCount(metadata);
+ Map<Integer, Long> columnSizes = columnSizes(schema, type, metadata, metricsConfig);
Map<Integer, Long> valueCounts = Maps.newHashMap();
Map<Integer, Long> nullValueCounts = Maps.newHashMap();
@@ -111,7 +133,7 @@ class ParquetMetrics {
Map<Integer, ByteBuffer> upperBounds = Maps.newHashMap();
Map<Integer, org.apache.iceberg.types.Type> originalTypes = Maps.newHashMap();
- for (FieldMetrics<ByteBuffer> metrics : results) {
+ for (FieldMetrics<?> metrics : fieldMetrics(schema, type, metricsConfig, metadata, fields)) {
int id = metrics.id();
if (null != metrics.originalType()) {
originalTypes.put(id, metrics.originalType());
@@ -130,11 +152,15 @@ class ParquetMetrics {
}
if (metrics.lowerBound() != null) {
- lowerBounds.put(id, metrics.lowerBound());
+ ByteBuffer lowerBound =
+ Conversions.toByteBuffer(metrics.originalType(), metrics.lowerBound());
+ lowerBounds.put(id, lowerBound);
}
if (metrics.upperBound() != null) {
- upperBounds.put(id, metrics.upperBound());
+ ByteBuffer upperBound =
+ Conversions.toByteBuffer(metrics.originalType(), metrics.upperBound());
+ upperBounds.put(id, upperBound);
}
}
@@ -149,8 +175,7 @@ class ParquetMetrics {
originalTypes);
}
- private static class MetricsVisitor
- extends TypeWithSchemaVisitor<Iterable<FieldMetrics<ByteBuffer>>> {
+ private static class MetricsVisitor extends TypeWithSchemaVisitor<Iterable<FieldMetrics<?>>> {
private final Schema schema;
private final MetricsConfig metricsConfig;
private final Map<Integer, FieldMetrics<?>> metricsById;
@@ -168,40 +193,38 @@ class ParquetMetrics {
}
@Override
- public Iterable<FieldMetrics<ByteBuffer>> message(
+ public Iterable<FieldMetrics<?>> message(
Types.StructType iStruct,
MessageType message,
- List<Iterable<FieldMetrics<ByteBuffer>>> fieldResults) {
+ List<Iterable<FieldMetrics<?>>> fieldResults) {
return Iterables.concat(fieldResults);
}
@Override
- public Iterable<FieldMetrics<ByteBuffer>> struct(
- Types.StructType iStruct,
- GroupType struct,
- List<Iterable<FieldMetrics<ByteBuffer>>> fieldResults) {
+ public Iterable<FieldMetrics<?>> struct(
+ Types.StructType iStruct, GroupType struct, List<Iterable<FieldMetrics<?>>> fieldResults) {
return Iterables.concat(fieldResults);
}
@Override
- public Iterable<FieldMetrics<ByteBuffer>> list(
- Types.ListType iList, GroupType array, Iterable<FieldMetrics<ByteBuffer>> elementResults) {
+ public Iterable<FieldMetrics<?>> list(
+ Types.ListType iList, GroupType array, Iterable<FieldMetrics<?>> elementResults) {
// remove lower and upper bounds for repeated fields
return ImmutableList.of();
}
@Override
- public Iterable<FieldMetrics<ByteBuffer>> map(
+ public Iterable<FieldMetrics<?>> map(
Types.MapType iMap,
GroupType map,
- Iterable<FieldMetrics<ByteBuffer>> keyResults,
- Iterable<FieldMetrics<ByteBuffer>> valueResults) {
+ Iterable<FieldMetrics<?>> keyResults,
+ Iterable<FieldMetrics<?>> valueResults) {
// repeated fields are not currently supported
return ImmutableList.of();
}
@Override
- public Iterable<FieldMetrics<ByteBuffer>> primitive(
+ public Iterable<FieldMetrics<?>> primitive(
org.apache.iceberg.types.Type.PrimitiveType iPrimitive, PrimitiveType primitive) {
Type.ID id = primitive.getId();
if (null == id) {
@@ -216,7 +239,8 @@ class ParquetMetrics {
int length = truncateLength(mode);
- FieldMetrics<ByteBuffer> metrics = metricsFromFieldMetrics(fieldId, iPrimitive, length);
+ FieldMetrics<?> metrics =
+ metricsFromFieldMetrics(metricsById.get(fieldId), iPrimitive, length);
if (metrics != null) {
return ImmutableList.of(metrics);
}
@@ -229,9 +253,10 @@ class ParquetMetrics {
return ImmutableList.of();
}
- private FieldMetrics<ByteBuffer> metricsFromFieldMetrics(
- int fieldId, org.apache.iceberg.types.Type.PrimitiveType icebergType, int truncateLength) {
- FieldMetrics<?> fieldMetrics = metricsById.get(fieldId);
+ private <T> FieldMetrics<T> metricsFromFieldMetrics(
+ FieldMetrics<T> fieldMetrics,
+ org.apache.iceberg.types.Type.PrimitiveType icebergType,
+ int truncateLength) {
if (null == fieldMetrics) {
return null;
} else if (truncateLength <= 0) {
@@ -241,24 +266,20 @@ class ParquetMetrics {
fieldMetrics.nullValueCount(),
fieldMetrics.nanValueCount());
} else {
- Object lowerBound =
- truncateLowerBound(icebergType, fieldMetrics.lowerBound(), truncateLength);
- Object upperBound =
- truncateUpperBound(icebergType, fieldMetrics.upperBound(), truncateLength);
- ByteBuffer lower = Conversions.toByteBuffer(icebergType, lowerBound);
- ByteBuffer upper = Conversions.toByteBuffer(icebergType, upperBound);
+ T lowerBound = truncateLowerBound(icebergType, fieldMetrics.lowerBound(), truncateLength);
+ T upperBound = truncateUpperBound(icebergType, fieldMetrics.upperBound(), truncateLength);
return new FieldMetrics<>(
fieldMetrics.id(),
fieldMetrics.valueCount(),
fieldMetrics.nullValueCount(),
fieldMetrics.nanValueCount(),
- lower,
- upper,
+ lowerBound,
+ upperBound,
icebergType);
}
}
- private FieldMetrics<ByteBuffer> metricsFromFooter(
+ private <T> FieldMetrics<T> metricsFromFooter(
int fieldId,
org.apache.iceberg.types.Type.PrimitiveType icebergType,
PrimitiveType primitive,
@@ -278,7 +299,7 @@ class ParquetMetrics {
return typeId == TypeID.GEOMETRY || typeId == TypeID.GEOGRAPHY;
}
- private FieldMetrics<ByteBuffer> counts(int fieldId) {
+ private <T> FieldMetrics<T> counts(int fieldId) {
ColumnPath path = ColumnPath.get(currentPath());
long valueCount = 0;
long nullCount = 0;
@@ -296,7 +317,7 @@ class ParquetMetrics {
return new FieldMetrics<>(fieldId, valueCount, nullCount);
}
- private <T> FieldMetrics<ByteBuffer> bounds(
+ private <T> FieldMetrics<T> bounds(
int fieldId,
org.apache.iceberg.types.Type.PrimitiveType icebergType,
PrimitiveType primitive,
@@ -343,16 +364,14 @@ class ParquetMetrics {
lowerBound = truncateLowerBound(icebergType, lowerBound, truncateLength);
upperBound = truncateUpperBound(icebergType, upperBound, truncateLength);
- ByteBuffer lower = Conversions.toByteBuffer(icebergType, lowerBound);
- ByteBuffer upper = Conversions.toByteBuffer(icebergType, upperBound);
-
- return new FieldMetrics<>(fieldId, valueCount, nullCount, lower, upper, icebergType);
+ return new FieldMetrics<>(
+ fieldId, valueCount, nullCount, lowerBound, upperBound, icebergType);
}
@Override
@SuppressWarnings("CyclomaticComplexity")
- public Iterable<FieldMetrics<ByteBuffer>> variant(
- Types.VariantType iVariant, GroupType variant, Iterable<FieldMetrics<ByteBuffer>> ignored) {
+ public Iterable<FieldMetrics<?>> variant(
+ Types.VariantType iVariant, GroupType variant, Iterable<FieldMetrics<?>> ignored) {
Type.ID id = variant.getId();
if (null == id) {
return ImmutableList.of();
@@ -409,8 +428,8 @@ class ParquetMetrics {
fieldId,
metadataCounts.valueCount(),
metadataCounts.nullCount(),
- ParquetVariantUtil.toByteBuffer(metadata, lowerBounds),
- ParquetVariantUtil.toByteBuffer(metadata, upperBounds),
+ Variant.of(metadata, lowerBounds),
+ Variant.of(metadata, upperBounds),
Types.VariantType.get()));
}