You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by sz...@apache.org on 2026/08/28 00:26:23 UTC
(iceberg) branch main updated: API, Core, Parquet: Carry avg value sizes for v4 content stats (#17451)
This is an automated email from the ASF dual-hosted git repository.
szehon-ho pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new a001f40f08 API, Core, Parquet: Carry avg value sizes for v4 content stats (#17451)
a001f40f08 is described below
commit a001f40f0859e842eb0bd0dfbbab6321ff235f1b
Author: Xin Huang <42...@users.noreply.github.com>
AuthorDate: Thu Aug 27 17:26:15 2026 -0700
API, Core, Parquet: Carry avg value sizes for v4 content stats (#17451)
---
.../main/java/org/apache/iceberg/ContentFile.java | 19 ++++--
api/src/main/java/org/apache/iceberg/Metrics.java | 52 +++++++++++++++
.../apache/iceberg/TestMetricsSerialization.java | 11 +++-
.../src/main/java/org/apache/iceberg/BaseFile.java | 17 +++++
.../org/apache/iceberg/ContentStatsBackedMap.java | 10 ++-
.../main/java/org/apache/iceberg/DataFiles.java | 5 ++
.../main/java/org/apache/iceberg/Delegates.java | 5 ++
.../main/java/org/apache/iceberg/FieldStats.java | 2 +-
.../main/java/org/apache/iceberg/FileMetadata.java | 5 ++
.../java/org/apache/iceberg/GenericDataFile.java | 1 +
.../java/org/apache/iceberg/GenericDeleteFile.java | 1 +
.../main/java/org/apache/iceberg/MetricsUtil.java | 14 +++--
.../org/apache/iceberg/TrackedFileAdapters.java | 5 ++
.../org/apache/iceberg/util/ContentFileUtil.java | 8 ++-
.../java/org/apache/iceberg/StatsTestUtil.java | 19 ++++++
.../apache/iceberg/TestContentStatsBackedMap.java | 13 ++++
.../test/java/org/apache/iceberg/TestMetrics.java | 8 ++-
.../java/org/apache/iceberg/TestMetricsUtil.java | 73 ++++++++++++++++++++++
.../apache/iceberg/TestTrackedFileAdapters.java | 24 +++++--
.../org/apache/iceberg/parquet/ParquetMetrics.java | 6 ++
.../iceberg/parquet/TestParquetDataWriter.java | 17 ++++-
21 files changed, 289 insertions(+), 26 deletions(-)
diff --git a/api/src/main/java/org/apache/iceberg/ContentFile.java b/api/src/main/java/org/apache/iceberg/ContentFile.java
index c4cd3c06ad..18ea03d91a 100644
--- a/api/src/main/java/org/apache/iceberg/ContentFile.java
+++ b/api/src/main/java/org/apache/iceberg/ContentFile.java
@@ -99,6 +99,17 @@ public interface ContentFile<F> {
/** Returns if collected, map from column ID to value upper bounds, null otherwise. */
Map<Integer, ByteBuffer> upperBounds();
+ /**
+ * Returns if collected, map from column ID to its average value size in memory (uncompressed) in
+ * bytes over non-null values, null otherwise.
+ *
+ * <p>This statistic is not persisted in manifests prior to v4, so it is generally only present on
+ * newly created or written files, and is null for files read back from such manifests.
+ */
+ default Map<Integer, Integer> avgValueSizes() {
+ return null;
+ }
+
/**
* Returns metadata about how this file is encrypted, or null if the file is stored in plain text.
*/
@@ -187,7 +198,7 @@ public interface ContentFile<F> {
* to copy data without stats when collecting files.
*
* @return a copy of this data file, without lower bounds, upper bounds, value counts, null value
- * counts, or nan value counts
+ * counts, nan value counts, or average value sizes
*/
F copyWithoutStats();
@@ -198,7 +209,7 @@ public interface ContentFile<F> {
*
* @param requestedColumnIds column IDs for which to keep stats.
* @return a copy of data file, with lower bounds, upper bounds, value counts, null value counts,
- * and nan value counts for only specific columns.
+ * nan value counts, and average value sizes for only specific columns.
*/
default F copyWithStats(Set<Integer> requestedColumnIds) {
throw new UnsupportedOperationException(
@@ -211,8 +222,8 @@ public interface ContentFile<F> {
*
* @param withStats Will copy this file without file stats if set to <code>false</code>.
* @return a copy of this data file. If <code>withStats</code> is set to <code>false</code> the
- * file will not contain lower bounds, upper bounds, value counts, null value counts, or nan
- * value counts
+ * file will not contain lower bounds, upper bounds, value counts, null value counts, nan
+ * value counts, or average value sizes
*/
default F copy(boolean withStats) {
return withStats ? copy() : copyWithoutStats();
diff --git a/api/src/main/java/org/apache/iceberg/Metrics.java b/api/src/main/java/org/apache/iceberg/Metrics.java
index 5b9d727a8a..ce16c68acb 100644
--- a/api/src/main/java/org/apache/iceberg/Metrics.java
+++ b/api/src/main/java/org/apache/iceberg/Metrics.java
@@ -38,6 +38,7 @@ public class Metrics implements Serializable {
private Map<Integer, Long> nanValueCounts = null;
private Map<Integer, ByteBuffer> lowerBounds = null;
private Map<Integer, ByteBuffer> upperBounds = null;
+ private Map<Integer, Integer> avgValueSizes = null;
// this is not serialized with all the other fields
private Map<Integer, Type> originalTypes = null;
@@ -126,6 +127,44 @@ public class Metrics implements Serializable {
Map<Integer, ByteBuffer> lowerBounds,
Map<Integer, ByteBuffer> upperBounds,
Map<Integer, Type> originalTypes) {
+ this(
+ rowCount,
+ columnSizes,
+ valueCounts,
+ nullValueCounts,
+ nanValueCounts,
+ lowerBounds,
+ upperBounds,
+ null /* avgValueSizes */,
+ originalTypes);
+ }
+
+ /**
+ * Creates a new metrics instance.
+ *
+ * @param rowCount the number of rows (records) in the file, or null if unknown
+ * @param columnSizes a map of field id to the size in bytes of the column, or null if unknown
+ * @param valueCounts a map of field id to the number of all values (including nulls, NaN, and
+ * repeated), or null if unknown
+ * @param nullValueCounts a map of field id to the number of null values, or null if unknown
+ * @param nanValueCounts a map of field id to the number of NaN values, or null if unknown
+ * @param lowerBounds a map of field id to the lower bound of the column, or null if unknown
+ * @param upperBounds a map of field id to the upper bound of the column, or null if unknown
+ * @param avgValueSizes a map of field id to the average size in bytes of the column's non-null
+ * values, or null if unknown
+ * @param originalTypes a map of field id to the original type of the lower/upper bound, or null
+ * if unknown
+ */
+ public Metrics(
+ Long rowCount,
+ Map<Integer, Long> columnSizes,
+ Map<Integer, Long> valueCounts,
+ Map<Integer, Long> nullValueCounts,
+ Map<Integer, Long> nanValueCounts,
+ Map<Integer, ByteBuffer> lowerBounds,
+ Map<Integer, ByteBuffer> upperBounds,
+ Map<Integer, Integer> avgValueSizes,
+ Map<Integer, Type> originalTypes) {
this.rowCount = rowCount;
this.columnSizes = columnSizes;
this.valueCounts = valueCounts;
@@ -133,6 +172,7 @@ public class Metrics implements Serializable {
this.nanValueCounts = nanValueCounts;
this.lowerBounds = lowerBounds;
this.upperBounds = upperBounds;
+ this.avgValueSizes = avgValueSizes;
this.originalTypes = originalTypes;
}
@@ -204,6 +244,16 @@ public class Metrics implements Serializable {
return upperBounds;
}
+ /**
+ * Get the average value size in memory (uncompressed) in bytes over non-null values, for all
+ * fields where it was collected.
+ *
+ * @return a Map of fieldId to average value size in bytes
+ */
+ public Map<Integer, Integer> avgValueSizes() {
+ return avgValueSizes;
+ }
+
/**
* Get the non-null original types for the upper/lower bound for all fields in a file.
*
@@ -228,6 +278,7 @@ public class Metrics implements Serializable {
writeByteBufferMap(out, lowerBounds);
writeByteBufferMap(out, upperBounds);
+ out.writeObject(avgValueSizes);
}
private static void writeByteBufferMap(
@@ -264,6 +315,7 @@ public class Metrics implements Serializable {
lowerBounds = readByteBufferMap(in);
upperBounds = readByteBufferMap(in);
+ avgValueSizes = (Map<Integer, Integer>) in.readObject();
}
@SuppressWarnings("DangerousJavaDeserialization")
diff --git a/api/src/test/java/org/apache/iceberg/TestMetricsSerialization.java b/api/src/test/java/org/apache/iceberg/TestMetricsSerialization.java
index a97aa5cfe8..da482d028c 100644
--- a/api/src/test/java/org/apache/iceberg/TestMetricsSerialization.java
+++ b/api/src/test/java/org/apache/iceberg/TestMetricsSerialization.java
@@ -93,7 +93,9 @@ public class TestMetricsSerialization {
Map<Integer, Type> originalTypes =
ImmutableMap.of(1, Types.IntegerType.get(), 2, Types.IntegerType.get());
- return new Metrics(0L, longMap1, longMap2, longMap3, null, byteMap1, byteMap2, originalTypes);
+ Map<Integer, Integer> avgValueSizes = ImmutableMap.of(9, 10);
+ return new Metrics(
+ 0L, longMap1, longMap2, longMap3, null, byteMap1, byteMap2, avgValueSizes, originalTypes);
}
private static Metrics generateMetricsWithNulls() {
@@ -106,7 +108,11 @@ public class TestMetricsSerialization {
byteMap.put(4, null);
Map<Integer, Type> originalTypes = ImmutableMap.of(4, Types.IntegerType.get());
- return new Metrics(null, null, longMap, longMap, null, null, byteMap, originalTypes);
+ Map<Integer, Integer> avgValueSizes = Maps.newHashMap();
+ avgValueSizes.put(null, 1);
+ avgValueSizes.put(2, null);
+ return new Metrics(
+ null, null, longMap, longMap, null, null, byteMap, avgValueSizes, originalTypes);
}
private static void assertEquals(Metrics expected, Metrics actual) {
@@ -114,6 +120,7 @@ public class TestMetricsSerialization {
assertThat(actual.columnSizes()).isEqualTo(expected.columnSizes());
assertThat(actual.valueCounts()).isEqualTo(expected.valueCounts());
assertThat(actual.nullValueCounts()).isEqualTo(expected.nullValueCounts());
+ assertThat(actual.avgValueSizes()).isEqualTo(expected.avgValueSizes());
assertEquals(expected.lowerBounds(), actual.lowerBounds());
assertEquals(expected.upperBounds(), actual.upperBounds());
diff --git a/core/src/main/java/org/apache/iceberg/BaseFile.java b/core/src/main/java/org/apache/iceberg/BaseFile.java
index 9c6a4d5ef6..f59ddadfdc 100644
--- a/core/src/main/java/org/apache/iceberg/BaseFile.java
+++ b/core/src/main/java/org/apache/iceberg/BaseFile.java
@@ -70,6 +70,7 @@ abstract class BaseFile<F> extends SupportsIndexProjection
private Map<Integer, Long> nanValueCounts = null;
private Map<Integer, ByteBuffer> lowerBounds = null;
private Map<Integer, ByteBuffer> upperBounds = null;
+ private Map<Integer, Integer> avgValueSizes = null;
private long[] splitOffsets = null;
private int[] equalityIds = null;
private byte[] keyMetadata = null;
@@ -148,6 +149,7 @@ abstract class BaseFile<F> extends SupportsIndexProjection
Map<Integer, Long> nanValueCounts,
Map<Integer, ByteBuffer> lowerBounds,
Map<Integer, ByteBuffer> upperBounds,
+ Map<Integer, Integer> avgValueSizes,
List<Long> splitOffsets,
int[] equalityFieldIds,
Integer sortOrderId,
@@ -180,6 +182,7 @@ abstract class BaseFile<F> extends SupportsIndexProjection
this.nanValueCounts = nanValueCounts;
this.lowerBounds = SerializableByteBufferMap.wrap(lowerBounds);
this.upperBounds = SerializableByteBufferMap.wrap(upperBounds);
+ this.avgValueSizes = avgValueSizes;
this.splitOffsets = ArrayUtil.toLongArray(splitOffsets);
this.equalityIds = equalityFieldIds;
this.sortOrderId = sortOrderId;
@@ -217,6 +220,7 @@ abstract class BaseFile<F> extends SupportsIndexProjection
this.nanValueCounts = copyMap(toCopy.nanValueCounts, requestedColumnIds);
this.lowerBounds = copyByteBufferMap(toCopy.lowerBounds, requestedColumnIds);
this.upperBounds = copyByteBufferMap(toCopy.upperBounds, requestedColumnIds);
+ this.avgValueSizes = copyAvgValueSizes(toCopy.avgValueSizes, requestedColumnIds);
} else {
this.columnSizes = null;
this.valueCounts = null;
@@ -224,6 +228,7 @@ abstract class BaseFile<F> extends SupportsIndexProjection
this.nanValueCounts = null;
this.lowerBounds = null;
this.upperBounds = null;
+ this.avgValueSizes = null;
}
this.keyMetadata =
toCopy.keyMetadata == null
@@ -521,6 +526,11 @@ abstract class BaseFile<F> extends SupportsIndexProjection
return toReadableByteBufferMap(upperBounds);
}
+ @Override
+ public Map<Integer, Integer> avgValueSizes() {
+ return toReadableMap(avgValueSizes);
+ }
+
@Override
public ByteBuffer keyMetadata() {
return keyMetadata != null ? ByteBuffer.wrap(keyMetadata) : null;
@@ -577,6 +587,12 @@ abstract class BaseFile<F> extends SupportsIndexProjection
return keys == null ? SerializableMap.copyOf(map) : SerializableMap.filteredCopyOf(map, keys);
}
+ private static Map<Integer, Integer> copyAvgValueSizes(
+ Map<Integer, Integer> map, Set<Integer> keys) {
+ Map<Integer, Integer> copy = copyMap(map, keys);
+ return copy == null || copy.isEmpty() ? null : copy;
+ }
+
private static Map<Integer, ByteBuffer> copyByteBufferMap(
Map<Integer, ByteBuffer> map, Set<Integer> keys) {
if (map == null) {
@@ -650,6 +666,7 @@ abstract class BaseFile<F> extends SupportsIndexProjection
.add("nan_value_counts", nanValueCounts)
.add("lower_bounds", lowerBounds)
.add("upper_bounds", upperBounds)
+ .add("avg_value_sizes", avgValueSizes)
.add("key_metadata", keyMetadata == null ? "null" : "(redacted)")
.add("split_offsets", splitOffsets == null ? "null" : splitOffsets())
.add("equality_ids", equalityIds == null ? "null" : equalityFieldIds())
diff --git a/core/src/main/java/org/apache/iceberg/ContentStatsBackedMap.java b/core/src/main/java/org/apache/iceberg/ContentStatsBackedMap.java
index 3f2599fc2d..aa827df529 100644
--- a/core/src/main/java/org/apache/iceberg/ContentStatsBackedMap.java
+++ b/core/src/main/java/org/apache/iceberg/ContentStatsBackedMap.java
@@ -36,7 +36,8 @@ class ContentStatsBackedMap<V> extends AbstractMap<Integer, V> {
NULL_VALUE_COUNT,
NAN_VALUE_COUNT,
LOWER_BOUND,
- UPPER_BOUND
+ UPPER_BOUND,
+ AVG_VALUE_SIZE
}
/** Per-column value counts, or null if no column tracks the value count. */
@@ -64,6 +65,11 @@ class ContentStatsBackedMap<V> extends AbstractMap<Integer, V> {
return viewOrNull(stats, Kind.UPPER_BOUND);
}
+ /** Per-column average non-null value sizes, or null if no column tracks the average size. */
+ static <V> Map<Integer, V> avgValueSizes(ContentStats stats) {
+ return viewOrNull(stats, Kind.AVG_VALUE_SIZE);
+ }
+
private final ContentStats stats;
private final Kind kind;
private Set<Entry<Integer, V>> materialized;
@@ -144,6 +150,7 @@ class ContentStatsBackedMap<V> extends AbstractMap<Integer, V> {
case NAN_VALUE_COUNT -> fieldStats.hasNanValueCount();
case LOWER_BOUND -> fieldStats.lowerBound() != null;
case UPPER_BOUND -> fieldStats.upperBound() != null;
+ case AVG_VALUE_SIZE -> fieldStats.avgValueSizeInBytes() != null;
};
}
@@ -160,6 +167,7 @@ class ContentStatsBackedMap<V> extends AbstractMap<Integer, V> {
(V) bound(fieldStats, fieldStats.lowerBound(), StatsUtil.LOWER_BOUND_NAME);
case UPPER_BOUND ->
(V) bound(fieldStats, fieldStats.upperBound(), StatsUtil.UPPER_BOUND_NAME);
+ case AVG_VALUE_SIZE -> (V) fieldStats.avgValueSizeInBytes();
};
}
diff --git a/core/src/main/java/org/apache/iceberg/DataFiles.java b/core/src/main/java/org/apache/iceberg/DataFiles.java
index 4a991f6230..9748e8f868 100644
--- a/core/src/main/java/org/apache/iceberg/DataFiles.java
+++ b/core/src/main/java/org/apache/iceberg/DataFiles.java
@@ -152,6 +152,7 @@ public class DataFiles {
private Map<Integer, Long> nanValueCounts = null;
private Map<Integer, ByteBuffer> lowerBounds = null;
private Map<Integer, ByteBuffer> upperBounds = null;
+ private Map<Integer, Integer> avgValueSizes = null;
private Map<Integer, Type> originalTypes = null;
private ByteBuffer keyMetadata = null;
private List<Long> splitOffsets = null;
@@ -179,6 +180,7 @@ public class DataFiles {
this.nanValueCounts = null;
this.lowerBounds = null;
this.upperBounds = null;
+ this.avgValueSizes = null;
this.splitOffsets = null;
this.sortOrderId = SortOrder.unsorted().orderId();
this.firstRowId = null;
@@ -200,6 +202,7 @@ public class DataFiles {
this.nanValueCounts = toCopy.nanValueCounts();
this.lowerBounds = toCopy.lowerBounds();
this.upperBounds = toCopy.upperBounds();
+ this.avgValueSizes = toCopy.avgValueSizes();
this.keyMetadata =
toCopy.keyMetadata() == null ? null : ByteBuffers.copy(toCopy.keyMetadata());
this.splitOffsets =
@@ -292,6 +295,7 @@ public class DataFiles {
this.nanValueCounts = metrics.nanValueCounts();
this.lowerBounds = metrics.lowerBounds();
this.upperBounds = metrics.upperBounds();
+ this.avgValueSizes = metrics.avgValueSizes();
this.originalTypes = metrics.originalTypes();
return this;
}
@@ -354,6 +358,7 @@ public class DataFiles {
nanValueCounts,
lowerBounds,
upperBounds,
+ avgValueSizes,
originalTypes),
keyMetadata,
splitOffsets,
diff --git a/core/src/main/java/org/apache/iceberg/Delegates.java b/core/src/main/java/org/apache/iceberg/Delegates.java
index 324fc1cccd..d89e9ded7f 100644
--- a/core/src/main/java/org/apache/iceberg/Delegates.java
+++ b/core/src/main/java/org/apache/iceberg/Delegates.java
@@ -232,6 +232,11 @@ class Delegates {
return wrapped.nanValueCounts();
}
+ @Override
+ public Map<Integer, Integer> avgValueSizes() {
+ return wrapped.avgValueSizes();
+ }
+
@Override
public Map<Integer, ByteBuffer> lowerBounds() {
return wrapped.lowerBounds();
diff --git a/core/src/main/java/org/apache/iceberg/FieldStats.java b/core/src/main/java/org/apache/iceberg/FieldStats.java
index 693765a897..ddbbc4d77c 100644
--- a/core/src/main/java/org/apache/iceberg/FieldStats.java
+++ b/core/src/main/java/org/apache/iceberg/FieldStats.java
@@ -63,7 +63,7 @@ public interface FieldStats<T> {
/**
* The avg value size in memory (uncompressed) in bytes for variable-length types (string, binary,
- * variant)
+ * variant, geometry, geography)
*/
Integer avgValueSizeInBytes();
diff --git a/core/src/main/java/org/apache/iceberg/FileMetadata.java b/core/src/main/java/org/apache/iceberg/FileMetadata.java
index a5266101c2..d8e8022482 100644
--- a/core/src/main/java/org/apache/iceberg/FileMetadata.java
+++ b/core/src/main/java/org/apache/iceberg/FileMetadata.java
@@ -57,6 +57,7 @@ public class FileMetadata {
private Map<Integer, Long> nanValueCounts = null;
private Map<Integer, ByteBuffer> lowerBounds = null;
private Map<Integer, ByteBuffer> upperBounds = null;
+ private Map<Integer, Integer> avgValueSizes = null;
private Map<Integer, Type> originalTypes = null;
private ByteBuffer keyMetadata = null;
private Integer sortOrderId = null;
@@ -86,6 +87,7 @@ public class FileMetadata {
this.nanValueCounts = null;
this.lowerBounds = null;
this.upperBounds = null;
+ this.avgValueSizes = null;
this.sortOrderId = null;
}
@@ -106,6 +108,7 @@ public class FileMetadata {
this.nanValueCounts = toCopy.nanValueCounts();
this.lowerBounds = toCopy.lowerBounds();
this.upperBounds = toCopy.upperBounds();
+ this.avgValueSizes = toCopy.avgValueSizes();
this.keyMetadata =
toCopy.keyMetadata() == null ? null : ByteBuffers.copy(toCopy.keyMetadata());
this.sortOrderId = toCopy.sortOrderId();
@@ -202,6 +205,7 @@ public class FileMetadata {
this.nanValueCounts = metrics.nanValueCounts();
this.lowerBounds = metrics.lowerBounds();
this.upperBounds = metrics.upperBounds();
+ this.avgValueSizes = metrics.avgValueSizes();
this.originalTypes = metrics.originalTypes();
return this;
}
@@ -300,6 +304,7 @@ public class FileMetadata {
nanValueCounts,
lowerBounds,
upperBounds,
+ avgValueSizes,
originalTypes),
equalityFieldIds,
sortOrderId,
diff --git a/core/src/main/java/org/apache/iceberg/GenericDataFile.java b/core/src/main/java/org/apache/iceberg/GenericDataFile.java
index 9033d0718b..3a7c1d7200 100644
--- a/core/src/main/java/org/apache/iceberg/GenericDataFile.java
+++ b/core/src/main/java/org/apache/iceberg/GenericDataFile.java
@@ -62,6 +62,7 @@ class GenericDataFile extends BaseFile<DataFile> implements DataFile {
metrics.nanValueCounts(),
metrics.lowerBounds(),
metrics.upperBounds(),
+ metrics.avgValueSizes(),
splitOffsets,
null /* no equality field IDs */,
sortOrderId,
diff --git a/core/src/main/java/org/apache/iceberg/GenericDeleteFile.java b/core/src/main/java/org/apache/iceberg/GenericDeleteFile.java
index 897d77c2fb..de732eb9d5 100644
--- a/core/src/main/java/org/apache/iceberg/GenericDeleteFile.java
+++ b/core/src/main/java/org/apache/iceberg/GenericDeleteFile.java
@@ -66,6 +66,7 @@ class GenericDeleteFile extends BaseFile<DeleteFile> implements DeleteFile {
metrics.nanValueCounts(),
metrics.lowerBounds(),
metrics.upperBounds(),
+ metrics.avgValueSizes(),
splitOffsets,
equalityFieldIds,
sortOrderId,
diff --git a/core/src/main/java/org/apache/iceberg/MetricsUtil.java b/core/src/main/java/org/apache/iceberg/MetricsUtil.java
index 96645bf074..5fa6c95a85 100644
--- a/core/src/main/java/org/apache/iceberg/MetricsUtil.java
+++ b/core/src/main/java/org/apache/iceberg/MetricsUtil.java
@@ -42,10 +42,11 @@ public class MetricsUtil {
private MetricsUtil() {}
/**
- * Copies a metrics object without value, NULL and NaN counts for given fields.
+ * Copies a metrics object without value, NULL and NaN counts or average value sizes for given
+ * fields.
*
- * @param excludedFieldIds field IDs for which the counts must be dropped
- * @return a new metrics object without counts for given fields
+ * @param excludedFieldIds field IDs for which the counts and average value sizes must be dropped
+ * @return a new metrics object without counts or average value sizes for given fields
*/
public static Metrics copyWithoutFieldCounts(Metrics metrics, Set<Integer> excludedFieldIds) {
return new Metrics(
@@ -56,13 +57,15 @@ public class MetricsUtil {
copyWithoutKeys(metrics.nanValueCounts(), excludedFieldIds),
metrics.lowerBounds(),
metrics.upperBounds(),
+ copyWithoutKeys(metrics.avgValueSizes(), excludedFieldIds),
metrics.originalTypes());
}
/**
- * Copies a metrics object without counts and bounds for given fields.
+ * Copies a metrics object without counts, average value sizes, and bounds for given fields.
*
- * @param excludedFieldIds field IDs for which the counts and bounds must be dropped
+ * @param excludedFieldIds field IDs for which the counts, average value sizes, and bounds must be
+ * dropped
* @return a new metrics object without lower and upper bounds for given fields
*/
public static Metrics copyWithoutFieldCountsAndBounds(
@@ -75,6 +78,7 @@ public class MetricsUtil {
copyWithoutKeys(metrics.nanValueCounts(), excludedFieldIds),
copyWithoutKeys(metrics.lowerBounds(), excludedFieldIds),
copyWithoutKeys(metrics.upperBounds(), excludedFieldIds),
+ copyWithoutKeys(metrics.avgValueSizes(), excludedFieldIds),
copyWithoutKeys(metrics.originalTypes(), excludedFieldIds));
}
diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java b/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java
index 154abb9f74..4750abd61e 100644
--- a/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java
+++ b/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java
@@ -187,6 +187,11 @@ class TrackedFileAdapters {
return ContentStatsBackedMap.nanValueCounts(file().contentStats());
}
+ @Override
+ public Map<Integer, Integer> avgValueSizes() {
+ return ContentStatsBackedMap.avgValueSizes(file().contentStats());
+ }
+
@Override
public Map<Integer, ByteBuffer> lowerBounds() {
return ContentStatsBackedMap.lowerBounds(file().contentStats());
diff --git a/core/src/main/java/org/apache/iceberg/util/ContentFileUtil.java b/core/src/main/java/org/apache/iceberg/util/ContentFileUtil.java
index ac7de6644c..aa7c13d6bf 100644
--- a/core/src/main/java/org/apache/iceberg/util/ContentFileUtil.java
+++ b/core/src/main/java/org/apache/iceberg/util/ContentFileUtil.java
@@ -175,7 +175,9 @@ public class ContentFileUtil {
file.nullValueCounts(),
file.nanValueCounts(),
lowerBounds == null ? null : Collections.unmodifiableMap(lowerBounds),
- upperBounds == null ? null : Collections.unmodifiableMap(upperBounds));
+ upperBounds == null ? null : Collections.unmodifiableMap(upperBounds),
+ file.avgValueSizes(),
+ null /* originalTypes */);
}
private static Metrics metricsWithPathBounds(DeleteFile file, ByteBuffer bound) {
@@ -197,6 +199,8 @@ public class ContentFileUtil {
file.nullValueCounts(),
file.nanValueCounts(),
lowerBounds == null ? null : Collections.unmodifiableMap(lowerBounds),
- upperBounds == null ? null : Collections.unmodifiableMap(upperBounds));
+ upperBounds == null ? null : Collections.unmodifiableMap(upperBounds),
+ file.avgValueSizes(),
+ null /* originalTypes */);
}
}
diff --git a/core/src/test/java/org/apache/iceberg/StatsTestUtil.java b/core/src/test/java/org/apache/iceberg/StatsTestUtil.java
index d4780d089a..93309ceb9b 100644
--- a/core/src/test/java/org/apache/iceberg/StatsTestUtil.java
+++ b/core/src/test/java/org/apache/iceberg/StatsTestUtil.java
@@ -187,6 +187,19 @@ class StatsTestUtil {
Long valueCount,
Long nullCount,
Long nanCount) {
+ return mockFieldStats(type, id, lower, upper, valueCount, nullCount, nanCount, null);
+ }
+
+ @SuppressWarnings("unchecked")
+ static FieldStats<Object> mockFieldStats(
+ Types.StructType type,
+ int id,
+ Object lower,
+ Object upper,
+ Long valueCount,
+ Long nullCount,
+ Long nanCount,
+ Integer avgValueSize) {
FieldStats<Object> stats = Mockito.mock(FieldStats.class);
Mockito.when(stats.fieldId()).thenReturn(id);
Mockito.when(stats.type()).thenReturn(type);
@@ -207,6 +220,12 @@ class StatsTestUtil {
Mockito.when(stats.nanValueCount()).thenReturn(nanCount);
}
+ // stub unconditionally: unlike the counts, avgValueSizeInBytes has no has*() gate, so
+ // ContentStatsBackedMap treats avgValueSizeInBytes() != null as the presence check. A Mockito
+ // mock defaults this method to 0, not null, so an absent stat must be stubbed to null here or
+ // it reads back as present with 0.
+ Mockito.when(stats.avgValueSizeInBytes()).thenReturn(avgValueSize);
+
return stats;
}
}
diff --git a/core/src/test/java/org/apache/iceberg/TestContentStatsBackedMap.java b/core/src/test/java/org/apache/iceberg/TestContentStatsBackedMap.java
index 865ce05827..31e6eb55d1 100644
--- a/core/src/test/java/org/apache/iceberg/TestContentStatsBackedMap.java
+++ b/core/src/test/java/org/apache/iceberg/TestContentStatsBackedMap.java
@@ -79,6 +79,18 @@ public class TestContentStatsBackedMap {
assertThat(map).containsOnly(Map.entry(3, 4L));
}
+ @Test
+ public void avgValueSizes() {
+ Schema schema = new Schema(optional(4, "str", Types.StringType.get()));
+ Types.StructType statsType = StatsUtil.statsReadSchema(schema, List.of(4));
+ Types.StructType fieldStatsType = statsType.field("str").type().asStructType();
+ ContentStatsStruct stats = new ContentStatsStruct(statsType);
+ stats.setStats(4, StatsTestUtil.mockFieldStats(fieldStatsType, 4, "a", "z", 10L, 1L, null, 12));
+
+ Map<Integer, Integer> map = ContentStatsBackedMap.avgValueSizes(stats);
+ assertThat(map).containsOnly(Map.entry(4, 12));
+ }
+
@Test
public void testLowerBounds() {
Map<Integer, ByteBuffer> lower = ContentStatsBackedMap.lowerBounds(POPULATED_STATS);
@@ -118,6 +130,7 @@ public class TestContentStatsBackedMap {
// only a required long column: it tracks neither null_value_count nor nan_value_count
assertThat(ContentStatsBackedMap.nullValueCounts(ONLY_REQUIRED_STATS)).isNull();
assertThat(ContentStatsBackedMap.nanValueCounts(ONLY_REQUIRED_STATS)).isNull();
+ assertThat(ContentStatsBackedMap.avgValueSizes(ONLY_REQUIRED_STATS)).isNull();
Map<Integer, Long> valueCounts = ContentStatsBackedMap.valueCounts(ONLY_REQUIRED_STATS);
assertThat(valueCounts).isNotNull().containsOnly(Map.entry(1, 10L));
diff --git a/core/src/test/java/org/apache/iceberg/TestMetrics.java b/core/src/test/java/org/apache/iceberg/TestMetrics.java
index 095feb2938..3c68b0e39d 100644
--- a/core/src/test/java/org/apache/iceberg/TestMetrics.java
+++ b/core/src/test/java/org/apache/iceberg/TestMetrics.java
@@ -271,9 +271,11 @@ public abstract class TestMetrics {
optional(3, "geog", Types.GeographyType.crs84()));
Record first = GenericRecord.create(schema);
+ ByteBuffer geom = wkbPoint(30, 10);
+ ByteBuffer geog = wkbPoint(-5, 40);
first.setField("id", 1L);
- first.setField("geom", wkbPoint(30, 10));
- first.setField("geog", wkbPoint(-5, 40));
+ first.setField("geom", geom);
+ first.setField("geog", geog);
Record second = GenericRecord.create(schema);
second.setField("id", 2L);
// both geo columns are left null
@@ -287,6 +289,8 @@ public abstract class TestMetrics {
assertBounds(2, Types.GeometryType.crs84(), null, null, metrics);
assertCounts(3, 2L, 1L, metrics);
assertBounds(3, Types.GeographyType.crs84(), null, null, metrics);
+ assertThat(metrics.avgValueSizes())
+ .containsOnly(Map.entry(2, geom.remaining()), Map.entry(3, geog.remaining()));
}
@TestTemplate
diff --git a/core/src/test/java/org/apache/iceberg/TestMetricsUtil.java b/core/src/test/java/org/apache/iceberg/TestMetricsUtil.java
new file mode 100644
index 0000000000..8d882db633
--- /dev/null
+++ b/core/src/test/java/org/apache/iceberg/TestMetricsUtil.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
+import org.junit.jupiter.api.Test;
+
+class TestMetricsUtil {
+
+ @Test
+ void copyWithoutFieldCountsDropsAvgValueSizesForExcludedFields() {
+ Metrics copy =
+ MetricsUtil.copyWithoutFieldCounts(metricsWithAvgValueSizes(), ImmutableSet.of(1));
+
+ assertThat(copy.avgValueSizes()).isEqualTo(ImmutableMap.of(2, 20));
+ }
+
+ @Test
+ void copyWithoutFieldCountsAndBoundsDropsAvgValueSizesForExcludedFields() {
+ Metrics copy =
+ MetricsUtil.copyWithoutFieldCountsAndBounds(metricsWithAvgValueSizes(), ImmutableSet.of(1));
+
+ assertThat(copy.avgValueSizes()).isEqualTo(ImmutableMap.of(2, 20));
+ }
+
+ @Test
+ void copyDropsAvgValueSizesEntirelyWhenAllFieldsExcluded() {
+ // once every tracked field is excluded the filtered map is empty, and copyWithoutKeys returns
+ // null rather than an empty map, matching the "null otherwise" contract of avgValueSizes()
+ Metrics copy =
+ MetricsUtil.copyWithoutFieldCounts(metricsWithAvgValueSizes(), ImmutableSet.of(1, 2));
+
+ assertThat(copy.avgValueSizes()).isNull();
+ }
+
+ @Test
+ void copyWithStatsReturnsNullAvgValueSizesWhenNoRequestedColumnMatches() {
+ DataFile file =
+ DataFiles.builder(PartitionSpec.unpartitioned())
+ .withPath("/path/to/data.parquet")
+ .withFileSizeInBytes(10)
+ .withRecordCount(1)
+ .withMetrics(metricsWithAvgValueSizes())
+ .build();
+
+ assertThat(file.copyWithStats(ImmutableSet.of(2)).avgValueSizes())
+ .isEqualTo(ImmutableMap.of(2, 20));
+ assertThat(file.copyWithStats(ImmutableSet.of(3)).avgValueSizes()).isNull();
+ }
+
+ private static Metrics metricsWithAvgValueSizes() {
+ return new Metrics(3L, null, null, null, null, null, null, ImmutableMap.of(1, 10, 2, 20), null);
+ }
+}
diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java
index c9a45928ad..284adc187d 100644
--- a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java
+++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java
@@ -66,21 +66,27 @@ class TestTrackedFileAdapters {
private static final Schema TABLE_SCHEMA =
new Schema(
- optional(1, "id", Types.IntegerType.get()), optional(2, "score", Types.FloatType.get()));
+ optional(1, "id", Types.IntegerType.get()),
+ optional(2, "score", Types.FloatType.get()),
+ optional(3, "geom", Types.GeometryType.crs84()));
private static final Types.StructType CONTENT_STATS_TYPE =
- StatsUtil.statsReadSchema(TABLE_SCHEMA, ImmutableList.of(1, 2));
+ StatsUtil.statsReadSchema(TABLE_SCHEMA, ImmutableList.of(1, 2, 3));
private static final FieldStats<?> ID_STATS =
StatsTestUtil.mockFieldStats(
CONTENT_STATS_TYPE.fieldType("id").asStructType(), 1, 1, 1000, 100L, 5L, null);
private static final FieldStats<?> SCORE_STATS =
StatsTestUtil.mockFieldStats(
CONTENT_STATS_TYPE.fieldType("score").asStructType(), 2, 1.0f, 100.0f, 100L, 10L, 3L);
+ private static final FieldStats<?> GEOM_STATS =
+ StatsTestUtil.mockFieldStats(
+ CONTENT_STATS_TYPE.fieldType("geom").asStructType(), 3, null, null, 100L, 20L, null, 12);
private static final ContentStatsStruct CONTENT_STATS =
new ContentStatsStruct(CONTENT_STATS_TYPE);
static {
CONTENT_STATS.setStats(1, ID_STATS);
CONTENT_STATS.setStats(2, SCORE_STATS);
+ CONTENT_STATS.setStats(3, GEOM_STATS);
}
@Test
@@ -136,9 +142,12 @@ class TestTrackedFileAdapters {
assertThat(dataFile.manifestLocation()).isEqualTo(MANIFEST_LOCATION);
assertThat(dataFile.equalityFieldIds()).isNull();
assertThat(dataFile.columnSizes()).isNull();
- assertThat(dataFile.valueCounts()).containsOnly(Map.entry(1, 100L), Map.entry(2, 100L));
- assertThat(dataFile.nullValueCounts()).containsOnly(Map.entry(1, 5L), Map.entry(2, 10L));
+ assertThat(dataFile.valueCounts())
+ .containsOnly(Map.entry(1, 100L), Map.entry(2, 100L), Map.entry(3, 100L));
+ assertThat(dataFile.nullValueCounts())
+ .containsOnly(Map.entry(1, 5L), Map.entry(2, 10L), Map.entry(3, 20L));
assertThat(dataFile.nanValueCounts()).containsOnly(Map.entry(2, 3L));
+ assertThat(dataFile.avgValueSizes()).containsOnly(Map.entry(3, 12));
assertThat(dataFile.lowerBounds())
.containsOnly(
Map.entry(1, Conversions.toByteBuffer(Types.IntegerType.get(), 1)),
@@ -213,9 +222,12 @@ class TestTrackedFileAdapters {
assertThat(deleteFile.manifestLocation()).isEqualTo(MANIFEST_LOCATION);
assertThat(deleteFile.equalityFieldIds()).containsExactly(1, 2, 3);
assertThat(deleteFile.columnSizes()).isNull();
- assertThat(deleteFile.valueCounts()).containsOnly(Map.entry(1, 100L), Map.entry(2, 100L));
- assertThat(deleteFile.nullValueCounts()).containsOnly(Map.entry(1, 5L), Map.entry(2, 10L));
+ assertThat(deleteFile.valueCounts())
+ .containsOnly(Map.entry(1, 100L), Map.entry(2, 100L), Map.entry(3, 100L));
+ assertThat(deleteFile.nullValueCounts())
+ .containsOnly(Map.entry(1, 5L), Map.entry(2, 10L), Map.entry(3, 20L));
assertThat(deleteFile.nanValueCounts()).containsOnly(Map.entry(2, 3L));
+ assertThat(deleteFile.avgValueSizes()).containsOnly(Map.entry(3, 12));
assertThat(deleteFile.lowerBounds())
.containsOnly(
Map.entry(1, Conversions.toByteBuffer(Types.IntegerType.get(), 1)),
diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java
index 8695cd2156..e64fb59c4d 100644
--- a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java
+++ b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java
@@ -129,6 +129,7 @@ class ParquetMetrics {
Map<Integer, Long> valueCounts = Maps.newHashMap();
Map<Integer, Long> nullValueCounts = Maps.newHashMap();
Map<Integer, Long> nanValueCounts = Maps.newHashMap();
+ Map<Integer, Integer> avgValueSizes = Maps.newHashMap();
Map<Integer, ByteBuffer> lowerBounds = Maps.newHashMap();
Map<Integer, ByteBuffer> upperBounds = Maps.newHashMap();
Map<Integer, org.apache.iceberg.types.Type> originalTypes = Maps.newHashMap();
@@ -151,6 +152,10 @@ class ParquetMetrics {
nanValueCounts.put(id, metrics.nanValueCount());
}
+ if (metrics.avgValueSizeInBytes() != null) {
+ avgValueSizes.put(id, metrics.avgValueSizeInBytes());
+ }
+
if (metrics.lowerBound() != null) {
ByteBuffer lowerBound =
Conversions.toByteBuffer(metrics.originalType(), metrics.lowerBound());
@@ -172,6 +177,7 @@ class ParquetMetrics {
nanValueCounts,
lowerBounds,
upperBounds,
+ avgValueSizes.isEmpty() ? null : avgValueSizes,
originalTypes);
}
diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetDataWriter.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetDataWriter.java
index ae7d5441c9..e6e1d175f8 100644
--- a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetDataWriter.java
+++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetDataWriter.java
@@ -27,8 +27,10 @@ import java.nio.ByteBuffer;
import java.nio.file.Path;
import java.security.SecureRandom;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
import java.util.Random;
+import java.util.Set;
import java.util.function.Function;
import java.util.function.UnaryOperator;
import org.apache.iceberg.DataFile;
@@ -121,10 +123,11 @@ public class TestParquetDataWriter {
Types.NestedField.optional(3, "geog", Types.GeographyType.crs84()));
GenericRecord record = GenericRecord.create(schema);
+ ByteBuffer geom = wkbPoint(30, 10);
+ ByteBuffer geog = wkbPoint(-5, 40);
List<Record> geoRecords =
ImmutableList.of(
- record.copy(
- ImmutableMap.of("id", 1L, "geom", wkbPoint(30, 10), "geog", wkbPoint(-5, 40))),
+ record.copy(ImmutableMap.of("id", 1L, "geom", geom, "geog", geog)),
// geog is left null
record.copy(ImmutableMap.of("id", 2L, "geom", wkbPoint(0, 0))),
// both geo columns are left null
@@ -144,7 +147,15 @@ public class TestParquetDataWriter {
}
}
- assertThat(dataWriter.toDataFile().recordCount()).isEqualTo(geoRecords.size());
+ DataFile dataFile = dataWriter.toDataFile();
+ assertThat(dataFile.recordCount()).isEqualTo(geoRecords.size());
+ assertThat(dataFile.avgValueSizes())
+ .containsOnly(Map.entry(2, geom.remaining()), Map.entry(3, geog.remaining()));
+ assertThat(dataFile.copy().avgValueSizes()).isEqualTo(dataFile.avgValueSizes());
+ assertThat(dataFile.copyWithStats(Set.of(2)).avgValueSizes())
+ .containsOnly(Map.entry(2, geom.remaining()));
+ assertThat(dataFile.copyWithStats(Set.of(1)).avgValueSizes()).isNull();
+ assertThat(dataFile.copyWithoutStats().avgValueSizes()).isNull();
List<Record> writtenRecords;
try (CloseableIterable<Record> reader =