You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by st...@apache.org on 2026/10/09 18:57:26 UTC
(iceberg) branch main updated: API, Core, Parquet, Spark: Update avg_value_size_in_bytes to total_bytes (#18310)
This is an automated email from the ASF dual-hosted git repository.
stevenzwu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 79bc69a51d API, Core, Parquet, Spark: Update avg_value_size_in_bytes to total_bytes (#18310)
79bc69a51d is described below
commit 79bc69a51da6d95ccd6da6d46ef156c74e30b8cf
Author: Eduard Tudenhoefner <et...@gmail.com>
AuthorDate: Fri Oct 9 20:57:19 2026 +0200
API, Core, Parquet, Spark: Update avg_value_size_in_bytes to total_bytes (#18310)
---
.../main/java/org/apache/iceberg/ContentFile.java | 27 +++++---
api/src/main/java/org/apache/iceberg/Metrics.java | 54 +++++++++++++---
.../apache/iceberg/TestMetricsSerialization.java | 14 ++---
.../src/main/java/org/apache/iceberg/BaseFile.java | 21 +++----
.../java/org/apache/iceberg/ContentFileParser.java | 26 ++++----
.../org/apache/iceberg/ContentStatsBackedMap.java | 13 ++--
.../main/java/org/apache/iceberg/DataFiles.java | 10 +--
.../main/java/org/apache/iceberg/Delegates.java | 4 +-
.../main/java/org/apache/iceberg/FieldMetrics.java | 73 +++++++++++++++++++++-
.../main/java/org/apache/iceberg/FieldStats.java | 26 +++++++-
.../java/org/apache/iceberg/FieldStatsStruct.java | 23 ++++---
.../main/java/org/apache/iceberg/FileMetadata.java | 10 +--
.../java/org/apache/iceberg/GenericDataFile.java | 2 +-
.../java/org/apache/iceberg/GenericDeleteFile.java | 2 +-
.../org/apache/iceberg/GeometryFieldMetrics.java | 18 +++---
.../main/java/org/apache/iceberg/MetricsUtil.java | 16 +++--
.../main/java/org/apache/iceberg/StatsUtil.java | 6 +-
.../org/apache/iceberg/TrackedFileAdapters.java | 4 +-
.../org/apache/iceberg/ValueSizeFieldMetrics.java | 10 ++-
.../org/apache/iceberg/util/ContentFileUtil.java | 4 +-
.../java/org/apache/iceberg/StatsTestUtil.java | 27 +++++---
.../org/apache/iceberg/TestContentFileParser.java | 18 +++---
.../apache/iceberg/TestContentStatsBackedMap.java | 11 ++--
.../org/apache/iceberg/TestContentStatsStruct.java | 6 +-
.../org/apache/iceberg/TestFieldStatsStruct.java | 30 ++++-----
.../apache/iceberg/TestGeometryFieldMetrics.java | 4 +-
.../test/java/org/apache/iceberg/TestMetrics.java | 2 +
.../java/org/apache/iceberg/TestMetricsUtil.java | 36 +++++------
.../java/org/apache/iceberg/TestStatsUtil.java | 24 +++----
.../apache/iceberg/TestTrackedFileAdapters.java | 6 +-
.../org/apache/iceberg/TestV4ManifestReader.java | 4 +-
.../apache/iceberg/TestValueSizeFieldMetrics.java | 4 +-
.../java/org/apache/iceberg/V4TestComparators.java | 3 +-
open-api/rest-catalog-open-api.py | 6 +-
open-api/rest-catalog-open-api.yaml | 10 +--
.../org/apache/iceberg/parquet/ParquetMetrics.java | 12 ++--
.../iceberg/parquet/ParquetValueWriters.java | 2 +-
.../iceberg/parquet/TestParquetDataWriter.java | 14 ++---
.../iceberg/parquet/TestParquetValueWriters.java | 10 ++-
.../iceberg/spark/data/TestSparkParquetWriter.java | 10 +--
.../iceberg/spark/data/TestSparkParquetWriter.java | 10 +--
41 files changed, 379 insertions(+), 233 deletions(-)
diff --git a/api/src/main/java/org/apache/iceberg/ContentFile.java b/api/src/main/java/org/apache/iceberg/ContentFile.java
index 18ea03d91a..afc880940b 100644
--- a/api/src/main/java/org/apache/iceberg/ContentFile.java
+++ b/api/src/main/java/org/apache/iceberg/ContentFile.java
@@ -99,15 +99,29 @@ public interface ContentFile<F> {
/** Returns if collected, map from column ID to value upper bounds, null otherwise. */
Map<Integer, ByteBuffer> upperBounds();
+ /**
+ * Returns map from column ID to the total uncompressed size in memory in bytes of non-null values
+ * if collected, null otherwise.
+ *
+ * <p>This statistic is not persisted in manifests prior to v4, so it is generally only present on
+ * newly created or written files, and is null for files read back from such manifests.
+ */
+ default Map<Integer, Long> totalBytes() {
+ return null;
+ }
+
/**
* Returns if collected, map from column ID to its average value size in memory (uncompressed) in
* bytes over non-null values, null otherwise.
*
* <p>This statistic is not persisted in manifests prior to v4, so it is generally only present on
* newly created or written files, and is null for files read back from such manifests.
+ *
+ * @deprecated since 1.13.0, will be removed in 2.0.0; use {@link #totalBytes()} instead.
*/
+ @Deprecated
default Map<Integer, Integer> avgValueSizes() {
- return null;
+ return Metrics.avgValueSizes(totalBytes(), valueCounts(), nullValueCounts());
}
/**
@@ -197,8 +211,7 @@ public interface ContentFile<F> {
* Copies this file without file stats. Manifest readers can reuse file instances; use this method
* to copy data without stats when collecting files.
*
- * @return a copy of this data file, without lower bounds, upper bounds, value counts, null value
- * counts, nan value counts, or average value sizes
+ * @return a copy of this data file without column stats
*/
F copyWithoutStats();
@@ -208,8 +221,7 @@ public interface ContentFile<F> {
* files.
*
* @param requestedColumnIds column IDs for which to keep stats.
- * @return a copy of data file, with lower bounds, upper bounds, value counts, null value counts,
- * nan value counts, and average value sizes for only specific columns.
+ * @return a copy of this data file with column stats for only the requested columns
*/
default F copyWithStats(Set<Integer> requestedColumnIds) {
throw new UnsupportedOperationException(
@@ -221,9 +233,8 @@ public interface ContentFile<F> {
* use this method to copy data when collecting files from tasks.
*
* @param withStats Will copy this file without file stats if set to <code>false</code>.
- * @return a copy of this data file. If <code>withStats</code> is set to <code>false</code> the
- * file will not contain lower bounds, upper bounds, value counts, null value counts, nan
- * value counts, or average value sizes
+ * @return a copy of this data file. If <code>withStats</code> is <code>false</code>, the copy
+ * will not contain column stats
*/
default F copy(boolean withStats) {
return withStats ? copy() : copyWithoutStats();
diff --git a/api/src/main/java/org/apache/iceberg/Metrics.java b/api/src/main/java/org/apache/iceberg/Metrics.java
index ce16c68acb..90eebb92e4 100644
--- a/api/src/main/java/org/apache/iceberg/Metrics.java
+++ b/api/src/main/java/org/apache/iceberg/Metrics.java
@@ -38,7 +38,7 @@ public class Metrics implements Serializable {
private Map<Integer, Long> nanValueCounts = null;
private Map<Integer, ByteBuffer> lowerBounds = null;
private Map<Integer, ByteBuffer> upperBounds = null;
- private Map<Integer, Integer> avgValueSizes = null;
+ private Map<Integer, Long> totalBytes = null;
// this is not serialized with all the other fields
private Map<Integer, Type> originalTypes = null;
@@ -135,7 +135,7 @@ public class Metrics implements Serializable {
nanValueCounts,
lowerBounds,
upperBounds,
- null /* avgValueSizes */,
+ null /* totalBytes */,
originalTypes);
}
@@ -150,8 +150,8 @@ public class Metrics implements Serializable {
* @param nanValueCounts a map of field id to the number of NaN values, or null if unknown
* @param lowerBounds a map of field id to the lower bound of the column, or null if unknown
* @param upperBounds a map of field id to the upper bound of the column, or null if unknown
- * @param avgValueSizes a map of field id to the average size in bytes of the column's non-null
- * values, or null if unknown
+ * @param totalBytes a map of field id to the total uncompressed size in bytes of the column's
+ * non-null values, or null if unknown
* @param originalTypes a map of field id to the original type of the lower/upper bound, or null
* if unknown
*/
@@ -163,7 +163,7 @@ public class Metrics implements Serializable {
Map<Integer, Long> nanValueCounts,
Map<Integer, ByteBuffer> lowerBounds,
Map<Integer, ByteBuffer> upperBounds,
- Map<Integer, Integer> avgValueSizes,
+ Map<Integer, Long> totalBytes,
Map<Integer, Type> originalTypes) {
this.rowCount = rowCount;
this.columnSizes = columnSizes;
@@ -172,7 +172,7 @@ public class Metrics implements Serializable {
this.nanValueCounts = nanValueCounts;
this.lowerBounds = lowerBounds;
this.upperBounds = upperBounds;
- this.avgValueSizes = avgValueSizes;
+ this.totalBytes = totalBytes;
this.originalTypes = originalTypes;
}
@@ -244,14 +244,50 @@ public class Metrics implements Serializable {
return upperBounds;
}
+ /**
+ * Get the total uncompressed size in memory in bytes of non-null values, for all fields where it
+ * was collected.
+ *
+ * @return a Map of fieldId to total uncompressed size in bytes
+ */
+ public Map<Integer, Long> totalBytes() {
+ return totalBytes;
+ }
+
/**
* Get the average value size in memory (uncompressed) in bytes over non-null values, for all
* fields where it was collected.
*
* @return a Map of fieldId to average value size in bytes
+ * @deprecated since 1.13.0, will be removed in 2.0.0; use {@link #totalBytes()} instead.
*/
+ @Deprecated
public Map<Integer, Integer> avgValueSizes() {
- return avgValueSizes;
+ return avgValueSizes(totalBytes, valueCounts, nullValueCounts);
+ }
+
+ static Map<Integer, Integer> avgValueSizes(
+ Map<Integer, Long> totalBytes,
+ Map<Integer, Long> valueCounts,
+ Map<Integer, Long> nullValueCounts) {
+ if (totalBytes == null || valueCounts == null) {
+ return null;
+ }
+
+ Map<Integer, Integer> avgValueSizes = Maps.newHashMapWithExpectedSize(totalBytes.size());
+ for (Map.Entry<Integer, Long> entry : totalBytes.entrySet()) {
+ Long total = entry.getValue();
+ Long valueCount = valueCounts.get(entry.getKey());
+ Long nullValueCount = nullValueCounts != null ? nullValueCounts.get(entry.getKey()) : null;
+ if (total != null && valueCount != null) {
+ long nonNullValueCount = valueCount - (nullValueCount != null ? nullValueCount : 0L);
+ if (nonNullValueCount > 0) {
+ avgValueSizes.put(entry.getKey(), Math.toIntExact(total / nonNullValueCount));
+ }
+ }
+ }
+
+ return avgValueSizes.isEmpty() ? null : avgValueSizes;
}
/**
@@ -278,7 +314,7 @@ public class Metrics implements Serializable {
writeByteBufferMap(out, lowerBounds);
writeByteBufferMap(out, upperBounds);
- out.writeObject(avgValueSizes);
+ out.writeObject(totalBytes);
}
private static void writeByteBufferMap(
@@ -315,7 +351,7 @@ public class Metrics implements Serializable {
lowerBounds = readByteBufferMap(in);
upperBounds = readByteBufferMap(in);
- avgValueSizes = (Map<Integer, Integer>) in.readObject();
+ totalBytes = (Map<Integer, Long>) in.readObject();
}
@SuppressWarnings("DangerousJavaDeserialization")
diff --git a/api/src/test/java/org/apache/iceberg/TestMetricsSerialization.java b/api/src/test/java/org/apache/iceberg/TestMetricsSerialization.java
index da482d028c..d16a6ba284 100644
--- a/api/src/test/java/org/apache/iceberg/TestMetricsSerialization.java
+++ b/api/src/test/java/org/apache/iceberg/TestMetricsSerialization.java
@@ -93,9 +93,9 @@ public class TestMetricsSerialization {
Map<Integer, Type> originalTypes =
ImmutableMap.of(1, Types.IntegerType.get(), 2, Types.IntegerType.get());
- Map<Integer, Integer> avgValueSizes = ImmutableMap.of(9, 10);
+ Map<Integer, Long> totalBytes = ImmutableMap.of(9, 10L);
return new Metrics(
- 0L, longMap1, longMap2, longMap3, null, byteMap1, byteMap2, avgValueSizes, originalTypes);
+ 0L, longMap1, longMap2, longMap3, null, byteMap1, byteMap2, totalBytes, originalTypes);
}
private static Metrics generateMetricsWithNulls() {
@@ -108,11 +108,11 @@ public class TestMetricsSerialization {
byteMap.put(4, null);
Map<Integer, Type> originalTypes = ImmutableMap.of(4, Types.IntegerType.get());
- Map<Integer, Integer> avgValueSizes = Maps.newHashMap();
- avgValueSizes.put(null, 1);
- avgValueSizes.put(2, null);
+ Map<Integer, Long> totalBytes = Maps.newHashMap();
+ totalBytes.put(null, 1L);
+ totalBytes.put(2, null);
return new Metrics(
- null, null, longMap, longMap, null, null, byteMap, avgValueSizes, originalTypes);
+ null, null, longMap, longMap, null, null, byteMap, totalBytes, originalTypes);
}
private static void assertEquals(Metrics expected, Metrics actual) {
@@ -120,7 +120,7 @@ public class TestMetricsSerialization {
assertThat(actual.columnSizes()).isEqualTo(expected.columnSizes());
assertThat(actual.valueCounts()).isEqualTo(expected.valueCounts());
assertThat(actual.nullValueCounts()).isEqualTo(expected.nullValueCounts());
- assertThat(actual.avgValueSizes()).isEqualTo(expected.avgValueSizes());
+ assertThat(actual.totalBytes()).isEqualTo(expected.totalBytes());
assertEquals(expected.lowerBounds(), actual.lowerBounds());
assertEquals(expected.upperBounds(), actual.upperBounds());
diff --git a/core/src/main/java/org/apache/iceberg/BaseFile.java b/core/src/main/java/org/apache/iceberg/BaseFile.java
index f59ddadfdc..8d94fd89be 100644
--- a/core/src/main/java/org/apache/iceberg/BaseFile.java
+++ b/core/src/main/java/org/apache/iceberg/BaseFile.java
@@ -70,7 +70,7 @@ abstract class BaseFile<F> extends SupportsIndexProjection
private Map<Integer, Long> nanValueCounts = null;
private Map<Integer, ByteBuffer> lowerBounds = null;
private Map<Integer, ByteBuffer> upperBounds = null;
- private Map<Integer, Integer> avgValueSizes = null;
+ private Map<Integer, Long> totalBytes = null;
private long[] splitOffsets = null;
private int[] equalityIds = null;
private byte[] keyMetadata = null;
@@ -149,7 +149,7 @@ abstract class BaseFile<F> extends SupportsIndexProjection
Map<Integer, Long> nanValueCounts,
Map<Integer, ByteBuffer> lowerBounds,
Map<Integer, ByteBuffer> upperBounds,
- Map<Integer, Integer> avgValueSizes,
+ Map<Integer, Long> totalBytes,
List<Long> splitOffsets,
int[] equalityFieldIds,
Integer sortOrderId,
@@ -182,7 +182,7 @@ abstract class BaseFile<F> extends SupportsIndexProjection
this.nanValueCounts = nanValueCounts;
this.lowerBounds = SerializableByteBufferMap.wrap(lowerBounds);
this.upperBounds = SerializableByteBufferMap.wrap(upperBounds);
- this.avgValueSizes = avgValueSizes;
+ this.totalBytes = totalBytes;
this.splitOffsets = ArrayUtil.toLongArray(splitOffsets);
this.equalityIds = equalityFieldIds;
this.sortOrderId = sortOrderId;
@@ -220,7 +220,7 @@ abstract class BaseFile<F> extends SupportsIndexProjection
this.nanValueCounts = copyMap(toCopy.nanValueCounts, requestedColumnIds);
this.lowerBounds = copyByteBufferMap(toCopy.lowerBounds, requestedColumnIds);
this.upperBounds = copyByteBufferMap(toCopy.upperBounds, requestedColumnIds);
- this.avgValueSizes = copyAvgValueSizes(toCopy.avgValueSizes, requestedColumnIds);
+ this.totalBytes = copyTotalBytes(toCopy.totalBytes, requestedColumnIds);
} else {
this.columnSizes = null;
this.valueCounts = null;
@@ -228,7 +228,7 @@ abstract class BaseFile<F> extends SupportsIndexProjection
this.nanValueCounts = null;
this.lowerBounds = null;
this.upperBounds = null;
- this.avgValueSizes = null;
+ this.totalBytes = null;
}
this.keyMetadata =
toCopy.keyMetadata == null
@@ -527,8 +527,8 @@ abstract class BaseFile<F> extends SupportsIndexProjection
}
@Override
- public Map<Integer, Integer> avgValueSizes() {
- return toReadableMap(avgValueSizes);
+ public Map<Integer, Long> totalBytes() {
+ return toReadableMap(totalBytes);
}
@Override
@@ -587,9 +587,8 @@ abstract class BaseFile<F> extends SupportsIndexProjection
return keys == null ? SerializableMap.copyOf(map) : SerializableMap.filteredCopyOf(map, keys);
}
- private static Map<Integer, Integer> copyAvgValueSizes(
- Map<Integer, Integer> map, Set<Integer> keys) {
- Map<Integer, Integer> copy = copyMap(map, keys);
+ private static Map<Integer, Long> copyTotalBytes(Map<Integer, Long> map, Set<Integer> keys) {
+ Map<Integer, Long> copy = copyMap(map, keys);
return copy == null || copy.isEmpty() ? null : copy;
}
@@ -666,7 +665,7 @@ abstract class BaseFile<F> extends SupportsIndexProjection
.add("nan_value_counts", nanValueCounts)
.add("lower_bounds", lowerBounds)
.add("upper_bounds", upperBounds)
- .add("avg_value_sizes", avgValueSizes)
+ .add("total_bytes", totalBytes)
.add("key_metadata", keyMetadata == null ? "null" : "(redacted)")
.add("split_offsets", splitOffsets == null ? "null" : splitOffsets())
.add("equality_ids", equalityIds == null ? "null" : equalityFieldIds())
diff --git a/core/src/main/java/org/apache/iceberg/ContentFileParser.java b/core/src/main/java/org/apache/iceberg/ContentFileParser.java
index c2edeca5c2..200f740d24 100644
--- a/core/src/main/java/org/apache/iceberg/ContentFileParser.java
+++ b/core/src/main/java/org/apache/iceberg/ContentFileParser.java
@@ -46,7 +46,7 @@ public class ContentFileParser {
private static final String LOWER_BOUNDS = "lower-bounds";
private static final String UPPER_BOUNDS = "upper-bounds";
private static final String CONTENT_STATS = "content-stats";
- private static final String AVG_VALUE_SIZE_IN_BYTES = "avg-value-size-in-bytes";
+ private static final String TOTAL_BYTES = "total-bytes";
private static final String KEY_METADATA = "key-metadata";
private static final String SPLIT_OFFSETS = "split-offsets";
private static final String EQUALITY_IDS = "equality-ids";
@@ -245,12 +245,12 @@ public class ContentFileParser {
SingleValueParser.toJson(DataFile.UPPER_BOUNDS.type(), contentFile.upperBounds(), generator);
}
- if (contentFile.avgValueSizes() != null) {
+ if (contentFile.totalBytes() != null) {
generator.writeFieldName(CONTENT_STATS);
generator.writeStartObject();
- for (Map.Entry<Integer, Integer> entry : contentFile.avgValueSizes().entrySet()) {
+ for (Map.Entry<Integer, Long> entry : contentFile.totalBytes().entrySet()) {
generator.writeObjectFieldStart(String.valueOf(entry.getKey()));
- generator.writeNumberField(AVG_VALUE_SIZE_IN_BYTES, entry.getValue());
+ generator.writeNumberField(TOTAL_BYTES, entry.getValue());
generator.writeEndObject();
}
@@ -305,9 +305,9 @@ public class ContentFileParser {
SingleValueParser.fromJson(DataFile.UPPER_BOUNDS.type(), jsonNode.get(UPPER_BOUNDS));
}
- Map<Integer, Integer> avgValueSizes = null;
+ Map<Integer, Long> totalBytes = null;
if (jsonNode.hasNonNull(CONTENT_STATS)) {
- avgValueSizes = avgValueSizesFromJson(jsonNode.get(CONTENT_STATS));
+ totalBytes = totalBytesFromJson(jsonNode.get(CONTENT_STATS));
}
return new Metrics(
@@ -318,17 +318,17 @@ public class ContentFileParser {
nanValueCounts,
lowerBounds,
upperBounds,
- avgValueSizes,
+ totalBytes,
null /* originalTypes */);
}
- private static Map<Integer, Integer> avgValueSizesFromJson(JsonNode contentStats) {
+ private static Map<Integer, Long> totalBytesFromJson(JsonNode contentStats) {
Preconditions.checkArgument(
contentStats.isObject(),
"Invalid JSON node for content stats: non-object (%s)",
contentStats);
- Map<Integer, Integer> avgValueSizes = Maps.newHashMap();
+ Map<Integer, Long> fieldIdToTotalBytes = Maps.newHashMap();
Iterator<String> fieldIds = contentStats.fieldNames();
while (fieldIds.hasNext()) {
String fieldId = fieldIds.next();
@@ -338,13 +338,13 @@ public class ContentFileParser {
"Cannot parse content stats for field %s from non-object: %s",
fieldId,
fieldStats);
- Integer avgValueSize = JsonUtil.getIntOrNull(AVG_VALUE_SIZE_IN_BYTES, fieldStats);
- if (avgValueSize != null) {
- avgValueSizes.put(parseFieldId(fieldId), avgValueSize);
+ Long totalBytes = JsonUtil.getLongOrNull(TOTAL_BYTES, fieldStats);
+ if (totalBytes != null) {
+ fieldIdToTotalBytes.put(parseFieldId(fieldId), totalBytes);
}
}
- return avgValueSizes.isEmpty() ? null : avgValueSizes;
+ return fieldIdToTotalBytes.isEmpty() ? null : fieldIdToTotalBytes;
}
private static int parseFieldId(String fieldId) {
diff --git a/core/src/main/java/org/apache/iceberg/ContentStatsBackedMap.java b/core/src/main/java/org/apache/iceberg/ContentStatsBackedMap.java
index c8a1c21e85..a6447a525d 100644
--- a/core/src/main/java/org/apache/iceberg/ContentStatsBackedMap.java
+++ b/core/src/main/java/org/apache/iceberg/ContentStatsBackedMap.java
@@ -38,7 +38,7 @@ class ContentStatsBackedMap<V> extends AbstractMap<Integer, V> {
NAN_VALUE_COUNT,
LOWER_BOUND,
UPPER_BOUND,
- AVG_VALUE_SIZE
+ TOTAL_BYTES
}
/** Per-column value counts, or null if no column tracks the value count. */
@@ -66,9 +66,9 @@ class ContentStatsBackedMap<V> extends AbstractMap<Integer, V> {
return viewOrNull(stats, Kind.UPPER_BOUND);
}
- /** Per-column average non-null value sizes, or null if no column tracks the average size. */
- static <V> Map<Integer, V> avgValueSizes(ContentStats stats) {
- return viewOrNull(stats, Kind.AVG_VALUE_SIZE);
+ /** Per-column total uncompressed size of non-null values, or null if no column tracks it. */
+ static <V> Map<Integer, V> totalBytes(ContentStats stats) {
+ return viewOrNull(stats, Kind.TOTAL_BYTES);
}
private final ContentStats stats;
@@ -151,7 +151,7 @@ class ContentStatsBackedMap<V> extends AbstractMap<Integer, V> {
case NAN_VALUE_COUNT -> fieldStats.hasNanValueCount();
case LOWER_BOUND -> fieldStats.lowerBound() != null;
case UPPER_BOUND -> fieldStats.upperBound() != null;
- case AVG_VALUE_SIZE -> fieldStats.avgValueSizeInBytes() != null;
+ case TOTAL_BYTES -> fieldStats.hasTotalBytes();
};
}
@@ -168,7 +168,8 @@ class ContentStatsBackedMap<V> extends AbstractMap<Integer, V> {
(V) bound(fieldStats, fieldStats.lowerBound(), StatsUtil.LOWER_BOUND_NAME);
case UPPER_BOUND ->
(V) bound(fieldStats, fieldStats.upperBound(), StatsUtil.UPPER_BOUND_NAME);
- case AVG_VALUE_SIZE -> (V) fieldStats.avgValueSizeInBytes();
+ case TOTAL_BYTES ->
+ fieldStats.hasTotalBytes() ? (V) Long.valueOf(fieldStats.totalBytes()) : null;
};
}
diff --git a/core/src/main/java/org/apache/iceberg/DataFiles.java b/core/src/main/java/org/apache/iceberg/DataFiles.java
index 9748e8f868..9602cc6206 100644
--- a/core/src/main/java/org/apache/iceberg/DataFiles.java
+++ b/core/src/main/java/org/apache/iceberg/DataFiles.java
@@ -152,7 +152,7 @@ public class DataFiles {
private Map<Integer, Long> nanValueCounts = null;
private Map<Integer, ByteBuffer> lowerBounds = null;
private Map<Integer, ByteBuffer> upperBounds = null;
- private Map<Integer, Integer> avgValueSizes = null;
+ private Map<Integer, Long> totalBytes = null;
private Map<Integer, Type> originalTypes = null;
private ByteBuffer keyMetadata = null;
private List<Long> splitOffsets = null;
@@ -180,7 +180,7 @@ public class DataFiles {
this.nanValueCounts = null;
this.lowerBounds = null;
this.upperBounds = null;
- this.avgValueSizes = null;
+ this.totalBytes = null;
this.splitOffsets = null;
this.sortOrderId = SortOrder.unsorted().orderId();
this.firstRowId = null;
@@ -202,7 +202,7 @@ public class DataFiles {
this.nanValueCounts = toCopy.nanValueCounts();
this.lowerBounds = toCopy.lowerBounds();
this.upperBounds = toCopy.upperBounds();
- this.avgValueSizes = toCopy.avgValueSizes();
+ this.totalBytes = toCopy.totalBytes();
this.keyMetadata =
toCopy.keyMetadata() == null ? null : ByteBuffers.copy(toCopy.keyMetadata());
this.splitOffsets =
@@ -295,7 +295,7 @@ public class DataFiles {
this.nanValueCounts = metrics.nanValueCounts();
this.lowerBounds = metrics.lowerBounds();
this.upperBounds = metrics.upperBounds();
- this.avgValueSizes = metrics.avgValueSizes();
+ this.totalBytes = metrics.totalBytes();
this.originalTypes = metrics.originalTypes();
return this;
}
@@ -358,7 +358,7 @@ public class DataFiles {
nanValueCounts,
lowerBounds,
upperBounds,
- avgValueSizes,
+ totalBytes,
originalTypes),
keyMetadata,
splitOffsets,
diff --git a/core/src/main/java/org/apache/iceberg/Delegates.java b/core/src/main/java/org/apache/iceberg/Delegates.java
index d89e9ded7f..2bd7e4dfeb 100644
--- a/core/src/main/java/org/apache/iceberg/Delegates.java
+++ b/core/src/main/java/org/apache/iceberg/Delegates.java
@@ -233,8 +233,8 @@ class Delegates {
}
@Override
- public Map<Integer, Integer> avgValueSizes() {
- return wrapped.avgValueSizes();
+ public Map<Integer, Long> totalBytes() {
+ return wrapped.totalBytes();
}
@Override
diff --git a/core/src/main/java/org/apache/iceberg/FieldMetrics.java b/core/src/main/java/org/apache/iceberg/FieldMetrics.java
index d037e71047..ce7a631986 100644
--- a/core/src/main/java/org/apache/iceberg/FieldMetrics.java
+++ b/core/src/main/java/org/apache/iceberg/FieldMetrics.java
@@ -26,6 +26,7 @@ public class FieldMetrics<T> {
private final long valueCount;
private final long nullValueCount;
private final long nanValueCount;
+ private final long totalBytes;
private final T lowerBound;
private final T upperBound;
private final Type originalType;
@@ -66,7 +67,33 @@ public class FieldMetrics<T> {
T lowerBound,
T upperBound,
Type originalType) {
- this(id, valueCount, nullValueCount, nanValueCount, lowerBound, upperBound, originalType, null);
+ this(id, valueCount, nullValueCount, nanValueCount, lowerBound, upperBound, originalType, -1L);
+ }
+
+ /**
+ * @deprecated since 1.13.0, will be removed in 1.14.0; use the constructor that takes {@code
+ * long} totalBytes instead.
+ */
+ @Deprecated
+ public FieldMetrics(
+ int id,
+ long valueCount,
+ long nullValueCount,
+ long nanValueCount,
+ T lowerBound,
+ T upperBound,
+ Type originalType,
+ Integer avgValueSizeInBytes) {
+ this(
+ id,
+ valueCount,
+ nullValueCount,
+ nanValueCount,
+ lowerBound,
+ upperBound,
+ originalType,
+ -1L,
+ avgValueSizeInBytes);
}
public FieldMetrics(
@@ -77,6 +104,28 @@ public class FieldMetrics<T> {
T lowerBound,
T upperBound,
Type originalType,
+ long totalBytes) {
+ this(
+ id,
+ valueCount,
+ nullValueCount,
+ nanValueCount,
+ lowerBound,
+ upperBound,
+ originalType,
+ totalBytes,
+ null);
+ }
+
+ private FieldMetrics(
+ int id,
+ long valueCount,
+ long nullValueCount,
+ long nanValueCount,
+ T lowerBound,
+ T upperBound,
+ Type originalType,
+ long totalBytes,
Integer avgValueSizeInBytes) {
this.id = id;
this.valueCount = valueCount;
@@ -85,6 +134,7 @@ public class FieldMetrics<T> {
this.lowerBound = lowerBound;
this.upperBound = upperBound;
this.originalType = originalType;
+ this.totalBytes = totalBytes;
this.avgValueSizeInBytes = avgValueSizeInBytes;
}
@@ -126,9 +176,26 @@ public class FieldMetrics<T> {
return originalType;
}
- /** Returns the average size in bytes over non-null values, or null if it is not known. */
+ /** Returns the total uncompressed size in bytes of non-null values. */
+ public long totalBytes() {
+ return totalBytes;
+ }
+
+ /**
+ * Returns the average size in bytes over non-null values, or null if it is not known.
+ *
+ * @deprecated since 1.13.0, will be removed in 1.14.0; use {@link #totalBytes()} instead.
+ */
+ @Deprecated
public Integer avgValueSizeInBytes() {
- return avgValueSizeInBytes;
+ if (avgValueSizeInBytes != null) {
+ return avgValueSizeInBytes;
+ }
+
+ long nonNullValueCount = valueCount - nullValueCount;
+ return totalBytes >= 0 && nonNullValueCount > 0
+ ? Math.toIntExact(totalBytes / nonNullValueCount)
+ : null;
}
/** Returns if the metrics has bounds (i.e. there is at least non-null value for this field) */
diff --git a/core/src/main/java/org/apache/iceberg/FieldStats.java b/core/src/main/java/org/apache/iceberg/FieldStats.java
index ddbbc4d77c..4edbe4f8c0 100644
--- a/core/src/main/java/org/apache/iceberg/FieldStats.java
+++ b/core/src/main/java/org/apache/iceberg/FieldStats.java
@@ -61,11 +61,35 @@ public interface FieldStats<T> {
/** The total NaN value count, defined only when {@link #hasNanValueCount()}. */
long nanValueCount();
+ /** Whether total bytes are tracked for this field. */
+ default boolean hasTotalBytes() {
+ return false;
+ }
+
+ /**
+ * The total uncompressed size in memory in bytes of non-null values, for variable-length types,
+ * defined only when {@link #hasTotalBytes()}.
+ */
+ default long totalBytes() {
+ return 0L;
+ }
+
/**
* The avg value size in memory (uncompressed) in bytes for variable-length types (string, binary,
* variant, geometry, geography)
+ *
+ * @deprecated since 1.13.0, will be removed in 1.14.0; use {@link #totalBytes()} instead.
*/
- Integer avgValueSizeInBytes();
+ @Deprecated
+ default Integer avgValueSizeInBytes() {
+ if (!hasTotalBytes() || !hasValueCount()) {
+ return null;
+ }
+
+ long nulls = hasNullValueCount() ? nullValueCount() : 0L;
+ long nonNullValueCount = valueCount() - nulls;
+ return nonNullValueCount > 0 ? Math.toIntExact(totalBytes() / nonNullValueCount) : null;
+ }
/** Returns a copy of this {@link FieldStats}. */
FieldStats<T> copy();
diff --git a/core/src/main/java/org/apache/iceberg/FieldStatsStruct.java b/core/src/main/java/org/apache/iceberg/FieldStatsStruct.java
index bb3aa1a556..084cfb9aa6 100644
--- a/core/src/main/java/org/apache/iceberg/FieldStatsStruct.java
+++ b/core/src/main/java/org/apache/iceberg/FieldStatsStruct.java
@@ -40,7 +40,7 @@ class FieldStatsStruct<T> implements FieldStats<T>, StructLike, Serializable {
private Long valueCount = null;
private Long nullValueCount = null;
private Long nanValueCount = null;
- private Integer avgValueSize = null;
+ private Long totalBytes = null;
FieldStatsStruct(Types.StructType struct) {
this.struct = struct;
@@ -57,7 +57,7 @@ class FieldStatsStruct<T> implements FieldStats<T>, StructLike, Serializable {
long valueCount,
long nullValueCount,
long nanValueCount,
- Integer avgValueSize) {
+ long totalBytes) {
this(struct);
setLowerBound(lowerBound);
setUpperBound(upperBound);
@@ -65,7 +65,7 @@ class FieldStatsStruct<T> implements FieldStats<T>, StructLike, Serializable {
this.valueCount = valueCount;
this.nullValueCount = nullValueCount;
this.nanValueCount = nanValueCount;
- this.avgValueSize = avgValueSize;
+ this.totalBytes = totalBytes;
}
private FieldStatsStruct(FieldStatsStruct<T> toCopy) {
@@ -76,7 +76,7 @@ class FieldStatsStruct<T> implements FieldStats<T>, StructLike, Serializable {
this.valueCount = toCopy.valueCount;
this.nullValueCount = toCopy.nullValueCount;
this.nanValueCount = toCopy.nanValueCount;
- this.avgValueSize = toCopy.avgValueSize;
+ this.totalBytes = toCopy.totalBytes;
}
void fromFieldMetrics(FieldMetrics<T> fieldMetrics) {
@@ -92,7 +92,7 @@ class FieldStatsStruct<T> implements FieldStats<T>, StructLike, Serializable {
this.valueCount = fieldMetrics.valueCount();
this.nullValueCount = fieldMetrics.nullValueCount() < 0 ? null : fieldMetrics.nullValueCount();
this.nanValueCount = fieldMetrics.nanValueCount() < 0 ? null : fieldMetrics.nanValueCount();
- this.avgValueSize = fieldMetrics.avgValueSizeInBytes();
+ this.totalBytes = fieldMetrics.totalBytes() < 0 ? null : fieldMetrics.totalBytes();
}
private boolean isBinary() {
@@ -168,8 +168,13 @@ class FieldStatsStruct<T> implements FieldStats<T>, StructLike, Serializable {
}
@Override
- public Integer avgValueSizeInBytes() {
- return avgValueSize;
+ public boolean hasTotalBytes() {
+ return totalBytes != null;
+ }
+
+ @Override
+ public long totalBytes() {
+ return totalBytes;
}
@Override
@@ -185,7 +190,7 @@ class FieldStatsStruct<T> implements FieldStats<T>, StructLike, Serializable {
case StatsUtil.VALUE_COUNT_OFFSET -> valueCount;
case StatsUtil.NULL_VALUE_COUNT_OFFSET -> nullValueCount;
case StatsUtil.NAN_VALUE_COUNT_OFFSET -> nanValueCount;
- case StatsUtil.AVG_VALUE_SIZE_OFFSET -> avgValueSize;
+ case StatsUtil.TOTAL_BYTES_OFFSET -> totalBytes;
default -> throw new UnsupportedOperationException("Unsupported stats offset: " + offset);
};
}
@@ -203,7 +208,7 @@ class FieldStatsStruct<T> implements FieldStats<T>, StructLike, Serializable {
case StatsUtil.VALUE_COUNT_OFFSET -> this.valueCount = (Long) value;
case StatsUtil.NULL_VALUE_COUNT_OFFSET -> this.nullValueCount = (Long) value;
case StatsUtil.NAN_VALUE_COUNT_OFFSET -> this.nanValueCount = (Long) value;
- case StatsUtil.AVG_VALUE_SIZE_OFFSET -> this.avgValueSize = (Integer) value;
+ case StatsUtil.TOTAL_BYTES_OFFSET -> this.totalBytes = (Long) value;
default -> throw new UnsupportedOperationException("Unsupported stats offset: " + offset);
}
}
diff --git a/core/src/main/java/org/apache/iceberg/FileMetadata.java b/core/src/main/java/org/apache/iceberg/FileMetadata.java
index d8e8022482..aacb3aaa5c 100644
--- a/core/src/main/java/org/apache/iceberg/FileMetadata.java
+++ b/core/src/main/java/org/apache/iceberg/FileMetadata.java
@@ -57,7 +57,7 @@ public class FileMetadata {
private Map<Integer, Long> nanValueCounts = null;
private Map<Integer, ByteBuffer> lowerBounds = null;
private Map<Integer, ByteBuffer> upperBounds = null;
- private Map<Integer, Integer> avgValueSizes = null;
+ private Map<Integer, Long> totalBytes = null;
private Map<Integer, Type> originalTypes = null;
private ByteBuffer keyMetadata = null;
private Integer sortOrderId = null;
@@ -87,7 +87,7 @@ public class FileMetadata {
this.nanValueCounts = null;
this.lowerBounds = null;
this.upperBounds = null;
- this.avgValueSizes = null;
+ this.totalBytes = null;
this.sortOrderId = null;
}
@@ -108,7 +108,7 @@ public class FileMetadata {
this.nanValueCounts = toCopy.nanValueCounts();
this.lowerBounds = toCopy.lowerBounds();
this.upperBounds = toCopy.upperBounds();
- this.avgValueSizes = toCopy.avgValueSizes();
+ this.totalBytes = toCopy.totalBytes();
this.keyMetadata =
toCopy.keyMetadata() == null ? null : ByteBuffers.copy(toCopy.keyMetadata());
this.sortOrderId = toCopy.sortOrderId();
@@ -205,7 +205,7 @@ public class FileMetadata {
this.nanValueCounts = metrics.nanValueCounts();
this.lowerBounds = metrics.lowerBounds();
this.upperBounds = metrics.upperBounds();
- this.avgValueSizes = metrics.avgValueSizes();
+ this.totalBytes = metrics.totalBytes();
this.originalTypes = metrics.originalTypes();
return this;
}
@@ -304,7 +304,7 @@ public class FileMetadata {
nanValueCounts,
lowerBounds,
upperBounds,
- avgValueSizes,
+ totalBytes,
originalTypes),
equalityFieldIds,
sortOrderId,
diff --git a/core/src/main/java/org/apache/iceberg/GenericDataFile.java b/core/src/main/java/org/apache/iceberg/GenericDataFile.java
index 3a7c1d7200..78c239313c 100644
--- a/core/src/main/java/org/apache/iceberg/GenericDataFile.java
+++ b/core/src/main/java/org/apache/iceberg/GenericDataFile.java
@@ -62,7 +62,7 @@ class GenericDataFile extends BaseFile<DataFile> implements DataFile {
metrics.nanValueCounts(),
metrics.lowerBounds(),
metrics.upperBounds(),
- metrics.avgValueSizes(),
+ metrics.totalBytes(),
splitOffsets,
null /* no equality field IDs */,
sortOrderId,
diff --git a/core/src/main/java/org/apache/iceberg/GenericDeleteFile.java b/core/src/main/java/org/apache/iceberg/GenericDeleteFile.java
index de732eb9d5..d3aafc2922 100644
--- a/core/src/main/java/org/apache/iceberg/GenericDeleteFile.java
+++ b/core/src/main/java/org/apache/iceberg/GenericDeleteFile.java
@@ -66,7 +66,7 @@ class GenericDeleteFile extends BaseFile<DeleteFile> implements DeleteFile {
metrics.nanValueCounts(),
metrics.lowerBounds(),
metrics.upperBounds(),
- metrics.avgValueSizes(),
+ metrics.totalBytes(),
splitOffsets,
equalityFieldIds,
sortOrderId,
diff --git a/core/src/main/java/org/apache/iceberg/GeometryFieldMetrics.java b/core/src/main/java/org/apache/iceberg/GeometryFieldMetrics.java
index 6a62b140bd..f58757474f 100644
--- a/core/src/main/java/org/apache/iceberg/GeometryFieldMetrics.java
+++ b/core/src/main/java/org/apache/iceberg/GeometryFieldMetrics.java
@@ -33,9 +33,9 @@ import org.apache.iceberg.types.Type;
* existing geometry conversion path. The null value count is left at zero here; it is reconciled by
* the optional-field writer that owns null accounting.
*
- * <p>The average WKB size over the non-null values is also tracked, matching the {@code
- * avg_value_size_in_bytes} metric that the counts-only geospatial writer produces, so adding bounds
- * does not drop that metric for generic Parquet geometry writes.
+ * <p>The total bytes of non-null WKB values is also tracked, matching the {@code total_bytes}
+ * metric that the counts-only geospatial writer produces, so adding bounds does not drop that
+ * metric for generic Parquet geometry writes.
*/
public class GeometryFieldMetrics extends FieldMetrics<GeospatialBound> {
@@ -45,8 +45,8 @@ public class GeometryFieldMetrics extends FieldMetrics<GeospatialBound> {
GeospatialBound lowerBound,
GeospatialBound upperBound,
Type type,
- Integer avgValueSizeInBytes) {
- super(id, valueCount, 0L, -1L, lowerBound, upperBound, type, avgValueSizeInBytes);
+ long totalBytes) {
+ super(id, valueCount, 0L, -1L, lowerBound, upperBound, type, totalBytes);
}
public static class Builder {
@@ -69,15 +69,13 @@ public class GeometryFieldMetrics extends FieldMetrics<GeospatialBound> {
public GeometryFieldMetrics build() {
// build() returns null bounds when a dimension has no value or a value could not be bounded,
- // which leaves the field with counts but no bounds. The average size is still reported: it
- // does not depend on whether a box could be formed.
+ // which leaves the field with counts but no bounds. The total bytes are still reported
+ // and do not depend on whether a box could be formed.
BoundingBox box = boundsBuilder.build();
GeospatialBound lowerBound = box == null ? null : box.min();
GeospatialBound upperBound = box == null ? null : box.max();
- Integer avgValueSizeInBytes =
- valueCount > 0 ? Math.toIntExact(totalValueSizeInBytes / valueCount) : null;
return new GeometryFieldMetrics(
- id, valueCount, lowerBound, upperBound, geometryType, avgValueSizeInBytes);
+ id, valueCount, lowerBound, upperBound, geometryType, totalValueSizeInBytes);
}
}
}
diff --git a/core/src/main/java/org/apache/iceberg/MetricsUtil.java b/core/src/main/java/org/apache/iceberg/MetricsUtil.java
index c93523aed4..fb1b5e9454 100644
--- a/core/src/main/java/org/apache/iceberg/MetricsUtil.java
+++ b/core/src/main/java/org/apache/iceberg/MetricsUtil.java
@@ -42,11 +42,10 @@ public class MetricsUtil {
private MetricsUtil() {}
/**
- * Copies a metrics object without value, NULL and NaN counts or average value sizes for given
- * fields.
+ * Copies a metrics object without value, NULL and NaN counts or total bytes for given fields.
*
- * @param excludedFieldIds field IDs for which the counts and average value sizes must be dropped
- * @return a new metrics object without counts or average value sizes for given fields
+ * @param excludedFieldIds field IDs for which the counts and total bytes must be dropped
+ * @return a new metrics object without counts or total bytes for given fields
*/
public static Metrics copyWithoutFieldCounts(Metrics metrics, Set<Integer> excludedFieldIds) {
return new Metrics(
@@ -57,15 +56,14 @@ public class MetricsUtil {
copyWithoutKeys(metrics.nanValueCounts(), excludedFieldIds),
metrics.lowerBounds(),
metrics.upperBounds(),
- copyWithoutKeys(metrics.avgValueSizes(), excludedFieldIds),
+ copyWithoutKeys(metrics.totalBytes(), excludedFieldIds),
metrics.originalTypes());
}
/**
- * Copies a metrics object without counts, average value sizes, and bounds for given fields.
+ * Copies a metrics object without counts, total bytes, and bounds for given fields.
*
- * @param excludedFieldIds field IDs for which the counts, average value sizes, and bounds must be
- * dropped
+ * @param excludedFieldIds field IDs for which the counts, total bytes, and bounds must be dropped
* @return a new metrics object without lower and upper bounds for given fields
*/
public static Metrics copyWithoutFieldCountsAndBounds(
@@ -78,7 +76,7 @@ public class MetricsUtil {
copyWithoutKeys(metrics.nanValueCounts(), excludedFieldIds),
copyWithoutKeys(metrics.lowerBounds(), excludedFieldIds),
copyWithoutKeys(metrics.upperBounds(), excludedFieldIds),
- copyWithoutKeys(metrics.avgValueSizes(), excludedFieldIds),
+ copyWithoutKeys(metrics.totalBytes(), excludedFieldIds),
copyWithoutKeys(metrics.originalTypes(), excludedFieldIds));
}
diff --git a/core/src/main/java/org/apache/iceberg/StatsUtil.java b/core/src/main/java/org/apache/iceberg/StatsUtil.java
index bef7cc41c1..c1a222fb21 100644
--- a/core/src/main/java/org/apache/iceberg/StatsUtil.java
+++ b/core/src/main/java/org/apache/iceberg/StatsUtil.java
@@ -53,7 +53,7 @@ class StatsUtil {
static final int VALUE_COUNT_OFFSET = 4;
static final int NULL_VALUE_COUNT_OFFSET = 5;
static final int NAN_VALUE_COUNT_OFFSET = 6;
- static final int AVG_VALUE_SIZE_OFFSET = 7;
+ static final int TOTAL_BYTES_OFFSET = 7;
static final String LOWER_BOUND_NAME = "lower_bound";
static final String UPPER_BOUND_NAME = "upper_bound";
@@ -305,9 +305,7 @@ class StatsUtil {
}
if (isVariableLength(type)) {
- fields.add(
- optional(
- baseId + AVG_VALUE_SIZE_OFFSET, "avg_value_size_in_bytes", Types.IntegerType.get()));
+ fields.add(optional(baseId + TOTAL_BYTES_OFFSET, "total_bytes", Types.LongType.get()));
}
return Types.StructType.of(fields);
diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java b/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java
index 2b351fc4bd..b53840ab56 100644
--- a/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java
+++ b/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java
@@ -189,8 +189,8 @@ class TrackedFileAdapters {
}
@Override
- public Map<Integer, Integer> avgValueSizes() {
- return ContentStatsBackedMap.avgValueSizes(file().contentStats());
+ public Map<Integer, Long> totalBytes() {
+ return ContentStatsBackedMap.totalBytes(file().contentStats());
}
@Override
diff --git a/core/src/main/java/org/apache/iceberg/ValueSizeFieldMetrics.java b/core/src/main/java/org/apache/iceberg/ValueSizeFieldMetrics.java
index dbd2ae7f50..f8332a285e 100644
--- a/core/src/main/java/org/apache/iceberg/ValueSizeFieldMetrics.java
+++ b/core/src/main/java/org/apache/iceberg/ValueSizeFieldMetrics.java
@@ -20,11 +20,11 @@ package org.apache.iceberg;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-/** Field-level metrics for tracking the average size of variable-length values. */
+/** Field-level metrics for tracking the total size of variable-length values. */
public class ValueSizeFieldMetrics extends FieldMetrics<Object> {
- private ValueSizeFieldMetrics(int id, long valueCount, Integer avgValueSizeInBytes) {
- super(id, valueCount, 0L, -1L, null, null, null, avgValueSizeInBytes);
+ private ValueSizeFieldMetrics(int id, long valueCount, long totalBytes) {
+ super(id, valueCount, 0L, -1L, null, null, null, totalBytes);
}
public static class Builder {
@@ -43,9 +43,7 @@ public class ValueSizeFieldMetrics extends FieldMetrics<Object> {
}
public ValueSizeFieldMetrics build() {
- Integer avgValueSizeInBytes =
- valueCount > 0 ? Math.toIntExact(totalValueSizeInBytes / valueCount) : null;
- return new ValueSizeFieldMetrics(id, valueCount, avgValueSizeInBytes);
+ return new ValueSizeFieldMetrics(id, valueCount, totalValueSizeInBytes);
}
}
}
diff --git a/core/src/main/java/org/apache/iceberg/util/ContentFileUtil.java b/core/src/main/java/org/apache/iceberg/util/ContentFileUtil.java
index aa7c13d6bf..52310e7427 100644
--- a/core/src/main/java/org/apache/iceberg/util/ContentFileUtil.java
+++ b/core/src/main/java/org/apache/iceberg/util/ContentFileUtil.java
@@ -176,7 +176,7 @@ public class ContentFileUtil {
file.nanValueCounts(),
lowerBounds == null ? null : Collections.unmodifiableMap(lowerBounds),
upperBounds == null ? null : Collections.unmodifiableMap(upperBounds),
- file.avgValueSizes(),
+ file.totalBytes(),
null /* originalTypes */);
}
@@ -200,7 +200,7 @@ public class ContentFileUtil {
file.nanValueCounts(),
lowerBounds == null ? null : Collections.unmodifiableMap(lowerBounds),
upperBounds == null ? null : Collections.unmodifiableMap(upperBounds),
- file.avgValueSizes(),
+ file.totalBytes(),
null /* originalTypes */);
}
}
diff --git a/core/src/test/java/org/apache/iceberg/StatsTestUtil.java b/core/src/test/java/org/apache/iceberg/StatsTestUtil.java
index a2bbf7bf53..9f06e17c4f 100644
--- a/core/src/test/java/org/apache/iceberg/StatsTestUtil.java
+++ b/core/src/test/java/org/apache/iceberg/StatsTestUtil.java
@@ -69,7 +69,7 @@ class StatsTestUtil {
validateTrackedMetric(type, "null_value_count", nullCount);
validateTrackedMetric(type, "nan_value_count", nanCount);
- return new TestFieldStats(fieldId, type, lower, upper, valueCount, nullCount, nanCount);
+ return new TestFieldStats(fieldId, type, lower, upper, valueCount, nullCount, nanCount, null);
}
private static void validateTrackedMetric(Types.StructType type, String metric, Object value) {
@@ -88,6 +88,7 @@ class StatsTestUtil {
private final Long valueCount;
private final Long nullValueCount;
private final Long nanValueCount;
+ private final Long totalBytes;
private TestFieldStats(
int fieldId,
@@ -96,7 +97,8 @@ class StatsTestUtil {
Object upperBound,
Long valueCount,
Long nullValueCount,
- Long nanValueCount) {
+ Long nanValueCount,
+ Long totalBytes) {
this.fieldId = fieldId;
this.type = type;
this.lowerBound = lowerBound;
@@ -104,6 +106,7 @@ class StatsTestUtil {
this.valueCount = valueCount;
this.nullValueCount = nullValueCount;
this.nanValueCount = nanValueCount;
+ this.totalBytes = totalBytes;
}
@Override
@@ -162,8 +165,13 @@ class StatsTestUtil {
}
@Override
- public Integer avgValueSizeInBytes() {
- return null;
+ public boolean hasTotalBytes() {
+ return totalBytes != null;
+ }
+
+ @Override
+ public long totalBytes() {
+ return totalBytes;
}
@Override
@@ -198,7 +206,7 @@ class StatsTestUtil {
Long valueCount,
Long nullCount,
Long nanCount,
- Integer avgValueSize) {
+ Long totalBytes) {
FieldStats<Object> stats = Mockito.mock(FieldStats.class);
Mockito.when(stats.fieldId()).thenReturn(id);
Mockito.when(stats.type()).thenReturn(type);
@@ -219,11 +227,10 @@ class StatsTestUtil {
Mockito.when(stats.nanValueCount()).thenReturn(nanCount);
}
- // stub unconditionally: unlike the counts, avgValueSizeInBytes has no has*() gate, so
- // ContentStatsBackedMap treats avgValueSizeInBytes() != null as the presence check. A Mockito
- // mock defaults this method to 0, not null, so an absent stat must be stubbed to null here or
- // it reads back as present with 0.
- Mockito.when(stats.avgValueSizeInBytes()).thenReturn(avgValueSize);
+ Mockito.when(stats.hasTotalBytes()).thenReturn(totalBytes != null);
+ if (totalBytes != null) {
+ Mockito.when(stats.totalBytes()).thenReturn(totalBytes);
+ }
return stats;
}
diff --git a/core/src/test/java/org/apache/iceberg/TestContentFileParser.java b/core/src/test/java/org/apache/iceberg/TestContentFileParser.java
index a4a8dd235a..f0ba23e7c2 100644
--- a/core/src/test/java/org/apache/iceberg/TestContentFileParser.java
+++ b/core/src/test/java/org/apache/iceberg/TestContentFileParser.java
@@ -90,7 +90,7 @@ public class TestContentFileParser {
ContentFile<?> contentFile =
ContentFileParser.fromJson(jsonNode, Map.of(0, PartitionSpec.unpartitioned()));
assertThat(contentFile).isInstanceOf(DataFile.class);
- assertThat(contentFile.avgValueSizes()).isNull();
+ assertThat(contentFile.totalBytes()).isNull();
}
@Test
@@ -98,7 +98,7 @@ public class TestContentFileParser {
String jsonStr =
"{\"spec-id\":0,\"content\":\"data\",\"file-path\":\"/path/to/data.parquet\","
+ "\"file-format\":\"parquet\",\"partition\":[],\"file-size-in-bytes\":10,"
- + "\"record-count\":1,\"content-stats\":{\"abc\":{\"avg-value-size-in-bytes\":8}}}";
+ + "\"record-count\":1,\"content-stats\":{\"abc\":{\"total-bytes\":8}}}";
JsonNode jsonNode = JsonUtil.mapper().readTree(jsonStr);
assertThatThrownBy(
() -> ContentFileParser.fromJson(jsonNode, Map.of(0, PartitionSpec.unpartitioned())))
@@ -384,7 +384,7 @@ public class TestContentFileParser {
+ "\"nan-value-counts\":{\"keys\":[3,4],\"values\":[0,0]},"
+ "\"lower-bounds\":{\"keys\":[3,4],\"values\":[\"01000000\",\"02000000\"]},"
+ "\"upper-bounds\":{\"keys\":[3,4],\"values\":[\"05000000\",\"0A000000\"]},"
- + "\"content-stats\":{\"3\":{\"avg-value-size-in-bytes\":8},\"4\":{\"avg-value-size-in-bytes\":16}},"
+ + "\"content-stats\":{\"3\":{\"total-bytes\":8},\"4\":{\"total-bytes\":16}},"
+ "\"key-metadata\":\"00000000000000000000000000000000\","
+ "\"split-offsets\":[128,256],\"sort-order-id\":1}";
} else {
@@ -396,7 +396,7 @@ public class TestContentFileParser {
+ "\"nan-value-counts\":{\"keys\":[3,4],\"values\":[0,0]},"
+ "\"lower-bounds\":{\"keys\":[3,4],\"values\":[\"01000000\",\"02000000\"]},"
+ "\"upper-bounds\":{\"keys\":[3,4],\"values\":[\"05000000\",\"0A000000\"]},"
- + "\"content-stats\":{\"3\":{\"avg-value-size-in-bytes\":8},\"4\":{\"avg-value-size-in-bytes\":16}},"
+ + "\"content-stats\":{\"3\":{\"total-bytes\":8},\"4\":{\"total-bytes\":16}},"
+ "\"key-metadata\":\"00000000000000000000000000000000\","
+ "\"split-offsets\":[128,256],\"sort-order-id\":1}";
}
@@ -423,7 +423,7 @@ public class TestContentFileParser {
Conversions.toByteBuffer(Types.IntegerType.get(), 5),
4,
Conversions.toByteBuffer(Types.IntegerType.get(), 10)), // upper bounds
- ImmutableMap.of(3, 8, 4, 16), // avg value sizes
+ ImmutableMap.of(3, 8L, 4, 16L), // total bytes
null /* originalTypes */))
.withFileSizeInBytes(350)
.withSplitOffsets(Arrays.asList(128L, 256L))
@@ -564,7 +564,7 @@ public class TestContentFileParser {
Conversions.toByteBuffer(Types.IntegerType.get(), 5),
4,
Conversions.toByteBuffer(Types.IntegerType.get(), 10)), // upper bounds
- ImmutableMap.of(3, 8, 4, 16), // avg value sizes
+ ImmutableMap.of(3, 8L, 4, 16L), // total bytes
null /* originalTypes */);
return new GenericDeleteFile(
@@ -604,7 +604,7 @@ public class TestContentFileParser {
+ "\"nan-value-counts\":{\"keys\":[3,4],\"values\":[0,0]},"
+ "\"lower-bounds\":{\"keys\":[3,4],\"values\":[\"01000000\",\"02000000\"]},"
+ "\"upper-bounds\":{\"keys\":[3,4],\"values\":[\"05000000\",\"0A000000\"]},"
- + "\"content-stats\":{\"3\":{\"avg-value-size-in-bytes\":8},\"4\":{\"avg-value-size-in-bytes\":16}},"
+ + "\"content-stats\":{\"3\":{\"total-bytes\":8},\"4\":{\"total-bytes\":16}},"
+ "\"key-metadata\":\"00000000000000000000000000000000\","
+ "\"split-offsets\":[128],\"equality-ids\":[3],\"sort-order-id\":1}";
} else {
@@ -616,7 +616,7 @@ public class TestContentFileParser {
+ "\"nan-value-counts\":{\"keys\":[3,4],\"values\":[0,0]},"
+ "\"lower-bounds\":{\"keys\":[3,4],\"values\":[\"01000000\",\"02000000\"]},"
+ "\"upper-bounds\":{\"keys\":[3,4],\"values\":[\"05000000\",\"0A000000\"]},"
- + "\"content-stats\":{\"3\":{\"avg-value-size-in-bytes\":8},\"4\":{\"avg-value-size-in-bytes\":16}},"
+ + "\"content-stats\":{\"3\":{\"total-bytes\":8},\"4\":{\"total-bytes\":16}},"
+ "\"key-metadata\":\"00000000000000000000000000000000\","
+ "\"split-offsets\":[128],\"equality-ids\":[3],\"sort-order-id\":1}";
}
@@ -640,7 +640,7 @@ public class TestContentFileParser {
assertThat(actual.nanValueCounts()).isEqualTo(expected.nanValueCounts());
assertThat(actual.lowerBounds()).isEqualTo(expected.lowerBounds());
assertThat(actual.upperBounds()).isEqualTo(expected.upperBounds());
- assertThat(actual.avgValueSizes()).isEqualTo(expected.avgValueSizes());
+ assertThat(actual.totalBytes()).isEqualTo(expected.totalBytes());
assertThat(actual.keyMetadata()).isEqualTo(expected.keyMetadata());
assertThat(actual.splitOffsets()).isEqualTo(expected.splitOffsets());
assertThat(actual.equalityFieldIds()).isEqualTo(expected.equalityFieldIds());
diff --git a/core/src/test/java/org/apache/iceberg/TestContentStatsBackedMap.java b/core/src/test/java/org/apache/iceberg/TestContentStatsBackedMap.java
index cd9a3c4387..db433cbb58 100644
--- a/core/src/test/java/org/apache/iceberg/TestContentStatsBackedMap.java
+++ b/core/src/test/java/org/apache/iceberg/TestContentStatsBackedMap.java
@@ -81,15 +81,16 @@ public class TestContentStatsBackedMap {
}
@Test
- public void avgValueSizes() {
+ public void totalBytes() {
Schema schema = new Schema(optional(4, "str", Types.StringType.get()));
Types.StructType statsType = StatsUtil.statsReadSchema(schema, List.of(4));
Types.StructType fieldStatsType = statsType.field("str").type().asStructType();
ContentStatsStruct stats = new ContentStatsStruct(statsType);
- stats.setStats(4, StatsTestUtil.mockFieldStats(fieldStatsType, 4, "a", "z", 10L, 1L, null, 12));
+ stats.setStats(
+ 4, StatsTestUtil.mockFieldStats(fieldStatsType, 4, "a", "z", 10L, 1L, null, 12L));
- Map<Integer, Integer> map = ContentStatsBackedMap.avgValueSizes(stats);
- assertThat(map).containsOnly(Map.entry(4, 12));
+ Map<Integer, Long> map = ContentStatsBackedMap.totalBytes(stats);
+ assertThat(map).containsOnly(Map.entry(4, 12L));
}
@Test
@@ -257,7 +258,7 @@ public class TestContentStatsBackedMap {
// only a required long column: it tracks neither null_value_count nor nan_value_count
assertThat(ContentStatsBackedMap.nullValueCounts(ONLY_REQUIRED_STATS)).isNull();
assertThat(ContentStatsBackedMap.nanValueCounts(ONLY_REQUIRED_STATS)).isNull();
- assertThat(ContentStatsBackedMap.avgValueSizes(ONLY_REQUIRED_STATS)).isNull();
+ assertThat(ContentStatsBackedMap.totalBytes(ONLY_REQUIRED_STATS)).isNull();
Map<Integer, Long> valueCounts = ContentStatsBackedMap.valueCounts(ONLY_REQUIRED_STATS);
assertThat(valueCounts).isNotNull().containsOnly(Map.entry(1, 10L));
diff --git a/core/src/test/java/org/apache/iceberg/TestContentStatsStruct.java b/core/src/test/java/org/apache/iceberg/TestContentStatsStruct.java
index 6d83f940f9..3a8161393b 100644
--- a/core/src/test/java/org/apache/iceberg/TestContentStatsStruct.java
+++ b/core/src/test/java/org/apache/iceberg/TestContentStatsStruct.java
@@ -55,7 +55,7 @@ public class TestContentStatsStruct {
private static final FieldStats<Long> ID_STATS =
new FieldStatsStruct<>(
- CONTENT_STATS_STRUCT.field("id").type().asStructType(), 0L, 25L, true, 26L, 0L, 0L, null);
+ CONTENT_STATS_STRUCT.field("id").type().asStructType(), 0L, 25L, true, 26L, 0L, 0L, 0L);
private static final FieldStats<String> DATA_STATS =
new FieldStatsStruct<>(
CONTENT_STATS_STRUCT.field("data").type().asStructType(),
@@ -65,7 +65,7 @@ public class TestContentStatsStruct {
26L,
0L,
0L,
- null);
+ 2L);
@Test
public void emptyContentStats() {
@@ -103,7 +103,7 @@ public class TestContentStatsStruct {
ContentStatsStruct stats = new ContentStatsStruct(CONTENT_STATS_STRUCT);
FieldStats<Integer> fieldStats =
- new FieldStatsStruct<>(UNKNOWN_FIELD_STATS_STRUCT, 0, 10, false, 8, 3, 0, null);
+ new FieldStatsStruct<>(UNKNOWN_FIELD_STATS_STRUCT, 0, 10, false, 8, 3, 0, 23L);
assertThatThrownBy(() -> stats.setStats(4, fieldStats))
.hasMessage("Cannot set stats for unknown field ID: 4")
diff --git a/core/src/test/java/org/apache/iceberg/TestFieldStatsStruct.java b/core/src/test/java/org/apache/iceberg/TestFieldStatsStruct.java
index 3bfe6737fc..d97d9528ee 100644
--- a/core/src/test/java/org/apache/iceberg/TestFieldStatsStruct.java
+++ b/core/src/test/java/org/apache/iceberg/TestFieldStatsStruct.java
@@ -61,7 +61,7 @@ public class TestFieldStatsStruct {
@Test
public void fieldAccess() {
- FieldStats<String> stats = new FieldStatsStruct<>(STRING_STATS, "a", "z", true, 28, 2, 0, 1);
+ FieldStats<String> stats = new FieldStatsStruct<>(STRING_STATS, "a", "z", true, 28, 2, 0, 1L);
assertThat(stats.fieldId()).isEqualTo(100);
assertThat(stats.lowerBound()).isEqualTo("a");
@@ -70,20 +70,20 @@ public class TestFieldStatsStruct {
assertThat(stats.valueCount()).isEqualTo(28L);
assertThat(stats.nullValueCount()).isEqualTo(2L);
assertThat(stats.nanValueCount()).isEqualTo(0L);
- assertThat(stats.avgValueSizeInBytes()).isEqualTo(1);
+ assertThat(stats.totalBytes()).isEqualTo(1L);
}
@Test
public void stringGetByPosition() {
FieldStatsStruct<String> stats =
- new FieldStatsStruct<>(STRING_STATS, "a", "z", true, 28, 2, 0, 1);
+ new FieldStatsStruct<>(STRING_STATS, "a", "z", true, 28, 2, 0, 1L);
assertThat(stats.get(pos(STRING_STATS, "lower_bound"), String.class)).isEqualTo("a");
assertThat(stats.get(pos(STRING_STATS, "upper_bound"), String.class)).isEqualTo("z");
assertThat(stats.get(pos(STRING_STATS, "tight_bounds"), Boolean.class)).isTrue();
assertThat(stats.get(pos(STRING_STATS, "value_count"), Long.class)).isEqualTo(28L);
assertThat(stats.get(pos(STRING_STATS, "null_value_count"), Long.class)).isEqualTo(2L);
- assertThat(stats.get(pos(STRING_STATS, "avg_value_size_in_bytes"), Integer.class)).isEqualTo(1);
+ assertThat(stats.get(pos(STRING_STATS, "total_bytes"), Long.class)).isEqualTo(1L);
}
@Test
@@ -95,20 +95,20 @@ public class TestFieldStatsStruct {
stats.set(pos(STRING_STATS, "tight_bounds"), false);
stats.set(pos(STRING_STATS, "value_count"), 28L);
stats.set(pos(STRING_STATS, "null_value_count"), 2L);
- stats.set(pos(STRING_STATS, "avg_value_size_in_bytes"), 1);
+ stats.set(pos(STRING_STATS, "total_bytes"), 1L);
assertThat(stats.lowerBound()).isEqualTo("a");
assertThat(stats.upperBound()).isEqualTo("z");
assertThat(stats.tightBounds()).isFalse();
assertThat(stats.valueCount()).isEqualTo(28L);
assertThat(stats.nullValueCount()).isEqualTo(2L);
- assertThat(stats.avgValueSizeInBytes()).isEqualTo(1);
+ assertThat(stats.totalBytes()).isEqualTo(1L);
}
@Test
public void doubleGetByPosition() {
FieldStatsStruct<Double> stats =
- new FieldStatsStruct<>(DOUBLE_STATS, 0.0d, 25.0d, true, 34, 2, 6, 0);
+ new FieldStatsStruct<>(DOUBLE_STATS, 0.0d, 25.0d, true, 34, 2, 6, 0L);
assertThat(stats.get(pos(DOUBLE_STATS, "lower_bound"), Double.class)).isEqualTo(0.0d);
assertThat(stats.get(pos(DOUBLE_STATS, "upper_bound"), Double.class)).isEqualTo(25.0d);
@@ -135,7 +135,7 @@ public class TestFieldStatsStruct {
assertThat(stats.valueCount()).isEqualTo(34L);
assertThat(stats.nullValueCount()).isEqualTo(2L);
assertThat(stats.nanValueCount()).isEqualTo(6L);
- assertThat(stats.avgValueSizeInBytes()).isNull();
+ assertThat(stats.hasTotalBytes()).isFalse();
}
@Test
@@ -150,14 +150,14 @@ public class TestFieldStatsStruct {
public void fromFieldMetricsString() {
FieldStatsStruct<String> stats = new FieldStatsStruct<>(STRING_STATS);
- stats.fromFieldMetrics(new FieldMetrics<>(100, 28, 2, -1, "a", "z", null, 4));
+ stats.fromFieldMetrics(new FieldMetrics<>(100, 28, 2, -1, "a", "z", null, 4L));
assertThat(stats.lowerBound()).isEqualTo("a");
assertThat(stats.upperBound()).isEqualTo("z");
assertThat(stats.tightBounds()).isFalse();
assertThat(stats.valueCount()).isEqualTo(28L);
assertThat(stats.nullValueCount()).isEqualTo(2L);
- assertThat(stats.avgValueSizeInBytes()).isEqualTo(4);
+ assertThat(stats.totalBytes()).isEqualTo(4L);
}
@Test
@@ -172,7 +172,7 @@ public class TestFieldStatsStruct {
assertThat(stats.valueCount()).isEqualTo(34L);
assertThat(stats.nullValueCount()).isEqualTo(2L);
assertThat(stats.nanValueCount()).isEqualTo(6L);
- assertThat(stats.avgValueSizeInBytes()).isNull(); // unknown
+ assertThat(stats.hasTotalBytes()).isFalse();
}
@Test
@@ -277,7 +277,7 @@ public class TestFieldStatsStruct {
28L,
2L,
isFloatingPoint ? 6L : 0L,
- isVariableLength ? 1 : null);
+ isVariableLength ? 12L : 0L);
Comparator<StructLike> comparator = Comparators.forType(statsStruct);
@@ -314,7 +314,7 @@ public class TestFieldStatsStruct {
upperBound.set(3, 8.0d);
FieldStatsStruct<Object> stats =
- new FieldStatsStruct<>(statsStruct, lowerBound, upperBound, false, 28L, 2L, 0L, null);
+ new FieldStatsStruct<>(statsStruct, lowerBound, upperBound, false, 28L, 2L, 0L, 23L);
Comparator<StructLike> comparator = Comparators.forType(statsStruct);
@@ -354,7 +354,7 @@ public class TestFieldStatsStruct {
upperObject.put("$['x']", Variants.of(10));
Variant upperBound = Variant.of(metadata, upperObject);
- int size = metadata.dictionarySize() + lowerObject.sizeInBytes();
+ long size = metadata.dictionarySize() + lowerObject.sizeInBytes();
FieldStatsStruct<Object> stats =
new FieldStatsStruct<>(statsStruct, lowerBound, upperBound, false, 28L, 2L, 0L, size);
@@ -364,7 +364,7 @@ public class TestFieldStatsStruct {
assertThat(copy.type()).isEqualTo(stats.type());
assertThat(copy.valueCount()).isEqualTo(28L);
assertThat(copy.nullValueCount()).isEqualTo(2L);
- assertThat(copy.avgValueSizeInBytes()).isEqualTo(size);
+ assertThat(copy.totalBytes()).isEqualTo(size);
Variant copyLower = (Variant) copy.lowerBound();
VariantTestUtil.assertEqual(lowerBound.metadata(), copyLower.metadata());
diff --git a/core/src/test/java/org/apache/iceberg/TestGeometryFieldMetrics.java b/core/src/test/java/org/apache/iceberg/TestGeometryFieldMetrics.java
index 953835b62e..21952acd2f 100644
--- a/core/src/test/java/org/apache/iceberg/TestGeometryFieldMetrics.java
+++ b/core/src/test/java/org/apache/iceberg/TestGeometryFieldMetrics.java
@@ -49,6 +49,7 @@ public class TestGeometryFieldMetrics {
assertThat(metrics.upperBound()).isEqualTo(GeospatialBound.createXY(30, 40));
// both WKB points are 21 bytes, so the average WKB size is 21
assertThat(metrics.avgValueSizeInBytes()).isEqualTo(21);
+ assertThat(metrics.totalBytes()).isEqualTo(42L);
}
@Test
@@ -62,6 +63,7 @@ public class TestGeometryFieldMetrics {
assertThat(metrics.lowerBound()).isEqualTo(GeospatialBound.createXY(12, 34));
assertThat(metrics.upperBound()).isEqualTo(GeospatialBound.createXY(12, 34));
assertThat(metrics.avgValueSizeInBytes()).isEqualTo(21);
+ assertThat(metrics.totalBytes()).isEqualTo(21L);
}
@Test
@@ -75,8 +77,8 @@ public class TestGeometryFieldMetrics {
assertThat(metrics.valueCount()).isEqualTo(1L);
assertThat(metrics.lowerBound()).isNull();
assertThat(metrics.upperBound()).isNull();
- // the value has no bounds but still has a size, so the average is still reported
assertThat(metrics.avgValueSizeInBytes()).isEqualTo(21);
+ assertThat(metrics.totalBytes()).isEqualTo(21L);
}
@Test
diff --git a/core/src/test/java/org/apache/iceberg/TestMetrics.java b/core/src/test/java/org/apache/iceberg/TestMetrics.java
index 89d528c22e..0ff153e991 100644
--- a/core/src/test/java/org/apache/iceberg/TestMetrics.java
+++ b/core/src/test/java/org/apache/iceberg/TestMetrics.java
@@ -299,6 +299,8 @@ public abstract class TestMetrics {
assertBounds(3, Types.GeographyType.crs84(), null, null, metrics);
assertThat(metrics.avgValueSizes())
.containsOnly(Map.entry(2, geom.remaining()), Map.entry(3, geog.remaining()));
+ assertThat(metrics.totalBytes())
+ .containsOnly(Map.entry(2, 2L * geom.remaining()), Map.entry(3, (long) geog.remaining()));
}
@TestTemplate
diff --git a/core/src/test/java/org/apache/iceberg/TestMetricsUtil.java b/core/src/test/java/org/apache/iceberg/TestMetricsUtil.java
index 8d882db633..fbee0917c7 100644
--- a/core/src/test/java/org/apache/iceberg/TestMetricsUtil.java
+++ b/core/src/test/java/org/apache/iceberg/TestMetricsUtil.java
@@ -27,47 +27,47 @@ import org.junit.jupiter.api.Test;
class TestMetricsUtil {
@Test
- void copyWithoutFieldCountsDropsAvgValueSizesForExcludedFields() {
- Metrics copy =
- MetricsUtil.copyWithoutFieldCounts(metricsWithAvgValueSizes(), ImmutableSet.of(1));
+ void copyWithoutFieldCountsDropsTotalBytesForExcludedFields() {
+ Metrics copy = MetricsUtil.copyWithoutFieldCounts(metricsWithTotalBytes(), ImmutableSet.of(1));
- assertThat(copy.avgValueSizes()).isEqualTo(ImmutableMap.of(2, 20));
+ assertThat(copy.totalBytes()).isEqualTo(ImmutableMap.of(2, 20L));
}
@Test
- void copyWithoutFieldCountsAndBoundsDropsAvgValueSizesForExcludedFields() {
+ void copyWithoutFieldCountsAndBoundsDropsTotalBytesForExcludedFields() {
Metrics copy =
- MetricsUtil.copyWithoutFieldCountsAndBounds(metricsWithAvgValueSizes(), ImmutableSet.of(1));
+ MetricsUtil.copyWithoutFieldCountsAndBounds(metricsWithTotalBytes(), ImmutableSet.of(1));
- assertThat(copy.avgValueSizes()).isEqualTo(ImmutableMap.of(2, 20));
+ assertThat(copy.totalBytes()).isEqualTo(ImmutableMap.of(2, 20L));
}
@Test
- void copyDropsAvgValueSizesEntirelyWhenAllFieldsExcluded() {
+ void copyDropsTotalBytesEntirelyWhenAllFieldsExcluded() {
// once every tracked field is excluded the filtered map is empty, and copyWithoutKeys returns
- // null rather than an empty map, matching the "null otherwise" contract of avgValueSizes()
+ // null rather than an empty map, matching the "null otherwise" contract of totalBytes()
Metrics copy =
- MetricsUtil.copyWithoutFieldCounts(metricsWithAvgValueSizes(), ImmutableSet.of(1, 2));
+ MetricsUtil.copyWithoutFieldCounts(metricsWithTotalBytes(), ImmutableSet.of(1, 2));
- assertThat(copy.avgValueSizes()).isNull();
+ assertThat(copy.totalBytes()).isNull();
}
@Test
- void copyWithStatsReturnsNullAvgValueSizesWhenNoRequestedColumnMatches() {
+ void copyWithStatsReturnsNullTotalBytesWhenNoRequestedColumnMatches() {
DataFile file =
DataFiles.builder(PartitionSpec.unpartitioned())
.withPath("/path/to/data.parquet")
.withFileSizeInBytes(10)
.withRecordCount(1)
- .withMetrics(metricsWithAvgValueSizes())
+ .withMetrics(metricsWithTotalBytes())
.build();
- assertThat(file.copyWithStats(ImmutableSet.of(2)).avgValueSizes())
- .isEqualTo(ImmutableMap.of(2, 20));
- assertThat(file.copyWithStats(ImmutableSet.of(3)).avgValueSizes()).isNull();
+ assertThat(file.copyWithStats(ImmutableSet.of(2)).totalBytes())
+ .isEqualTo(ImmutableMap.of(2, 20L));
+ assertThat(file.copyWithStats(ImmutableSet.of(3)).totalBytes()).isNull();
}
- private static Metrics metricsWithAvgValueSizes() {
- return new Metrics(3L, null, null, null, null, null, null, ImmutableMap.of(1, 10, 2, 20), null);
+ private static Metrics metricsWithTotalBytes() {
+ return new Metrics(
+ 3L, null, null, null, null, null, null, ImmutableMap.of(1, 10L, 2, 20L), null);
}
}
diff --git a/core/src/test/java/org/apache/iceberg/TestStatsUtil.java b/core/src/test/java/org/apache/iceberg/TestStatsUtil.java
index 183c4336f7..c40c50ea05 100644
--- a/core/src/test/java/org/apache/iceberg/TestStatsUtil.java
+++ b/core/src/test/java/org/apache/iceberg/TestStatsUtil.java
@@ -137,7 +137,7 @@ public class TestStatsUtil {
@FieldSource("FIXED_WIDTH_TYPES")
public void testFixedWidthPrimitiveStruct(Type type) {
// fixed-width, non-floating-point types track bounds (including tight bounds) but have no NaN
- // count or average value size
+ // count or total bytes
Types.StructType expected =
Types.StructType.of(
Types.NestedField.optional(30_001, "lower_bound", type),
@@ -157,7 +157,7 @@ public class TestStatsUtil {
@ParameterizedTest
@FieldSource("VARIABLE_WIDTH_TYPES")
public void testVariableWidthPrimitiveStruct(Type type) {
- // variable-width types track bounds (including tight bounds) and an average value size, but no
+ // variable-width types track bounds (including tight bounds) and total bytes, but no
// NaN count
Types.StructType expected =
Types.StructType.of(
@@ -166,7 +166,7 @@ public class TestStatsUtil {
Types.NestedField.optional(30_003, "tight_bounds", Types.BooleanType.get()),
Types.NestedField.optional(30_004, "value_count", Types.LongType.get()),
Types.NestedField.optional(30_005, "null_value_count", Types.LongType.get()),
- Types.NestedField.optional(30_007, "avg_value_size_in_bytes", Types.IntegerType.get()));
+ Types.NestedField.optional(30_007, "total_bytes", Types.LongType.get()));
Types.StructType actual = StatsUtil.fieldStatsStruct(type, 30_000, MetricsModes.Full.get());
@@ -184,7 +184,7 @@ public class TestStatsUtil {
@FieldSource("GEO_TYPES")
public void testGeoStruct(Type type) {
// geometry and geography use bounding-box structs for their bounds, do not track tight bounds,
- // and record an average value size
+ // and record total bytes
Types.StructType lowerBound =
Types.StructType.of(
Types.NestedField.required(30_010, "x", Types.DoubleType.get()),
@@ -203,7 +203,7 @@ public class TestStatsUtil {
Types.NestedField.optional(30_002, "upper_bound", upperBound),
Types.NestedField.optional(30_004, "value_count", Types.LongType.get()),
Types.NestedField.optional(30_005, "null_value_count", Types.LongType.get()),
- Types.NestedField.optional(30_007, "avg_value_size_in_bytes", Types.IntegerType.get()));
+ Types.NestedField.optional(30_007, "total_bytes", Types.LongType.get()));
Types.StructType actual = StatsUtil.fieldStatsStruct(type, 30_000, MetricsModes.Full.get());
@@ -217,7 +217,7 @@ public class TestStatsUtil {
@FieldSource("FLOATING_POINT_TYPES")
public void testFloatingPointStruct(Type type) {
// floating-point types track bounds (including tight bounds) and a NaN count, but have no
- // average value size
+ // total bytes
Types.StructType expected =
Types.StructType.of(
Types.NestedField.optional(30_001, "lower_bound", type),
@@ -282,7 +282,7 @@ public class TestStatsUtil {
Types.StructType.of(
Types.NestedField.optional(30_004, "value_count", Types.LongType.get()),
Types.NestedField.optional(30_005, "null_value_count", Types.LongType.get()),
- Types.NestedField.optional(30_007, "avg_value_size_in_bytes", Types.IntegerType.get()));
+ Types.NestedField.optional(30_007, "total_bytes", Types.LongType.get()));
Types.StructType actual =
StatsUtil.fieldStatsStruct(Types.StringType.get(), 30_000, MetricsModes.Counts.get());
@@ -305,7 +305,7 @@ public class TestStatsUtil {
Types.NestedField.optional(30_003, "tight_bounds", Types.BooleanType.get()),
Types.NestedField.optional(30_004, "value_count", Types.LongType.get()),
Types.NestedField.optional(30_005, "null_value_count", Types.LongType.get()),
- Types.NestedField.optional(30_007, "avg_value_size_in_bytes", Types.IntegerType.get()));
+ Types.NestedField.optional(30_007, "total_bytes", Types.LongType.get()));
Types.StructType actual = StatsUtil.fieldStatsStruct(string, 30_000, mode);
@@ -333,7 +333,7 @@ public class TestStatsUtil {
Types.NestedField.optional(10_403, "tight_bounds", Types.BooleanType.get()),
Types.NestedField.optional(10_404, "value_count", Types.LongType.get()),
Types.NestedField.optional(10_405, "null_value_count", Types.LongType.get()),
- Types.NestedField.optional(10_407, "avg_value_size_in_bytes", Types.IntegerType.get()));
+ Types.NestedField.optional(10_407, "total_bytes", Types.LongType.get()));
Types.StructType expected =
Types.StructType.of(
Types.NestedField.optional(10_200, "id", idStats),
@@ -367,7 +367,7 @@ public class TestStatsUtil {
Types.NestedField.optional(10_603, "tight_bounds", Types.BooleanType.get()),
Types.NestedField.optional(10_604, "value_count", Types.LongType.get()),
Types.NestedField.optional(10_605, "null_value_count", Types.LongType.get()),
- Types.NestedField.optional(10_607, "avg_value_size_in_bytes", Types.IntegerType.get()));
+ Types.NestedField.optional(10_607, "total_bytes", Types.LongType.get()));
Types.StructType expected =
Types.StructType.of(
Types.NestedField.optional(10_200, "id", idStats),
@@ -455,7 +455,7 @@ public class TestStatsUtil {
Types.NestedField.optional(10_403, "tight_bounds", Types.BooleanType.get()),
Types.NestedField.optional(10_404, "value_count", Types.LongType.get()),
Types.NestedField.optional(10_405, "null_value_count", Types.LongType.get()),
- Types.NestedField.optional(10_407, "avg_value_size_in_bytes", Types.IntegerType.get()));
+ Types.NestedField.optional(10_407, "total_bytes", Types.LongType.get()));
Types.StructType expected =
Types.StructType.of(
Types.NestedField.optional(10_200, "id", idStats),
@@ -490,7 +490,7 @@ public class TestStatsUtil {
Types.NestedField.optional(10_603, "tight_bounds", Types.BooleanType.get()),
Types.NestedField.optional(10_604, "value_count", Types.LongType.get()),
Types.NestedField.optional(10_605, "null_value_count", Types.LongType.get()),
- Types.NestedField.optional(10_607, "avg_value_size_in_bytes", Types.IntegerType.get()));
+ Types.NestedField.optional(10_607, "total_bytes", Types.LongType.get()));
Types.StructType expected =
Types.StructType.of(
Types.NestedField.optional(10_200, "id", idStats),
diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java
index 31d6758b80..7f6e61ee43 100644
--- a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java
+++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java
@@ -86,7 +86,7 @@ class TestTrackedFileAdapters {
CONTENT_STATS_TYPE.fieldType("score").asStructType(), 2, 1.0f, 100.0f, 100L, 10L, 3L);
private static final FieldStats<?> GEOM_STATS =
StatsTestUtil.mockFieldStats(
- CONTENT_STATS_TYPE.fieldType("geom").asStructType(), 3, null, null, 100L, 20L, null, 12);
+ CONTENT_STATS_TYPE.fieldType("geom").asStructType(), 3, null, null, 100L, 20L, null, 12L);
private static final ContentStatsStruct CONTENT_STATS =
new ContentStatsStruct(CONTENT_STATS_TYPE);
@@ -185,7 +185,7 @@ class TestTrackedFileAdapters {
assertThat(dataFile.nullValueCounts())
.containsOnly(Map.entry(1, 5L), Map.entry(2, 10L), Map.entry(3, 20L));
assertThat(dataFile.nanValueCounts()).containsOnly(Map.entry(2, 3L));
- assertThat(dataFile.avgValueSizes()).containsOnly(Map.entry(3, 12));
+ assertThat(dataFile.totalBytes()).containsOnly(Map.entry(3, 12L));
assertThat(dataFile.lowerBounds())
.containsOnly(
Map.entry(1, Conversions.toByteBuffer(Types.IntegerType.get(), 1)),
@@ -264,7 +264,7 @@ class TestTrackedFileAdapters {
assertThat(deleteFile.nullValueCounts())
.containsOnly(Map.entry(1, 5L), Map.entry(2, 10L), Map.entry(3, 20L));
assertThat(deleteFile.nanValueCounts()).containsOnly(Map.entry(2, 3L));
- assertThat(deleteFile.avgValueSizes()).containsOnly(Map.entry(3, 12));
+ assertThat(deleteFile.totalBytes()).containsOnly(Map.entry(3, 12L));
assertThat(deleteFile.lowerBounds())
.containsOnly(
Map.entry(1, Conversions.toByteBuffer(Types.IntegerType.get(), 1)),
diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java
index 6db0059742..89e9b448cb 100644
--- a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java
+++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java
@@ -110,10 +110,10 @@ class TestV4ManifestReader {
private static final FieldStatsStruct<Integer> ID_STATS =
new FieldStatsStruct<>(
- STATS_TYPE.fieldType("id").asStructType(), 0, 99, true, RECORD_COUNT, 0, 0, null);
+ STATS_TYPE.fieldType("id").asStructType(), 0, 99, true, RECORD_COUNT, 0, 0, 0L);
private static final FieldStatsStruct<String> DATA_STATS =
new FieldStatsStruct<>(
- STATS_TYPE.fieldType("data").asStructType(), "a", "z", false, RECORD_COUNT, 20, 0, null);
+ STATS_TYPE.fieldType("data").asStructType(), "a", "z", false, RECORD_COUNT, 20, 0, 2L);
private static final ContentStatsStruct CONTENT_STATS = new ContentStatsStruct(STATS_TYPE);
static {
diff --git a/core/src/test/java/org/apache/iceberg/TestValueSizeFieldMetrics.java b/core/src/test/java/org/apache/iceberg/TestValueSizeFieldMetrics.java
index 67d4a081ab..82d8d9ffb0 100644
--- a/core/src/test/java/org/apache/iceberg/TestValueSizeFieldMetrics.java
+++ b/core/src/test/java/org/apache/iceberg/TestValueSizeFieldMetrics.java
@@ -26,7 +26,7 @@ import org.junit.jupiter.api.Test;
class TestValueSizeFieldMetrics {
@Test
- void averageValueSize() {
+ void totalBytes() {
ValueSizeFieldMetrics.Builder builder = new ValueSizeFieldMetrics.Builder(2);
builder.addValueSize(21);
builder.addValueSize(42);
@@ -37,7 +37,7 @@ class TestValueSizeFieldMetrics {
assertThat(metrics.valueCount()).isEqualTo(2);
assertThat(metrics.nullValueCount()).isZero();
assertThat(metrics.nanValueCount()).isEqualTo(-1);
- assertThat(metrics.avgValueSizeInBytes()).isEqualTo(31);
+ assertThat(metrics.totalBytes()).isEqualTo(63L);
}
@Test
diff --git a/core/src/test/java/org/apache/iceberg/V4TestComparators.java b/core/src/test/java/org/apache/iceberg/V4TestComparators.java
index 088604b5a5..50f8976663 100644
--- a/core/src/test/java/org/apache/iceberg/V4TestComparators.java
+++ b/core/src/test/java/org/apache/iceberg/V4TestComparators.java
@@ -176,7 +176,8 @@ class V4TestComparators {
comparator.thenComparing(
fs -> fs.hasNanValueCount() ? fs.nanValueCount() : null, natural());
} else if (type.typeId() == Type.TypeID.STRING || type.typeId() == Type.TypeID.BINARY) {
- comparator = comparator.thenComparing(FieldStats::avgValueSizeInBytes, natural());
+ comparator =
+ comparator.thenComparing(fs -> fs.hasTotalBytes() ? fs.totalBytes() : null, natural());
}
}
diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py
index ad3eed0043..9a177f3715 100644
--- a/open-api/rest-catalog-open-api.py
+++ b/open-api/rest-catalog-open-api.py
@@ -1226,10 +1226,10 @@ class CountMap(BaseModel):
class FieldStatistics(BaseModel):
- avg_value_size_in_bytes: int | None = Field(
+ total_bytes: int | None = Field(
None,
- alias='avg-value-size-in-bytes',
- description='Avg value size in memory (uncompressed) in bytes over non-null values to estimate memory consumption',
+ alias='total-bytes',
+ description='Estimated uncompressed bytes in memory of non-null values. The estimate depends on the in-memory representation',
)
diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml
index 2064e670a4..ed3f4ef63c 100644
--- a/open-api/rest-catalog-open-api.yaml
+++ b/open-api/rest-catalog-open-api.yaml
@@ -5495,15 +5495,15 @@ components:
FieldStatistics:
type: object
properties:
- avg-value-size-in-bytes:
+ total-bytes:
type: integer
- format: int32
+ format: int64
description:
- Avg value size in memory (uncompressed) in bytes over non-null
- values to estimate memory consumption
+ Estimated uncompressed bytes in memory of non-null values. The
+ estimate depends on the in-memory representation
example:
{
- "avg-value-size-in-bytes": 8
+ "total-bytes": 8
}
ValueMap:
diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java
index c45c0a507a..690c9077c6 100644
--- a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java
+++ b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java
@@ -136,7 +136,7 @@ class ParquetMetrics {
Map<Integer, Long> valueCounts = Maps.newHashMap();
Map<Integer, Long> nullValueCounts = Maps.newHashMap();
Map<Integer, Long> nanValueCounts = Maps.newHashMap();
- Map<Integer, Integer> avgValueSizes = Maps.newHashMap();
+ Map<Integer, Long> totalBytes = Maps.newHashMap();
Map<Integer, ByteBuffer> lowerBounds = Maps.newHashMap();
Map<Integer, ByteBuffer> upperBounds = Maps.newHashMap();
Map<Integer, org.apache.iceberg.types.Type> originalTypes = Maps.newHashMap();
@@ -159,8 +159,8 @@ class ParquetMetrics {
nanValueCounts.put(id, metrics.nanValueCount());
}
- if (metrics.avgValueSizeInBytes() != null) {
- avgValueSizes.put(id, metrics.avgValueSizeInBytes());
+ if (metrics.totalBytes() >= 0) {
+ totalBytes.put(id, metrics.totalBytes());
}
if (metrics.lowerBound() != null) {
@@ -184,7 +184,7 @@ class ParquetMetrics {
nanValueCounts,
lowerBounds,
upperBounds,
- avgValueSizes.isEmpty() ? null : avgValueSizes,
+ totalBytes.isEmpty() ? null : totalBytes,
originalTypes);
}
@@ -278,7 +278,7 @@ class ParquetMetrics {
null,
null,
null,
- fieldMetrics.avgValueSizeInBytes());
+ fieldMetrics.totalBytes());
} else {
T lowerBound = truncateLowerBound(icebergType, fieldMetrics.lowerBound(), truncateLength);
T upperBound = truncateUpperBound(icebergType, fieldMetrics.upperBound(), truncateLength);
@@ -290,7 +290,7 @@ class ParquetMetrics {
lowerBound,
upperBound,
icebergType,
- fieldMetrics.avgValueSizeInBytes());
+ fieldMetrics.totalBytes());
}
}
diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetValueWriters.java b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetValueWriters.java
index a64cc380cb..d11f2c6f52 100644
--- a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetValueWriters.java
+++ b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetValueWriters.java
@@ -522,7 +522,7 @@ public class ParquetValueWriters {
metrics.lowerBound(),
metrics.upperBound(),
metrics.originalType(),
- metrics.avgValueSizeInBytes());
+ metrics.totalBytes());
}
}
diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetDataWriter.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetDataWriter.java
index e6e1d175f8..fe336d12a1 100644
--- a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetDataWriter.java
+++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetDataWriter.java
@@ -149,13 +149,13 @@ public class TestParquetDataWriter {
DataFile dataFile = dataWriter.toDataFile();
assertThat(dataFile.recordCount()).isEqualTo(geoRecords.size());
- assertThat(dataFile.avgValueSizes())
- .containsOnly(Map.entry(2, geom.remaining()), Map.entry(3, geog.remaining()));
- assertThat(dataFile.copy().avgValueSizes()).isEqualTo(dataFile.avgValueSizes());
- assertThat(dataFile.copyWithStats(Set.of(2)).avgValueSizes())
- .containsOnly(Map.entry(2, geom.remaining()));
- assertThat(dataFile.copyWithStats(Set.of(1)).avgValueSizes()).isNull();
- assertThat(dataFile.copyWithoutStats().avgValueSizes()).isNull();
+ assertThat(dataFile.totalBytes())
+ .containsOnly(Map.entry(2, 2L * geom.remaining()), Map.entry(3, (long) geog.remaining()));
+ assertThat(dataFile.copy().totalBytes()).isEqualTo(dataFile.totalBytes());
+ assertThat(dataFile.copyWithStats(Set.of(2)).totalBytes())
+ .containsOnly(Map.entry(2, 2L * geom.remaining()));
+ assertThat(dataFile.copyWithStats(Set.of(1)).totalBytes()).isNull();
+ assertThat(dataFile.copyWithoutStats().totalBytes()).isNull();
List<Record> writtenRecords;
try (CloseableIterable<Record> reader =
diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetValueWriters.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetValueWriters.java
index 499f1b08fc..2b23696c72 100644
--- a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetValueWriters.java
+++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetValueWriters.java
@@ -117,8 +117,8 @@ class TestParquetValueWriters {
assertThat(metrics.valueCount()).isEqualTo(2);
assertThat(metrics.nullValueCount()).isEqualTo(1);
});
- assertThat(metricsById.get(4).avgValueSizeInBytes()).isEqualTo(21);
- assertThat(metricsById.get(5).avgValueSizeInBytes()).isEqualTo(42);
+ assertThat(metricsById.get(4).totalBytes()).isEqualTo(21L);
+ assertThat(metricsById.get(5).totalBytes()).isEqualTo(42L);
}
@Test
@@ -143,7 +143,7 @@ class TestParquetValueWriters {
FieldMetrics<?> metrics = writer.metrics().findFirst().orElseThrow();
assertThat(metrics.valueCount()).isEqualTo(3);
assertThat(metrics.nullValueCount()).isEqualTo(1);
- assertThat(metrics.avgValueSizeInBytes()).isEqualTo(31);
+ assertThat(metrics.totalBytes()).isEqualTo(63L);
}
@Test
@@ -165,12 +165,10 @@ class TestParquetValueWriters {
writer.write(0, ByteBuffer.allocate(42));
writer.write(0, null);
- // the geometry writer adds bounds but must keep the same average WKB size metric as the
- // counts-only geospatial writer: the average is over the two non-null values, (21 + 42) / 2
FieldMetrics<?> metrics = writer.metrics().findFirst().orElseThrow();
assertThat(metrics.valueCount()).isEqualTo(3);
assertThat(metrics.nullValueCount()).isEqualTo(1);
- assertThat(metrics.avgValueSizeInBytes()).isEqualTo(31);
+ assertThat(metrics.totalBytes()).isEqualTo(63L);
}
@Test
diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkParquetWriter.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkParquetWriter.java
index 6a5bbe8d39..169285e434 100644
--- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkParquetWriter.java
+++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkParquetWriter.java
@@ -193,14 +193,14 @@ public class TestSparkParquetWriter {
}
@Test
- public void testGeospatialAvgValueSizeMetrics() throws IOException {
+ public void testGeospatialTotalBytesMetrics() throws IOException {
Schema geoSchema =
new Schema(
required(1, "id", Types.LongType.get()),
optional(2, "geom", Types.GeometryType.crs84()),
optional(3, "geog", Types.GeographyType.crs84()));
- // WKB payloads of 3 and 5 bytes for geometry (avg 4), 7 bytes for geography.
+ // WKB payloads of 3 and 5 bytes for geometry (total 8), 7 bytes for geography.
byte[] geomWkbSmall = new byte[] {0x01, 0x02, 0x03};
byte[] geomWkbLarge = new byte[] {0x01, 0x02, 0x03, 0x04, 0x05};
byte[] geogWkb = new byte[] {0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a};
@@ -213,7 +213,7 @@ public class TestSparkParquetWriter {
InternalRow second = new GenericInternalRow(3);
second.update(0, 2L);
second.update(1, STUtils.stGeomFromWKB(geomWkbLarge));
- // geography left null on the second row, so it must not affect the average.
+ // geography left null on the second row, so it must not contribute to total bytes.
File testFile = File.createTempFile("junit", null, temp.toFile());
assertThat(testFile.delete()).as("Delete should succeed").isTrue();
@@ -237,12 +237,12 @@ public class TestSparkParquetWriter {
FieldMetrics<?> geomMetrics = metricsById.get(geomId);
assertThat(geomMetrics.valueCount()).isEqualTo(2);
assertThat(geomMetrics.nullValueCount()).isZero();
- assertThat(geomMetrics.avgValueSizeInBytes()).isEqualTo(4);
+ assertThat(geomMetrics.totalBytes()).isEqualTo(8L);
FieldMetrics<?> geogMetrics = metricsById.get(geogId);
assertThat(geogMetrics.valueCount()).isEqualTo(2);
assertThat(geogMetrics.nullValueCount()).isEqualTo(1);
- assertThat(geogMetrics.avgValueSizeInBytes()).isEqualTo(7);
+ assertThat(geogMetrics.totalBytes()).isEqualTo(7L);
}
private static int fieldId(MessageType parquetSchema, String column) {
diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkParquetWriter.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkParquetWriter.java
index 15290afa4d..cd46857733 100644
--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkParquetWriter.java
+++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkParquetWriter.java
@@ -242,14 +242,14 @@ public class TestSparkParquetWriter {
}
@Test
- public void testGeospatialAvgValueSizeMetrics() throws IOException {
+ public void testGeospatialTotalBytesMetrics() throws IOException {
Schema geoSchema =
new Schema(
required(1, "id", Types.LongType.get()),
optional(2, "geom", Types.GeometryType.crs84()),
optional(3, "geog", Types.GeographyType.crs84()));
- // WKB payloads of 21 and 41 bytes for geometry (avg 31), 21 bytes for geography.
+ // WKB payloads of 21 and 41 bytes for geometry (total 62), 21 bytes for geography.
byte[] geomWkbSmall = pointWkb(30.0, 10.0);
byte[] geomWkbLarge = lineStringWkb(30.0, 10.0, 40.0, 20.0);
byte[] geogWkb = pointWkb(-71.0, 42.0);
@@ -262,7 +262,7 @@ public class TestSparkParquetWriter {
InternalRow second = new GenericInternalRow(3);
second.update(0, 2L);
second.update(1, STUtils.stGeomFromWKB(geomWkbLarge));
- // geography left null on the second row, so it must not affect the average.
+ // geography left null on the second row, so it must not contribute to total bytes.
File testFile = File.createTempFile("junit", null, temp.toFile());
assertThat(testFile.delete()).as("Delete should succeed").isTrue();
@@ -286,12 +286,12 @@ public class TestSparkParquetWriter {
FieldMetrics<?> geomMetrics = metricsById.get(geomId);
assertThat(geomMetrics.valueCount()).isEqualTo(2);
assertThat(geomMetrics.nullValueCount()).isZero();
- assertThat(geomMetrics.avgValueSizeInBytes()).isEqualTo(31);
+ assertThat(geomMetrics.totalBytes()).isEqualTo(62L);
FieldMetrics<?> geogMetrics = metricsById.get(geogId);
assertThat(geogMetrics.valueCount()).isEqualTo(2);
assertThat(geogMetrics.nullValueCount()).isEqualTo(1);
- assertThat(geogMetrics.avgValueSizeInBytes()).isEqualTo(21);
+ assertThat(geogMetrics.totalBytes()).isEqualTo(21L);
}
private static int fieldId(MessageType parquetSchema, String column) {