You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by hu...@apache.org on 2026/06/16 21:00:55 UTC
(iceberg) branch main updated: Parquet: Variant shredding follow-ups from PR #14297 (#16818)
This is an automated email from the ASF dual-hosted git repository.
huaxingao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 85ffa1984e Parquet: Variant shredding follow-ups from PR #14297 (#16818)
85ffa1984e is described below
commit 85ffa1984e115e80ba1571f3eb017fcf0ba39031
Author: Neelesh Salian <ns...@users.noreply.github.com>
AuthorDate: Tue Jun 16 14:00:47 2026 -0700
Parquet: Variant shredding follow-ups from PR #14297 (#16818)
* Parquet: Variant shredding follow-ups from PR #14297
* inference cleanup
* PR comment, fix cache invalidation
---
.../apache/iceberg/parquet/ParquetFormatModel.java | 9 +++
.../iceberg/parquet/VariantShreddingAnalyzer.java | 94 ++++++++++++++--------
.../parquet/TestVariantShreddingAnalyzer.java | 26 ++++++
3 files changed, 94 insertions(+), 35 deletions(-)
diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetFormatModel.java b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetFormatModel.java
index 9a4a62cae6..b097df8fc6 100644
--- a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetFormatModel.java
+++ b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetFormatModel.java
@@ -46,9 +46,13 @@ import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.Type;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class ParquetFormatModel<D, S, R>
extends BaseFormatModel<D, S, ParquetValueWriter<?>, R, MessageType> {
+ private static final Logger LOG = LoggerFactory.getLogger(ParquetFormatModel.class);
+
private final boolean isBatchReader;
private final VariantShreddingAnalyzer<D, S> variantAnalyzer;
private final Function<S, UnaryOperator<D>> copyFuncFactory;
@@ -292,12 +296,17 @@ public class ParquetFormatModel<D, S, R>
Preconditions.checkState(copyFuncFactory != null, "copyFuncFactory must not be null");
UnaryOperator<D> copyFunc = copyFuncFactory.apply(engineSchema);
Preconditions.checkState(copyFunc != null, "copyFunc must not return null");
+ LOG.debug("Building shredded variant appender with bufferSize={}", bufferSize);
return new BufferedFileAppender<>(
bufferSize,
bufferedRows -> {
Map<Integer, Type> shreddedTypes =
variantAnalyzer.analyzeVariantColumns(bufferedRows, schema, engineSchema);
+ LOG.debug(
+ "Variant inference: rows={}, shredded fields={}",
+ bufferedRows.size(),
+ shreddedTypes.size());
if (!shreddedTypes.isEmpty()) {
internal.variantShreddingFunc((fieldId, name) -> shreddedTypes.get(fieldId));
diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/VariantShreddingAnalyzer.java b/parquet/src/main/java/org/apache/iceberg/parquet/VariantShreddingAnalyzer.java
index d2a058c112..b9f567d3ef 100644
--- a/parquet/src/main/java/org/apache/iceberg/parquet/VariantShreddingAnalyzer.java
+++ b/parquet/src/main/java/org/apache/iceberg/parquet/VariantShreddingAnalyzer.java
@@ -19,8 +19,10 @@
package org.apache.iceberg.parquet;
import java.math.BigDecimal;
+import java.util.Comparator;
import java.util.List;
import java.util.Map;
+import java.util.PriorityQueue;
import java.util.Set;
import org.apache.iceberg.Schema;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
@@ -31,7 +33,6 @@ import org.apache.iceberg.types.Types.NestedField;
import org.apache.iceberg.variants.PhysicalType;
import org.apache.iceberg.variants.VariantArray;
import org.apache.iceberg.variants.VariantObject;
-import org.apache.iceberg.variants.VariantPrimitive;
import org.apache.iceberg.variants.VariantValue;
import org.apache.parquet.schema.GroupType;
import org.apache.parquet.schema.LogicalTypeAnnotation;
@@ -48,9 +49,9 @@ import org.apache.parquet.schema.Types;
* determinism is not guaranteed.
*
* <ul>
- * <li>Object fields use a TreeMap, so field ordering is alphabetical and deterministic.
- * <li>Type selection picks the most common type with explicit tie-break priority (see
- * TIE_BREAK_PRIORITY), not enum ordinal.
+ * <li>Object fields are emitted in alphabetical order in the shredded schema.
+ * <li>Type selection picks the most common type with explicit tie-break priority (see {@link
+ * FieldInfo#TIE_BREAK_PRIORITY}), not enum ordinal.
* <li>Integer types (INT8/16/32/64) and decimal types (DECIMAL4/8/16) are each promoted to the
* widest observed before competing with other types.
* <li>Fields below {@code MIN_FIELD_FREQUENCY} are pruned. Above {@code MAX_SHREDDED_FIELDS}, the
@@ -168,17 +169,23 @@ public abstract class VariantShreddingAnalyzer<T, S> {
// Cap at MAX_SHREDDED_FIELDS, keep the most frequently observed
if (node.objectChildren.size() > MAX_SHREDDED_FIELDS) {
- List<Map.Entry<String, PathNode>> sorted = Lists.newArrayList(node.objectChildren.entrySet());
- sorted.sort(
- (a, b) -> {
- int cmp =
- Integer.compare(
- b.getValue().info.observationCount, a.getValue().info.observationCount);
- return cmp != 0 ? cmp : a.getKey().compareTo(b.getKey());
- });
- Set<String> keep = Sets.newHashSet();
- for (int i = 0; i < MAX_SHREDDED_FIELDS; i++) {
- keep.add(sorted.get(i).getKey());
+ Comparator<Map.Entry<String, PathNode>> worstFirst =
+ Comparator.<Map.Entry<String, PathNode>>comparingInt(
+ e -> e.getValue().info.observationCount)
+ .thenComparing(Map.Entry::getKey, Comparator.reverseOrder());
+ PriorityQueue<Map.Entry<String, PathNode>> topK =
+ new PriorityQueue<>(MAX_SHREDDED_FIELDS, worstFirst);
+ for (Map.Entry<String, PathNode> entry : node.objectChildren.entrySet()) {
+ if (topK.size() < MAX_SHREDDED_FIELDS) {
+ topK.offer(entry);
+ } else if (worstFirst.compare(entry, topK.peek()) > 0) {
+ topK.poll();
+ topK.offer(entry);
+ }
+ }
+ Set<String> keep = Sets.newHashSetWithExpectedSize(MAX_SHREDDED_FIELDS);
+ for (Map.Entry<String, PathNode> entry : topK) {
+ keep.add(entry.getKey());
}
node.objectChildren.entrySet().removeIf(entry -> !keep.contains(entry.getKey()));
}
@@ -275,8 +282,12 @@ public abstract class VariantShreddingAnalyzer<T, S> {
Types.GroupBuilder<GroupType> builder = Types.buildGroup(Type.Repetition.OPTIONAL);
boolean hasFields = false;
- for (PathNode child : node.objectChildren.values()) {
- Type fieldType = buildFieldGroup(child);
+ // Sort by field name so the emitted schema field order is deterministic.
+ List<Map.Entry<String, PathNode>> sortedChildren =
+ Lists.newArrayList(node.objectChildren.entrySet());
+ sortedChildren.sort(Map.Entry.comparingByKey());
+ for (Map.Entry<String, PathNode> entry : sortedChildren) {
+ Type fieldType = buildFieldGroup(entry.getValue());
if (fieldType != null) {
builder.addField(fieldType);
hasFields = true;
@@ -312,7 +323,7 @@ public abstract class VariantShreddingAnalyzer<T, S> {
private static class PathNode {
private final String fieldName;
- private final Map<String, PathNode> objectChildren = Maps.newTreeMap();
+ private final Map<String, PathNode> objectChildren = Maps.newHashMap();
private PathNode arrayElement = null;
private FieldInfo info = null;
@@ -407,10 +418,14 @@ public abstract class VariantShreddingAnalyzer<T, S> {
/** Tracks occurrence count and types for a single field. */
private static class FieldInfo {
- private final Map<PhysicalType, Integer> typeCounts = Maps.newHashMap();
+ private static final PhysicalType[] PHYSICAL_TYPES = PhysicalType.values();
+
+ private final int[] typeCounts = new int[PHYSICAL_TYPES.length];
private int maxDecimalScale = 0;
private int maxDecimalIntegerDigits = 0;
private int observationCount = 0;
+ private boolean mostCommonComputed = false;
+ private PhysicalType mostCommonCached = null;
private static final Map<PhysicalType, Integer> INTEGER_PRIORITY =
ImmutableMap.of(
@@ -425,6 +440,7 @@ public abstract class VariantShreddingAnalyzer<T, S> {
PhysicalType.DECIMAL8, 1,
PhysicalType.DECIMAL16, 2);
+ /** Tie-break ordering when two physical types have equal counts. Higher value wins. */
private static final Map<PhysicalType, Integer> TIE_BREAK_PRIORITY =
ImmutableMap.<PhysicalType, Integer>builder()
.put(PhysicalType.BOOLEAN_TRUE, 0)
@@ -449,20 +465,17 @@ public abstract class VariantShreddingAnalyzer<T, S> {
.buildOrThrow();
void observe(VariantValue value) {
+ mostCommonComputed = false;
observationCount++;
// Use BOOLEAN_TRUE for both TRUE/FALSE values
PhysicalType type =
value.type() == PhysicalType.BOOLEAN_FALSE ? PhysicalType.BOOLEAN_TRUE : value.type();
- typeCounts.compute(type, (k, v) -> (v == null) ? 1 : v + 1);
+ typeCounts[type.ordinal()]++;
// Track max precision and scale for decimal types
- if (type == PhysicalType.DECIMAL4
- || type == PhysicalType.DECIMAL8
- || type == PhysicalType.DECIMAL16) {
- VariantPrimitive<?> primitive = value.asPrimitive();
- Object decimalValue = primitive.get();
- if (decimalValue instanceof BigDecimal bd) {
+ if (isDecimalType(type)) {
+ if (value.asPrimitive().get() instanceof BigDecimal bd) {
maxDecimalIntegerDigits = Math.max(maxDecimalIntegerDigits, bd.precision() - bd.scale());
maxDecimalScale = Math.max(maxDecimalScale, bd.scale());
}
@@ -470,6 +483,10 @@ public abstract class VariantShreddingAnalyzer<T, S> {
}
PhysicalType getMostCommonType() {
+ if (mostCommonComputed) {
+ return mostCommonCached;
+ }
+
Map<PhysicalType, Integer> combinedCounts = Maps.newHashMap();
int integerTotalCount = 0;
@@ -478,9 +495,12 @@ public abstract class VariantShreddingAnalyzer<T, S> {
int decimalTotalCount = 0;
PhysicalType mostCapableDecimal = null;
- for (Map.Entry<PhysicalType, Integer> entry : typeCounts.entrySet()) {
- PhysicalType type = entry.getKey();
- int count = entry.getValue();
+ for (int i = 0; i < typeCounts.length; i++) {
+ int count = typeCounts[i];
+ if (count == 0) {
+ continue;
+ }
+ PhysicalType type = PHYSICAL_TYPES[i];
if (isIntegerType(type)) {
integerTotalCount += count;
@@ -508,12 +528,16 @@ public abstract class VariantShreddingAnalyzer<T, S> {
}
// Pick the most common type with tie-breaking
- return combinedCounts.entrySet().stream()
- .max(
- Map.Entry.<PhysicalType, Integer>comparingByValue()
- .thenComparingInt(entry -> TIE_BREAK_PRIORITY.getOrDefault(entry.getKey(), -1)))
- .map(Map.Entry::getKey)
- .orElse(null);
+ mostCommonCached =
+ combinedCounts.entrySet().stream()
+ .max(
+ Map.Entry.<PhysicalType, Integer>comparingByValue()
+ .thenComparingInt(
+ entry -> TIE_BREAK_PRIORITY.getOrDefault(entry.getKey(), -1)))
+ .map(Map.Entry::getKey)
+ .orElse(null);
+ mostCommonComputed = true;
+ return mostCommonCached;
}
private static boolean isIntegerType(PhysicalType type) {
diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantShreddingAnalyzer.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantShreddingAnalyzer.java
index 5ac10f74cc..e13b9b27a6 100644
--- a/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantShreddingAnalyzer.java
+++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantShreddingAnalyzer.java
@@ -23,6 +23,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import java.math.BigDecimal;
import java.util.List;
import java.util.Locale;
+import java.util.UUID;
import java.util.function.Function;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.variants.ShreddedObject;
@@ -133,6 +134,9 @@ public class TestVariantShreddingAnalyzer {
assertThat(schema).isInstanceOf(GroupType.class);
GroupType typedValue = (GroupType) schema;
assertThat(typedValue.getFieldCount()).isLessThanOrEqualTo(300).isGreaterThan(0);
+ assertThat(typedValue.containsField("field_0000")).isTrue();
+ assertThat(typedValue.containsField("field_0299")).isTrue();
+ assertThat(typedValue.containsField("field_0300")).isFalse();
}
@Test
@@ -408,6 +412,28 @@ public class TestVariantShreddingAnalyzer {
assertThat(elementFields.containsField("key")).isTrue();
}
+ @Test
+ public void testUuidFieldIsTrackedAndShredded() {
+ VariantMetadata meta = Variants.metadata("id");
+ List<VariantValue> rows = Lists.newArrayList();
+ for (int i = 0; i < 100; i++) {
+ ShreddedObject obj = Variants.object(meta);
+ obj.put("id", Variants.ofUUID(UUID.randomUUID()));
+ rows.add(obj);
+ }
+
+ DirectAnalyzer analyzer = new DirectAnalyzer();
+ Type schema = analyzer.analyzeAndCreateSchema(rows, 0);
+
+ assertThat(schema).isNotNull();
+ GroupType typedValue = (GroupType) schema;
+ assertThat(typedValue.containsField("id")).isTrue();
+ GroupType idGroup = typedValue.getType("id").asGroupType();
+ PrimitiveType idTyped = idGroup.getType("typed_value").asPrimitiveType();
+ assertThat(idTyped.getLogicalTypeAnnotation())
+ .isInstanceOf(LogicalTypeAnnotation.UUIDLogicalTypeAnnotation.class);
+ }
+
/**
* Builds 100 variant rows where "common" appears in every row and "rare" appears in only {@code
* rareCount} rows (below MIN_FIELD_FREQUENCY = 0.10 when rareCount < 10).