You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by pv...@apache.org on 2026/06/10 13:02:41 UTC
(iceberg) branch main updated: Data: Add TCK coverage for reader default values (#16638)
This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 491fc36ad4 Data: Add TCK coverage for reader default values (#16638)
491fc36ad4 is described below
commit 491fc36ad439d8eb31a5cd0cb043bec4e25d3b0c
Author: Joy Haldar <jo...@gmail.com>
AuthorDate: Wed Jun 10 18:32:28 2026 +0530
Data: Add TCK coverage for reader default values (#16638)
---
.../apache/iceberg/data/BaseFormatModelTests.java | 412 +++++++++++++++++++--
.../org/apache/iceberg/data/DataGenerators.java | 76 ++++
2 files changed, 462 insertions(+), 26 deletions(-)
diff --git a/data/src/test/java/org/apache/iceberg/data/BaseFormatModelTests.java b/data/src/test/java/org/apache/iceberg/data/BaseFormatModelTests.java
index ad8fe31266..d7ad9d2624 100644
--- a/data/src/test/java/org/apache/iceberg/data/BaseFormatModelTests.java
+++ b/data/src/test/java/org/apache/iceberg/data/BaseFormatModelTests.java
@@ -39,6 +39,7 @@ import java.util.UUID;
import java.util.function.BiConsumer;
import java.util.function.BiFunction;
import java.util.function.Function;
+import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.apache.iceberg.ContentFile;
import org.apache.iceberg.DataFile;
@@ -79,6 +80,7 @@ import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.apache.iceberg.types.Comparators;
import org.apache.iceberg.types.Conversions;
import org.apache.iceberg.types.Type;
@@ -668,48 +670,406 @@ public abstract class BaseFormatModelTests<T> {
@ParameterizedTest
@FieldSource("FILE_FORMATS")
- void testReaderSchemaEvolutionNewColumnWithDefault(FileFormat fileFormat) throws IOException {
+ void testDefaultValues(FileFormat fileFormat) throws IOException {
+ assumeSupports(fileFormat, FEATURE_READER_DEFAULT);
+
+ Types.NestedField idField = Types.NestedField.required(1, "id", Types.LongType.get());
+
+ Types.NestedField dataField =
+ Types.NestedField.optional("data")
+ .withId(2)
+ .ofType(Types.StringType.get())
+ .withInitialDefault(Literal.of("wrong!"))
+ .withDoc("Should not produce default value")
+ .build();
+
+ Schema writeSchema = new Schema(idField, dataField);
+
+ List<Record> genericRecords = RandomGenericData.generate(writeSchema, 10, 1L);
+ writeGenericRecords(fileFormat, writeSchema, genericRecords);
+
+ Schema expectedSchema =
+ new Schema(
+ idField,
+ dataField,
+ Types.NestedField.required("missing_str")
+ .withId(6)
+ .ofType(Types.StringType.get())
+ .withInitialDefault(Literal.of("orange"))
+ .build(),
+ Types.NestedField.optional("missing_int")
+ .withId(7)
+ .ofType(Types.IntegerType.get())
+ .withInitialDefault(Literal.of(34))
+ .build());
+
+ readAndAssertEngineRecords(
+ fileFormat,
+ expectedSchema,
+ genericRecords,
+ record -> {
+ Record expected = GenericRecord.create(expectedSchema);
+ expected.setField("id", record.getField("id"));
+ expected.setField("data", record.getField("data"));
+ expected.setField("missing_str", "orange");
+ expected.setField("missing_int", 34);
+
+ return expected;
+ });
+ }
+ @ParameterizedTest
+ @FieldSource("FILE_FORMATS")
+ void testNullDefaultValue(FileFormat fileFormat) throws IOException {
assumeSupports(fileFormat, FEATURE_READER_DEFAULT);
- DataGenerator dataGenerator = new DataGenerators.DefaultSchema();
- Schema writeSchema = dataGenerator.schema();
- List<Record> genericRecords = dataGenerator.generateRecords();
+ Types.NestedField idField = Types.NestedField.required(1, "id", Types.LongType.get());
+ Types.NestedField dataField =
+ Types.NestedField.optional("data")
+ .withId(2)
+ .ofType(Types.StringType.get())
+ .withInitialDefault(Literal.of("wrong!"))
+ .withDoc("Should not produce default value")
+ .build();
+
+ Schema writeSchema = new Schema(idField, dataField);
+
+ List<Record> genericRecords = RandomGenericData.generate(writeSchema, 10, 1L);
writeGenericRecords(fileFormat, writeSchema, genericRecords);
- String defaultStringValue = "default_value";
- int defaultIntValue = 42;
+ Schema expectedSchema =
+ new Schema(
+ idField,
+ dataField,
+ Types.NestedField.optional("missing_date")
+ .withId(3)
+ .ofType(Types.DateType.get())
+ .build());
- int maxFieldId =
- writeSchema.columns().stream().mapToInt(Types.NestedField::fieldId).max().orElse(0);
+ readAndAssertEngineRecords(
+ fileFormat,
+ expectedSchema,
+ genericRecords,
+ record -> {
+ Record expected = GenericRecord.create(expectedSchema);
+ expected.setField("id", record.getField("id"));
+ expected.setField("data", record.getField("data"));
+ expected.setField("missing_date", null);
- List<Types.NestedField> evolvedColumns = Lists.newArrayList(writeSchema.columns());
- evolvedColumns.add(
- Types.NestedField.required("col_f")
- .withId(maxFieldId + 1)
+ return expected;
+ });
+ }
+
+ @ParameterizedTest
+ @FieldSource("FILE_FORMATS")
+ void testNestedDefaultValue(FileFormat fileFormat) throws IOException {
+ assumeSupports(fileFormat, FEATURE_READER_DEFAULT);
+
+ Types.NestedField idField = Types.NestedField.required(1, "id", Types.LongType.get());
+ Types.NestedField dataField =
+ Types.NestedField.optional("data")
+ .withId(2)
.ofType(Types.StringType.get())
- .withInitialDefault(Literal.of(defaultStringValue))
- .build());
- evolvedColumns.add(
- Types.NestedField.optional("col_g")
- .withId(maxFieldId + 2)
- .ofType(Types.IntegerType.get())
- .withInitialDefault(Literal.of(defaultIntValue))
- .build());
+ .withInitialDefault(Literal.of("wrong!"))
+ .withDoc("Should not produce default value")
+ .build();
+
+ Schema writeSchema =
+ new Schema(
+ idField,
+ dataField,
+ Types.NestedField.optional("nested")
+ .withId(3)
+ .ofType(
+ Types.StructType.of(
+ Types.NestedField.required(4, "inner", Types.StringType.get())))
+ .withDoc("Used to test nested field defaults")
+ .build());
+
+ List<Record> genericRecords = RandomGenericData.generate(writeSchema, 10, 1L);
+ writeGenericRecords(fileFormat, writeSchema, genericRecords);
- Schema evolvedSchema = new Schema(evolvedColumns);
- readAndAssertGenericRecords(
+ Schema expectedSchema =
+ new Schema(
+ idField,
+ dataField,
+ Types.NestedField.optional("nested")
+ .withId(3)
+ .ofType(
+ Types.StructType.of(
+ Types.NestedField.required(4, "inner", Types.StringType.get()),
+ Types.NestedField.optional("missing_inner_float")
+ .withId(5)
+ .ofType(Types.FloatType.get())
+ .withInitialDefault(Literal.of(-0.0F))
+ .build()))
+ .withDoc("Used to test nested field defaults")
+ .build());
+
+ readAndAssertEngineRecords(
fileFormat,
- evolvedSchema,
+ expectedSchema,
genericRecords,
record -> {
- Record expected = copy(record, writeSchema, evolvedSchema);
- expected.setField("col_f", defaultStringValue);
- expected.setField("col_g", defaultIntValue);
+ Record expected = copy(record, writeSchema, expectedSchema);
+
+ Record writtenNested = (Record) record.getField("nested");
+ if (writtenNested != null) {
+ Record expectedNested =
+ GenericRecord.create(expectedSchema.findField("nested").type().asStructType());
+ expectedNested.setField("inner", writtenNested.getField("inner"));
+ expectedNested.setField("missing_inner_float", -0.0F);
+ expected.setField("nested", expectedNested);
+ }
+
return expected;
});
}
+ @ParameterizedTest
+ @FieldSource("FILE_FORMATS")
+ void testMapNestedDefaultValue(FileFormat fileFormat) throws IOException {
+ assumeSupports(fileFormat, FEATURE_READER_DEFAULT);
+
+ Types.NestedField idField = Types.NestedField.required(1, "id", Types.LongType.get());
+ Types.NestedField dataField =
+ Types.NestedField.optional("data")
+ .withId(2)
+ .ofType(Types.StringType.get())
+ .withInitialDefault(Literal.of("wrong!"))
+ .withDoc("Should not produce default value")
+ .build();
+
+ Schema writeSchema =
+ new Schema(
+ idField,
+ dataField,
+ Types.NestedField.optional("nested_map")
+ .withId(3)
+ .ofType(
+ Types.MapType.ofOptional(
+ 4,
+ 5,
+ Types.StringType.get(),
+ Types.StructType.of(
+ Types.NestedField.required(6, "value_str", Types.StringType.get()))))
+ .withDoc("Used to test nested map value field defaults")
+ .build());
+
+ List<Record> genericRecords = RandomGenericData.generate(writeSchema, 10, 1L);
+ writeGenericRecords(fileFormat, writeSchema, genericRecords);
+
+ Schema expectedSchema =
+ new Schema(
+ idField,
+ dataField,
+ Types.NestedField.optional("nested_map")
+ .withId(3)
+ .ofType(
+ Types.MapType.ofOptional(
+ 4,
+ 5,
+ Types.StringType.get(),
+ Types.StructType.of(
+ Types.NestedField.required(6, "value_str", Types.StringType.get()),
+ Types.NestedField.optional("value_int")
+ .withId(7)
+ .ofType(Types.IntegerType.get())
+ .withInitialDefault(Literal.of(34))
+ .build())))
+ .withDoc("Used to test nested field defaults")
+ .build());
+
+ readAndAssertEngineRecords(
+ fileFormat,
+ expectedSchema,
+ genericRecords,
+ record -> {
+ Record expected = copy(record, writeSchema, expectedSchema);
+
+ @SuppressWarnings("unchecked")
+ Map<Object, Record> writtenMap = (Map<Object, Record>) expected.getField("nested_map");
+ if (writtenMap != null) {
+ Types.StructType valueType =
+ expectedSchema
+ .findField("nested_map")
+ .type()
+ .asMapType()
+ .valueType()
+ .asStructType();
+ Map<Object, Record> rebuilt = Maps.newLinkedHashMap();
+ writtenMap.forEach(
+ (key, value) ->
+ rebuilt.put(
+ key,
+ value == null
+ ? null
+ : GenericRecord.create(valueType)
+ .copy("value_str", value.getField("value_str"), "value_int", 34)));
+ expected.setField("nested_map", rebuilt);
+ }
+
+ return expected;
+ });
+ }
+
+ @ParameterizedTest
+ @FieldSource("FILE_FORMATS")
+ void testListNestedDefaultValue(FileFormat fileFormat) throws IOException {
+ assumeSupports(fileFormat, FEATURE_READER_DEFAULT);
+
+ Types.NestedField idField = Types.NestedField.required(1, "id", Types.LongType.get());
+ Types.NestedField dataField =
+ Types.NestedField.optional("data")
+ .withId(2)
+ .ofType(Types.StringType.get())
+ .withInitialDefault(Literal.of("wrong!"))
+ .withDoc("Should not produce default value")
+ .build();
+
+ Schema writeSchema =
+ new Schema(
+ idField,
+ dataField,
+ Types.NestedField.optional("nested_list")
+ .withId(3)
+ .ofType(
+ Types.ListType.ofOptional(
+ 4,
+ Types.StructType.of(
+ Types.NestedField.required(5, "element_str", Types.StringType.get()))))
+ .withDoc("Used to test nested field defaults")
+ .build());
+
+ List<Record> genericRecords = RandomGenericData.generate(writeSchema, 10, 1L);
+ writeGenericRecords(fileFormat, writeSchema, genericRecords);
+
+ Schema expectedSchema =
+ new Schema(
+ idField,
+ dataField,
+ Types.NestedField.optional("nested_list")
+ .withId(3)
+ .ofType(
+ Types.ListType.ofOptional(
+ 4,
+ Types.StructType.of(
+ Types.NestedField.required(5, "element_str", Types.StringType.get()),
+ Types.NestedField.optional("element_int")
+ .withId(7)
+ .ofType(Types.IntegerType.get())
+ .withInitialDefault(Literal.of(34))
+ .build())))
+ .withDoc("Used to test nested field defaults")
+ .build());
+
+ readAndAssertEngineRecords(
+ fileFormat,
+ expectedSchema,
+ genericRecords,
+ record -> {
+ Record expected = copy(record, writeSchema, expectedSchema);
+
+ @SuppressWarnings("unchecked")
+ List<Record> writtenList = (List<Record>) expected.getField("nested_list");
+ if (writtenList != null) {
+ Types.StructType elementType =
+ expectedSchema
+ .findField("nested_list")
+ .type()
+ .asListType()
+ .elementType()
+ .asStructType();
+ List<Record> rebuilt =
+ writtenList.stream()
+ .map(
+ element ->
+ element == null
+ ? null
+ : GenericRecord.create(elementType)
+ .copy(
+ "element_str",
+ element.getField("element_str"),
+ "element_int",
+ 34))
+ .collect(Collectors.toList());
+ expected.setField("nested_list", rebuilt);
+ }
+
+ return expected;
+ });
+ }
+
+ @ParameterizedTest
+ @FieldSource("FILE_FORMATS")
+ void testMissingRequiredWithoutDefault(FileFormat fileFormat) throws IOException {
+ assumeSupports(fileFormat, FEATURE_READER_DEFAULT);
+
+ Schema writeSchema = new Schema(Types.NestedField.required(1, "id", Types.LongType.get()));
+
+ List<Record> genericRecords = RandomGenericData.generate(writeSchema, 10, 1L);
+ writeGenericRecords(fileFormat, writeSchema, genericRecords);
+
+ Schema expectedSchema =
+ new Schema(
+ Types.NestedField.required(1, "id", Types.LongType.get()),
+ Types.NestedField.required("missing_str")
+ .withId(6)
+ .ofType(Types.StringType.get())
+ .withDoc("Missing required field with no default")
+ .build());
+
+ assertThatThrownBy(
+ () ->
+ readAndAssertEngineRecords(
+ fileFormat, expectedSchema, genericRecords, record -> record))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Missing required field: missing_str");
+ }
+
+ @ParameterizedTest
+ @FieldSource("FILE_FORMATS")
+ void testPrimitiveDefaultValues(FileFormat fileFormat) throws IOException {
+ assumeSupports(fileFormat, FEATURE_READER_DEFAULT);
+
+ Schema writeSchema = DataGenerators.PrimitiveDefaults.WRITE_SCHEMA;
+ Schema readSchema = filterUnsupported(DataGenerators.PrimitiveDefaults.READ_SCHEMA);
+
+ List<Record> sourceRecords = RandomGenericData.generate(writeSchema, 10, 1L);
+ writeGenericRecords(fileFormat, writeSchema, sourceRecords);
+
+ readAndAssertEngineRecords(
+ fileFormat,
+ readSchema,
+ sourceRecords,
+ record -> {
+ Record expected = GenericRecord.create(readSchema);
+ expected.setField("id", record.getField("id"));
+ for (Types.NestedField field : readSchema.columns()) {
+ if (!field.name().equals("id")) {
+ expected.setField(
+ field.name(),
+ GenericDataUtil.internalToGeneric(field.type(), field.initialDefault()));
+ }
+ }
+
+ return expected;
+ });
+ }
+
+ @ParameterizedTest
+ @FieldSource("FILE_FORMATS")
+ void testPrimitiveDefaultValuesNotApplied(FileFormat fileFormat) throws IOException {
+ assumeSupports(fileFormat, FEATURE_READER_DEFAULT);
+
+ Schema readSchema = filterUnsupported(DataGenerators.PrimitiveDefaults.READ_SCHEMA);
+
+ List<Record> sourceRecords = RandomGenericData.generate(readSchema, 10, 1L);
+ writeGenericRecords(fileFormat, readSchema, sourceRecords);
+
+ readAndAssertEngineRecords(fileFormat, readSchema, sourceRecords, record -> record);
+ }
+
@ParameterizedTest
@FieldSource("FILE_FORMATS")
void testReaderBuilderRecordsPerBatchNotSupported(FileFormat fileFormat) throws IOException {
diff --git a/data/src/test/java/org/apache/iceberg/data/DataGenerators.java b/data/src/test/java/org/apache/iceberg/data/DataGenerators.java
index b5a9de23e6..a78dccd052 100644
--- a/data/src/test/java/org/apache/iceberg/data/DataGenerators.java
+++ b/data/src/test/java/org/apache/iceberg/data/DataGenerators.java
@@ -20,8 +20,13 @@ package org.apache.iceberg.data;
import static org.apache.iceberg.types.Types.NestedField.required;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
import org.apache.iceberg.Schema;
+import org.apache.iceberg.expressions.Literal;
+import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.DateTimeUtil;
/**
* Test data generators for different schema types. Add new generators to ALL array to include them
@@ -244,4 +249,75 @@ class DataGenerators {
return schema;
}
}
+
+ // Generator for reader default-value tests across primitive types.
+ // FIXED is excluded: Spark's InternalRowConverter expects a ByteBuffer but the generator produces
+ // byte[] (ClassCastException).
+ // TODO: include FIXED once Spark's converter handles it.
+ static class PrimitiveDefaults implements DataGenerator {
+ static final Schema READ_SCHEMA =
+ new Schema(
+ required(1, "id", Types.LongType.get()),
+ optionalWithDefault(2, "bool_with_default", Types.BooleanType.get(), Literal.of(false)),
+ optionalWithDefault(3, "int_with_default", Types.IntegerType.get(), Literal.of(34)),
+ optionalWithDefault(
+ 4, "long_with_default", Types.LongType.get(), Literal.of(4900000000L)),
+ optionalWithDefault(5, "float_with_default", Types.FloatType.get(), Literal.of(12.21F)),
+ optionalWithDefault(
+ 6, "double_with_default", Types.DoubleType.get(), Literal.of(-0.0D)),
+ optionalWithDefault(
+ 7,
+ "date_with_default",
+ Types.DateType.get(),
+ Literal.of(DateTimeUtil.isoDateToDays("2024-12-17"))),
+ optionalWithDefault(
+ 8,
+ "timestamptz_with_default",
+ Types.TimestampType.withZone(),
+ Literal.of(
+ DateTimeUtil.isoTimestamptzToMicros("2024-12-17T23:59:59.999999+00:00"))),
+ optionalWithDefault(
+ 9,
+ "timestamp_with_default",
+ Types.TimestampType.withoutZone(),
+ Literal.of(DateTimeUtil.isoTimestampToMicros("2024-12-17T23:59:59.999999"))),
+ optionalWithDefault(
+ 10, "string_with_default", Types.StringType.get(), Literal.of("iceberg")),
+ optionalWithDefault(
+ 11,
+ "uuid_with_default",
+ Types.UUIDType.get(),
+ Literal.of(java.util.UUID.randomUUID())),
+ optionalWithDefault(
+ 12,
+ "binary_with_default",
+ Types.BinaryType.get(),
+ Literal.of(ByteBuffer.wrap(new byte[] {0x0a, 0x0b}))),
+ optionalWithDefault(
+ 13,
+ "decimal_with_default",
+ Types.DecimalType.of(9, 2),
+ Literal.of(new BigDecimal("12.34"))),
+ optionalWithDefault(
+ 14,
+ "time_with_default",
+ Types.TimeType.get(),
+ Literal.of(DateTimeUtil.isoTimeToMicros("23:59:59.999999"))));
+
+ static final Schema WRITE_SCHEMA = new Schema(required(1, "id", Types.LongType.get()));
+
+ private static Types.NestedField optionalWithDefault(
+ int id, String name, Type type, Literal<?> initialDefault) {
+ return Types.NestedField.optional(name)
+ .withId(id)
+ .ofType(type)
+ .withInitialDefault(initialDefault)
+ .build();
+ }
+
+ @Override
+ public Schema schema() {
+ return READ_SCHEMA;
+ }
+ }
}