You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by im...@apache.org on 2026/07/29 18:15:27 UTC
(asterixdb) 01/05: [NO ISSUE][EXT]: Add Iceberg VARIANT shredding tests
This is an automated email from the ASF dual-hosted git repository.
parshimers pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 037424e2bf181dc3a7b33cc743f3b6b24bd7895b
Author: Hussain Towaileb <Hu...@Couchbase.com>
AuthorDate: Mon Jul 20 07:01:01 2026 +0300
[NO ISSUE][EXT]: Add Iceberg VARIANT shredding tests
Details:
- add tests for fully shreeded variants.
- add tests for partially shredded variants.
Ext-ref: MB-72919
Change-Id: I9be40b9de107bf5cb7df952a77a505d84b9cfc77
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21451
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
Tested-by: Hussain Towaileb <hu...@gmail.com>
Reviewed-by: Michael Blow <mb...@apache.org>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../apache/asterix/test/iceberg/IcebergTest.java | 199 +++++++++++++++++++++
.../variant-partially-shredded/test.000.ddl.sqlpp | 26 +++
.../variant-partially-shredded/test.010.ddl.sqlpp | 42 +++++
.../test.020.query.sqlpp | 22 +++
.../test.030.query.sqlpp | 30 ++++
.../variant-partially-shredded/test.999.ddl.sqlpp | 22 +++
.../iceberg/variant-shredded/test.000.ddl.sqlpp | 26 +++
.../iceberg/variant-shredded/test.010.ddl.sqlpp | 41 +++++
.../iceberg/variant-shredded/test.020.query.sqlpp | 22 +++
.../iceberg/variant-shredded/test.030.query.sqlpp | 30 ++++
.../iceberg/variant-shredded/test.999.ddl.sqlpp | 22 +++
.../variant-partially-shredded/result.020.adm | 12 ++
.../variant-partially-shredded/result.030.adm | 2 +
.../iceberg/variant-shredded/result.020.adm | 12 ++
.../iceberg/variant-shredded/result.030.adm | 2 +
.../test/resources/runtimets/testsuite_iceberg.xml | 12 ++
asterixdb/asterix-external-data/pom.xml | 20 +++
asterixdb/pom.xml | 4 +-
18 files changed, 544 insertions(+), 2 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/iceberg/IcebergTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/iceberg/IcebergTest.java
index 217d859717..ef07fd21e7 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/iceberg/IcebergTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/iceberg/IcebergTest.java
@@ -25,6 +25,8 @@ import static org.apache.asterix.test.cloud_storage.CloudStorageTest.MOCK_SERVER
import static org.apache.hyracks.util.file.FileUtil.joinPath;
import static org.apache.iceberg.types.Types.NestedField.required;
+import java.io.IOException;
+import java.lang.reflect.Method;
import java.math.BigDecimal;
import java.net.URI;
import java.nio.ByteBuffer;
@@ -44,6 +46,8 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
@@ -85,6 +89,7 @@ import org.apache.iceberg.io.FileAppender;
import org.apache.iceberg.io.OutputFile;
import org.apache.iceberg.nessie.NessieCatalog;
import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.parquet.VariantShreddingFunction;
import org.apache.iceberg.types.Types;
import org.apache.iceberg.variants.ShreddedObject;
import org.apache.iceberg.variants.Variant;
@@ -92,7 +97,12 @@ import org.apache.iceberg.variants.VariantMetadata;
import org.apache.iceberg.variants.Variants;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import org.apache.parquet.hadoop.ParquetFileReader;
+import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.Type;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.FixMethodOrder;
import org.junit.Test;
@@ -143,6 +153,9 @@ public class IcebergTest {
private static final TableIdentifier ALL_TYPES_VARIANT_TABLE_ID = TableIdentifier.of(NAMESPACE, "allTypesVariant");
private static final TableIdentifier DEPTH_TEST_VARIANT_TABLE_ID =
TableIdentifier.of(NAMESPACE, "depthTestVariant");
+ private static final TableIdentifier SHREDDED_VARIANT_TABLE_ID = TableIdentifier.of(NAMESPACE, "shreddedVariant");
+ private static final TableIdentifier PARTIALLY_SHREDDED_VARIANT_TABLE_ID =
+ TableIdentifier.of(NAMESPACE, "partiallyShreddedVariant");
protected TestCaseContext tcCtx;
@@ -203,6 +216,8 @@ public class IcebergTest {
writeAllTypesTable(catalog);
writeAllTypesVariantTable(catalog);
writeDepthTestVariantTable(catalog);
+ writeShreddedVariantTable(catalog);
+ writePartiallyShreddedVariantTable(catalog);
}
}
@@ -461,6 +476,190 @@ public class IcebergTest {
return Variant.of(metadata, depthOne);
}
+ // Top-level object fields promoted to typed_value in the PARTIALLY shredded fixture; every other field stays in
+ // the object-level residual value. Deliberately a mix (scalars, a date, a nested object) so the residual carries
+ // scalars, decimals, timestamps, binary, uuid, and the top-level array — the exact path Iceberg 1.10.x corrupted
+ // and 1.11.0 fixed (issue #15086 / PR #15087).
+ private static final Set<String> PARTIAL_SHREDDED_FIELDS =
+ Set.of("int32Value", "stringValue", "boolTrueValue", "doubleValue", "dateValue", "objectValue");
+
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.TEST_GENERATED, notes = "Read-side FULL-shredding counterpart to IcebergVariantSerializedAndShreddedTest: writes the SAME multi-row variant fixture serialized (first half of ids) and FULLY shredded (second half, every field promoted to a typed sub-column), so an ORDER BY id query proves the read pipeline reconstructs identical values whet [...]
+ private static void writeShreddedVariantTable(NessieCatalog catalog) throws Exception {
+ Table table = createTable(catalog, SHREDDED_VARIANT_TABLE_ID, buildAllTypesVariantSchema(),
+ PartitionSpec.unpartitioned());
+ LOGGER.info("[TABLE] name={}", table.name());
+ LOGGER.info("[TABLE] location={}", table.location());
+ LOGGER.info("[TABLE] schema(shredded_variant)={}", table.schema());
+ // The shredding schema is derived from the all-PhysicalType object (row 1) so every supported type gets a
+ // typed sub-column. The serialized file (ids 1..N) and the shredded file (ids N+1..2N) carry the SAME rows, so
+ // matching them across ids proves full shredding reconstructs each value identically to serialized.
+ Set<String> allFields = new TreeSet<>();
+ buildVariantAllPhysicalTypes().value().asObject().fieldNames().forEach(allFields::add);
+ List<Variant> rows = variantFixtureRows();
+ DataFile serialized =
+ writeVariantEncodingFile(table, "shredded_variant_serialized.parquet", 1, rows, null, null);
+ DataFile fullyShredded = writeVariantEncodingFile(table, "shredded_variant_full.parquet", rows.size() + 1,
+ variantFixtureRows(), shreddingFunc(buildVariantAllPhysicalTypes(), null), allFields);
+ table.newAppend().appendFile(serialized).appendFile(fullyShredded).commit();
+ LOGGER.info("[WRITE] shredded_variant table committed with {} record(s)", 2 * rows.size());
+ }
+
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.TEST_GENERATED, notes = "Dedicated PARTIAL-shredding read test: writes the SAME multi-row variant fixture serialized (first half of ids) and PARTIALLY shredded (second half, a subset of fields promoted to typed sub-columns and the rest kept in the object-level residual value), so an ORDER BY id query proves the read pipeline reconstructs [...]
+ private static void writePartiallyShreddedVariantTable(NessieCatalog catalog) throws Exception {
+ Table table = createTable(catalog, PARTIALLY_SHREDDED_VARIANT_TABLE_ID, buildAllTypesVariantSchema(),
+ PartitionSpec.unpartitioned());
+ LOGGER.info("[TABLE] name={}", table.name());
+ LOGGER.info("[TABLE] location={}", table.location());
+ LOGGER.info("[TABLE] schema(partially_shredded_variant)={}", table.schema());
+ List<Variant> rows = variantFixtureRows();
+ DataFile serialized =
+ writeVariantEncodingFile(table, "partially_shredded_variant_serialized.parquet", 1, rows, null, null);
+ DataFile partiallyShredded = writeVariantEncodingFile(table, "partially_shredded_variant_partial.parquet",
+ rows.size() + 1, variantFixtureRows(),
+ shreddingFunc(buildVariantAllPhysicalTypes(), PARTIAL_SHREDDED_FIELDS), PARTIAL_SHREDDED_FIELDS);
+ table.newAppend().appendFile(serialized).appendFile(partiallyShredded).commit();
+ LOGGER.info("[WRITE] partially_shredded_variant table committed with {} record(s)", 2 * rows.size());
+ }
+
+ /**
+ * The rows each encoding file carries. It deliberately varies the <b>top-level Variant PhysicalType</b> across
+ * rows — an object, a bare int, a bare string, an array, a top-level null value, and a null column — rather than
+ * only objects. The shredding schema is object-oriented (derived from the all-types object), so the non-object
+ * rows fall to the residual {@code value} and must still reconstruct, exercising the mixed shredded/residual and
+ * value-as-variant read paths across several records. Fresh instances per call so the same fixture can be written
+ * to more than one file.
+ */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.TEST_GENERATED, notes = "Multi-record fixture varying the top-level PhysicalType: all-types object, bare int, bare string, array, top-level null value, and a null column, so the shredded read path is exercised across heterogeneous rows (typed_value + residual + value-as-variant + null handling)")
+ private static List<Variant> variantFixtureRows() {
+ List<Variant> rows = new ArrayList<>();
+ rows.add(buildVariantAllPhysicalTypes()); // object covering every PhysicalType (shreddable)
+ rows.add(Variant.of(Variants.emptyMetadata(), Variants.of(123))); // bare int primitive
+ rows.add(Variant.of(Variants.emptyMetadata(), Variants.of("top level string"))); // bare string primitive
+ org.apache.iceberg.variants.ValueArray array = Variants.array();
+ array.add(Variants.of(10));
+ array.add(Variants.of("twenty"));
+ array.add(Variants.of(true));
+ rows.add(Variant.of(Variants.emptyMetadata(), array)); // top-level array
+ rows.add(Variant.of(Variants.emptyMetadata(), Variants.ofNull())); // top-level null value (distinct from null column)
+ rows.add(null); // null variant_field column
+ return rows;
+ }
+
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.TEST_GENERATED, notes = "Writes a multi-row Parquet data file (a null element writes a null variant column) using the given shredding function (null = serialized), then verifies the on-disk layout matches expectedShreddedFields (null = no typed_value) so a read test can never silently fall back to testing serialized data")
+ private static DataFile writeVariantEncodingFile(Table table, String fileName, int startId, List<Variant> rowValues,
+ VariantShreddingFunction shreddingFunc, Set<String> expectedShreddedFields) throws Exception {
+ String path = table.location() + "/data/" + fileName;
+ OutputFile out = table.io().newOutputFile(path);
+ Parquet.WriteBuilder builder =
+ Parquet.write(out).schema(table.schema()).createWriterFunc(GenericParquetWriter::create);
+ if (shreddingFunc != null) {
+ builder = builder.variantShreddingFunc(shreddingFunc);
+ }
+ GenericRecord template = GenericRecord.create(table.schema());
+ int id = startId;
+ try (FileAppender<Record> writer = builder.build()) {
+ for (Variant value : rowValues) {
+ Record row = template.copy();
+ row.setField("id", id++);
+ row.setField("variant_field", value);
+ writer.add(row);
+ }
+ }
+ long bytes = out.toInputFile().getLength();
+ // Guard against silently testing serialized data: confirm the file's physical layout on disk matches the
+ // intended encoding before any read test is allowed to rely on it.
+ assertVariantLayoutOnDisk(table, path, expectedShreddedFields);
+ LOGGER.info("[APPEND] file={} sizeBytes={} rows={} shreddedFields={}", path, bytes, rowValues.size(),
+ expectedShreddedFields == null ? "(serialized)" : expectedShreddedFields);
+ return DataFiles.builder(table.spec()).withPath(path).withRecordCount(rowValues.size())
+ .withFileSizeInBytes(bytes).withFormat(FileFormat.PARQUET).build();
+ }
+
+ /**
+ * Returns a {@link VariantShreddingFunction} for {@code value}. When {@code keep} is {@code null} it fully shreds —
+ * every object field is promoted to a typed sub-column, leaving nothing in the object-level residual {@code value}.
+ * When {@code keep} names a subset, only those fields are shredded and the rest stay in the residual value (partial
+ * shredding). Per-field {@code typed_value} Parquet types come from Iceberg's own
+ * {@code ParquetVariantUtil.toParquetSchema} (package-private, reached via reflection) so each of the ~23 Variant
+ * PhysicalTypes is encoded exactly as Iceberg expects; the partial variant just keeps a subset of that group's
+ * fields. Partial shredding round-trips correctly as of Iceberg 1.11.0 (the residual serialization bug in 1.10.x —
+ * issue #15086 / PR #15087 — is fixed).
+ */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.TEST_GENERATED, notes = "Reflectively uses Iceberg's ParquetVariantUtil.toParquetSchema to build a canonical typed_value; keeps all fields (full shred) or a named subset (partial shred, rest to residual)")
+ private static VariantShreddingFunction shreddingFunc(Variant value, Set<String> keep) throws Exception {
+ Method toParquetSchema = Class.forName("org.apache.iceberg.parquet.ParquetVariantUtil")
+ .getDeclaredMethod("toParquetSchema", org.apache.iceberg.variants.VariantValue.class);
+ toParquetSchema.setAccessible(true);
+ GroupType fullTyped = ((Type) toParquetSchema.invoke(null, value.value())).asGroupType();
+ Type typedValue;
+ if (keep == null) {
+ typedValue = fullTyped;
+ } else {
+ org.apache.parquet.schema.Types.GroupBuilder<GroupType> builder =
+ org.apache.parquet.schema.Types.buildGroup(fullTyped.getRepetition());
+ for (Type field : fullTyped.getFields()) {
+ if (keep.contains(field.getName())) {
+ builder.addField(field);
+ }
+ }
+ typedValue = builder.named(fullTyped.getName());
+ }
+ Type tv = typedValue;
+ // Single variant column, so fieldId/name are ignored; the same typed_value applies.
+ return (fieldId, name) -> tv;
+ }
+
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.TEST_GENERATED, notes = "Reads the just-written file's Parquet footer through the table's own FileIO and asserts the variant column's typed_value holds exactly the expected shredded fields (null => serialized, no typed_value), so the fixture can never silently degrade to serialized or shred a different set than intended")
+ private static void assertVariantLayoutOnDisk(Table table, String path, Set<String> expectedShreddedFields)
+ throws Exception {
+ GroupType variant;
+ try (ParquetFileReader reader = ParquetFileReader.open(parquetInput(table.io().newInputFile(path)))) {
+ MessageType schema = reader.getFooter().getFileMetaData().getSchema();
+ variant = schema.getType("variant_field").asGroupType();
+ }
+ // Every encoding keeps the variant metadata and a value slot (residual for shredded, whole blob for serialized).
+ Assert.assertTrue("variant must keep metadata", variant.containsField("metadata"));
+ Assert.assertTrue("variant must keep a value blob", variant.containsField("value"));
+ if (expectedShreddedFields == null) {
+ Assert.assertFalse("serialized variant must NOT have typed_value", variant.containsField("typed_value"));
+ LOGGER.info("[VERIFY] file={} confirmed serialized (no typed_value)", path);
+ return;
+ }
+ Assert.assertTrue("shredded variant must have typed_value", variant.containsField("typed_value"));
+ GroupType typedValue = variant.getType("typed_value").asGroupType();
+ Set<String> actual = new TreeSet<>();
+ typedValue.getFields().forEach(field -> actual.add(field.getName()));
+ Assert.assertEquals("shredded typed_value fields on disk", new TreeSet<>(expectedShreddedFields), actual);
+ LOGGER.info("[VERIFY] file={} confirmed shredded; typed sub-columns={}", path, actual);
+ }
+
+ /** Adapts an Iceberg {@link org.apache.iceberg.io.InputFile} to a Parquet InputFile for footer inspection. */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.TEST_GENERATED, notes = "Bridges Iceberg FileIO to a Parquet InputFile so the shredded-layout check reads the footer through the table's own IO instead of a separate S3 client")
+ private static org.apache.parquet.io.InputFile parquetInput(org.apache.iceberg.io.InputFile in) {
+ return new org.apache.parquet.io.InputFile() {
+ @Override
+ public long getLength() throws IOException {
+ return in.getLength();
+ }
+
+ @Override
+ public org.apache.parquet.io.SeekableInputStream newStream() throws IOException {
+ org.apache.iceberg.io.SeekableInputStream delegate = in.newStream();
+ return new org.apache.parquet.io.DelegatingSeekableInputStream(delegate) {
+ @Override
+ public long getPos() throws IOException {
+ return delegate.getPos();
+ }
+
+ @Override
+ public void seek(long newPos) throws IOException {
+ delegate.seek(newPos);
+ }
+ };
+ }
+ };
+ }
+
private static org.apache.iceberg.nessie.NessieCatalog createNessieCatalog() throws Exception {
LOGGER.info("[CATALOG] Initializing Nessie catalog at {}", NESSIE_URI);
Map<String, String> properties = new HashMap<>();
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-partially-shredded/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-partially-shredded/test.000.ddl.sqlpp
new file mode 100644
index 0000000000..e02f0a42ea
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-partially-shredded/test.000.ddl.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+CREATE CATALOG myNessieCatalog
+TYPE Iceberg
+SOURCE NESSIE
+WITH {
+ "uri": "%NESSIE_SERVER_URI%",
+ "warehouse": "s3://iceberg-container/nessie/warehouse"
+};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-partially-shredded/test.010.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-partially-shredded/test.010.ddl.sqlpp
new file mode 100644
index 0000000000..0a8c8c3b63
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-partially-shredded/test.010.ddl.sqlpp
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// The partiallyShreddedVariant table holds two rows carrying the SAME logical variant object (covering every
+// supported Variant PhysicalType): one row written serialized, one row written PARTIALLY shredded (a subset of
+// fields promoted to typed sub-columns, the rest kept in the object-level residual value). Reading the collection
+// exercises the typed_value + residual merge — the path Iceberg 1.10.x corrupted and 1.11.0 fixed (issue #15086 /
+// PR #15087) — and proves the read pipeline reconstructs identical values regardless of encoding. Full shredding is
+// covered separately by variant-shredded. decimal-to-double is required for the DECIMAL4/8/16 values.
+DROP COLLECTION partiallyShreddedVariant IF EXISTS;
+DROP TYPE partiallyShreddedVariant IF EXISTS;
+
+CREATE TYPE partiallyShreddedVariant AS open {};
+CREATE EXTERNAL COLLECTION partiallyShreddedVariant(partiallyShreddedVariant)
+USING S3
+(
+ ("table-format"="iceberg"),
+ ("namespace"="my_namespace"),
+ ("tableName"="partiallyShreddedVariant"),
+ ("region"="custom-region"),
+ ("serviceEndpoint"="%DOCKER_S3_CONTAINER%"),
+ ("pathStyleAddressing"="true"),
+ ("catalogName"="myNessieCatalog"),
+ ("format"="parquet"),
+ ("decimal-to-double"="true")
+);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-partially-shredded/test.020.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-partially-shredded/test.020.query.sqlpp
new file mode 100644
index 0000000000..ac8c622ed4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-partially-shredded/test.020.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+SELECT id, variant_field
+FROM partiallyShreddedVariant
+ORDER BY id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-partially-shredded/test.030.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-partially-shredded/test.030.query.sqlpp
new file mode 100644
index 0000000000..0016433c7a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-partially-shredded/test.030.query.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+-- Project shredded typed sub-columns, including a 2-level nested path (objectValue.nestedString), so they come
+-- back as addressable, correctly typed fields. Restricted to the two all-types rows (serialized id=1, partially
+-- shredded id=7) via the real id column — a variant sub-field can't be used in a pushed-down predicate, but
+-- projecting one (at any depth) is fine. Both copies must yield identical projected values.
+SELECT id,
+ variant_field.int32Value AS i,
+ variant_field.stringValue AS s,
+ variant_field.objectValue.nestedString AS nested
+FROM partiallyShreddedVariant
+WHERE id = 1 OR id = 7
+ORDER BY id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-partially-shredded/test.999.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-partially-shredded/test.999.ddl.sqlpp
new file mode 100644
index 0000000000..54d5fa77d1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-partially-shredded/test.999.ddl.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP COLLECTION partiallyShreddedVariant;
+DROP TYPE partiallyShreddedVariant;
+DROP CATALOG myNessieCatalog;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-shredded/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-shredded/test.000.ddl.sqlpp
new file mode 100644
index 0000000000..e02f0a42ea
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-shredded/test.000.ddl.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+CREATE CATALOG myNessieCatalog
+TYPE Iceberg
+SOURCE NESSIE
+WITH {
+ "uri": "%NESSIE_SERVER_URI%",
+ "warehouse": "s3://iceberg-container/nessie/warehouse"
+};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-shredded/test.010.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-shredded/test.010.ddl.sqlpp
new file mode 100644
index 0000000000..c7abbfe55a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-shredded/test.010.ddl.sqlpp
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// The shreddedVariant table holds two rows carrying the SAME logical variant object (covering every supported
+// Variant PhysicalType): one row written serialized, one row written FULLY shredded (every field promoted to a
+// typed sub-column). Reading the collection exercises the (fully) shredded reconstruction path and proves the read
+// pipeline is transparent to the on-disk encoding. Partial shredding is covered separately by
+// variant-partially-shredded. decimal-to-double is required for the DECIMAL4/8/16 values.
+DROP COLLECTION shreddedVariant IF EXISTS;
+DROP TYPE shreddedVariant IF EXISTS;
+
+CREATE TYPE shreddedVariant AS open {};
+CREATE EXTERNAL COLLECTION shreddedVariant(shreddedVariant)
+USING S3
+(
+ ("table-format"="iceberg"),
+ ("namespace"="my_namespace"),
+ ("tableName"="shreddedVariant"),
+ ("region"="custom-region"),
+ ("serviceEndpoint"="%DOCKER_S3_CONTAINER%"),
+ ("pathStyleAddressing"="true"),
+ ("catalogName"="myNessieCatalog"),
+ ("format"="parquet"),
+ ("decimal-to-double"="true")
+);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-shredded/test.020.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-shredded/test.020.query.sqlpp
new file mode 100644
index 0000000000..fa1f5de57d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-shredded/test.020.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+SELECT id, variant_field
+FROM shreddedVariant
+ORDER BY id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-shredded/test.030.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-shredded/test.030.query.sqlpp
new file mode 100644
index 0000000000..ec87514fbd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-shredded/test.030.query.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+-- Project shredded typed sub-columns, including a 2-level nested path (objectValue.nestedString), so they come
+-- back as addressable, correctly typed fields. Restricted to the two all-types rows (serialized id=1, fully
+-- shredded id=7) via the real id column — a variant sub-field can't be used in a pushed-down predicate, but
+-- projecting one (at any depth) is fine. Both copies must yield identical projected values.
+SELECT id,
+ variant_field.int32Value AS i,
+ variant_field.stringValue AS s,
+ variant_field.objectValue.nestedString AS nested
+FROM shreddedVariant
+WHERE id = 1 OR id = 7
+ORDER BY id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-shredded/test.999.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-shredded/test.999.ddl.sqlpp
new file mode 100644
index 0000000000..7f59eb3dfb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/variant-shredded/test.999.ddl.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP COLLECTION shreddedVariant;
+DROP TYPE shreddedVariant;
+DROP CATALOG myNessieCatalog;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/iceberg/variant-partially-shredded/result.020.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/iceberg/variant-partially-shredded/result.020.adm
new file mode 100644
index 0000000000..947821f647
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/iceberg/variant-partially-shredded/result.020.adm
@@ -0,0 +1,12 @@
+{ "id": 1, "variant_field": { "arrayValue": [ 1, "two", 3.0, { "deepBool": true } ], "binaryValue": hex("0102030405"), "boolFalseValue": false, "boolTrueValue": true, "dateValue": date("2024-01-01"), "decimal16Value": 1.2345678901234568E20, "decimal4Value": 3.14, "decimal8Value": 1.23456789123E8, "doubleValue": 2.718281828459045, "floatValue": 3.14, "int16Value": 1000, "int32Value": 42, "int64Value": 9223372036854775807, "int8Value": 5, "nullValue": null, "objectValue": { "nestedArray": [...]
+{ "id": 2, "variant_field": 123 }
+{ "id": 3, "variant_field": "top level string" }
+{ "id": 4, "variant_field": [ 10, "twenty", true ] }
+{ "id": 5, "variant_field": null }
+{ "id": 6, "variant_field": null }
+{ "id": 7, "variant_field": { "arrayValue": [ 1, "two", 3.0, { "deepBool": true } ], "binaryValue": hex("0102030405"), "boolFalseValue": false, "boolTrueValue": true, "dateValue": date("2024-01-01"), "decimal16Value": 1.2345678901234568E20, "decimal4Value": 3.14, "decimal8Value": 1.23456789123E8, "doubleValue": 2.718281828459045, "floatValue": 3.14, "int16Value": 1000, "int32Value": 42, "int64Value": 9223372036854775807, "int8Value": 5, "nullValue": null, "objectValue": { "nestedArray": [...]
+{ "id": 8, "variant_field": 123 }
+{ "id": 9, "variant_field": "top level string" }
+{ "id": 10, "variant_field": [ 10, "twenty", true ] }
+{ "id": 11, "variant_field": null }
+{ "id": 12, "variant_field": null }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/iceberg/variant-partially-shredded/result.030.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/iceberg/variant-partially-shredded/result.030.adm
new file mode 100644
index 0000000000..e217eb3fa6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/iceberg/variant-partially-shredded/result.030.adm
@@ -0,0 +1,2 @@
+{ "id": 1, "i": 42, "s": "hello variant", "nested": "inner" }
+{ "id": 7, "i": 42, "s": "hello variant", "nested": "inner" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/iceberg/variant-shredded/result.020.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/iceberg/variant-shredded/result.020.adm
new file mode 100644
index 0000000000..947821f647
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/iceberg/variant-shredded/result.020.adm
@@ -0,0 +1,12 @@
+{ "id": 1, "variant_field": { "arrayValue": [ 1, "two", 3.0, { "deepBool": true } ], "binaryValue": hex("0102030405"), "boolFalseValue": false, "boolTrueValue": true, "dateValue": date("2024-01-01"), "decimal16Value": 1.2345678901234568E20, "decimal4Value": 3.14, "decimal8Value": 1.23456789123E8, "doubleValue": 2.718281828459045, "floatValue": 3.14, "int16Value": 1000, "int32Value": 42, "int64Value": 9223372036854775807, "int8Value": 5, "nullValue": null, "objectValue": { "nestedArray": [...]
+{ "id": 2, "variant_field": 123 }
+{ "id": 3, "variant_field": "top level string" }
+{ "id": 4, "variant_field": [ 10, "twenty", true ] }
+{ "id": 5, "variant_field": null }
+{ "id": 6, "variant_field": null }
+{ "id": 7, "variant_field": { "arrayValue": [ 1, "two", 3.0, { "deepBool": true } ], "binaryValue": hex("0102030405"), "boolFalseValue": false, "boolTrueValue": true, "dateValue": date("2024-01-01"), "decimal16Value": 1.2345678901234568E20, "decimal4Value": 3.14, "decimal8Value": 1.23456789123E8, "doubleValue": 2.718281828459045, "floatValue": 3.14, "int16Value": 1000, "int32Value": 42, "int64Value": 9223372036854775807, "int8Value": 5, "nullValue": null, "objectValue": { "nestedArray": [...]
+{ "id": 8, "variant_field": 123 }
+{ "id": 9, "variant_field": "top level string" }
+{ "id": 10, "variant_field": [ 10, "twenty", true ] }
+{ "id": 11, "variant_field": null }
+{ "id": 12, "variant_field": null }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/iceberg/variant-shredded/result.030.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/iceberg/variant-shredded/result.030.adm
new file mode 100644
index 0000000000..e217eb3fa6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/iceberg/variant-shredded/result.030.adm
@@ -0,0 +1,2 @@
+{ "id": 1, "i": 42, "s": "hello variant", "nested": "inner" }
+{ "id": 7, "i": 42, "s": "hello variant", "nested": "inner" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_iceberg.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_iceberg.xml
index 9a0823a575..99e71eccba 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_iceberg.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_iceberg.xml
@@ -47,6 +47,18 @@
<output-dir compare="Text">variant-all-types</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="iceberg">
+ <compilation-unit name="variant-shredded">
+ <placeholder name="adapter" value="S3" />
+ <output-dir compare="Text">variant-shredded</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="iceberg">
+ <compilation-unit name="variant-partially-shredded">
+ <placeholder name="adapter" value="S3" />
+ <output-dir compare="Text">variant-partially-shredded</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="negative">
<test-case FilePath="iceberg/negative">
diff --git a/asterixdb/asterix-external-data/pom.xml b/asterixdb/asterix-external-data/pom.xml
index becb1370e6..3252d2729c 100644
--- a/asterixdb/asterix-external-data/pom.xml
+++ b/asterixdb/asterix-external-data/pom.xml
@@ -219,6 +219,10 @@
<usedDependency>io.netty:netty-codec-http2</usedDependency>
<usedDependency>io.netty:netty-handler-proxy</usedDependency>
<usedDependency>io.netty:netty-handler-proxy</usedDependency>
+ <!-- iceberg-orc is used only at runtime: Iceberg 1.11.0's FormatModelRegistry eagerly registers the
+ ORC format model when the generic data model reads any table with deletes, so the iceberg-orc
+ classes must be on the classpath even though we never read ORC data and reference no ORC type. -->
+ <usedDependency>org.apache.iceberg:iceberg-orc</usedDependency>
</usedDependencies>
</configuration>
</plugin>
@@ -675,6 +679,22 @@
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-data</artifactId>
</dependency>
+ <!-- Iceberg 1.11.0 introduced FormatModelRegistry, which eagerly registers the Avro, Parquet AND ORC format
+ models (GenericFormatModels#register) the first time the generic data model reads a table with deletes
+ (BaseDeleteLoader) or via IcebergGenerics. Without iceberg-orc on the classpath that static init fails with
+ NoClassDefFoundError: org/apache/iceberg/orc/OrcRowWriter. We never read ORC data, so we pull in only the
+ iceberg-orc classes (needed for the registration to load) and exclude the heavy orc-core / hive-storage-api
+ stack, which is only exercised when actually reading ORC files. -->
+ <dependency>
+ <groupId>org.apache.iceberg</groupId>
+ <artifactId>iceberg-orc</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
<!-- apply patch for HADOOP-17225 to workaround CVE-2019-10172 -->
<repositories>
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index 141f43f68e..86d9b5ce69 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -118,10 +118,10 @@
<io.opencensus.version>0.31.1</io.opencensus.version>
<protobuf-java.version>4.34.2</protobuf-java.version>
- <icebergjavasdk.version>1.10.2</icebergjavasdk.version>
+ <icebergjavasdk.version>1.11.0</icebergjavasdk.version>
<!-- @derived-from: org.apache.iceberg:iceberg-nessie:${icebergjavasdk.version}#dep:nessie-client -->
- <nessieproject.version>0.104.5</nessieproject.version>
+ <nessieproject.version>0.107.5</nessieproject.version>
<implementation.title>Apache AsterixDB - ${project.name}</implementation.title>
<implementation.url>https://asterixdb.apache.org/</implementation.url>