You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ao...@apache.org on 2022/04/13 20:14:55 UTC
[iceberg] branch master updated: Core: Fix filter pushdown for metadata tables with evolved specs (#4520)
This is an automated email from the ASF dual-hosted git repository.
aokolnychyi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new a2392355e Core: Fix filter pushdown for metadata tables with evolved specs (#4520)
a2392355e is described below
commit a2392355e2a9afe757b700409882f26737a5062e
Author: Szehon Ho <sz...@gmail.com>
AuthorDate: Wed Apr 13 13:14:49 2022 -0700
Core: Fix filter pushdown for metadata tables with evolved specs (#4520)
---
.../java/org/apache/iceberg/BaseFilesTable.java | 39 ++-
.../java/org/apache/iceberg/BaseMetadataTable.java | 10 +-
.../java/org/apache/iceberg/PartitionsTable.java | 2 +-
.../apache/iceberg/TestMetadataTableFilters.java | 275 +++++++++++++++++++++
.../TestMetadataTablesWithPartitionEvolution.java | 148 ++++++++++-
5 files changed, 435 insertions(+), 39 deletions(-)
diff --git a/core/src/main/java/org/apache/iceberg/BaseFilesTable.java b/core/src/main/java/org/apache/iceberg/BaseFilesTable.java
index a2c7f1107..b91b05ca2 100644
--- a/core/src/main/java/org/apache/iceberg/BaseFilesTable.java
+++ b/core/src/main/java/org/apache/iceberg/BaseFilesTable.java
@@ -19,16 +19,18 @@
package org.apache.iceberg;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
import java.util.Map;
import org.apache.iceberg.expressions.Expression;
import org.apache.iceberg.expressions.Expressions;
import org.apache.iceberg.expressions.ManifestEvaluator;
-import org.apache.iceberg.expressions.Projections;
import org.apache.iceberg.expressions.ResidualEvaluator;
import org.apache.iceberg.io.CloseableIterable;
import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.apache.iceberg.relocated.com.google.common.collect.Sets;
import org.apache.iceberg.types.TypeUtil;
import org.apache.iceberg.types.Types.StructType;
@@ -92,7 +94,16 @@ abstract class BaseFilesTable extends BaseMetadataTable {
protected CloseableIterable<FileScanTask> planFiles(TableOperations ops, Snapshot snapshot, Expression rowFilter,
boolean ignoreResiduals, boolean caseSensitive,
boolean colStats) {
- CloseableIterable<ManifestFile> filtered = filterManifests(manifests(), rowFilter, caseSensitive);
+ Map<Integer, PartitionSpec> specsById = table().specs();
+
+ LoadingCache<Integer, ManifestEvaluator> evalCache = Caffeine.newBuilder().build(specId -> {
+ PartitionSpec spec = specsById.get(specId);
+ PartitionSpec transformedSpec = transformSpec(fileSchema, spec);
+ return ManifestEvaluator.forRowFilter(rowFilter, transformedSpec, caseSensitive);
+ });
+
+ CloseableIterable<ManifestFile> filteredManifests = CloseableIterable.filter(manifests(),
+ manifest -> evalCache.get(manifest.partitionSpecId()).eval(manifest));
String schemaString = SchemaParser.toJson(schema());
String specString = PartitionSpecParser.toJson(PartitionSpec.unpartitioned());
@@ -103,28 +114,14 @@ abstract class BaseFilesTable extends BaseMetadataTable {
// This data task needs to use the table schema, which may not include a partition schema to avoid having an
// empty struct in the schema for unpartitioned tables. Some engines, like Spark, can't handle empty structs in
// all cases.
- return CloseableIterable.transform(filtered, manifest ->
- new ManifestReadTask(ops.io(), ops.current().specsById(),
- manifest, schema(), schemaString, specString, residuals));
+ return CloseableIterable.transform(filteredManifests, manifest ->
+ new ManifestReadTask(table(), manifest, schema(), schemaString, specString, residuals));
}
/**
* Returns an iterable of manifest files to explore for this Files metadata table scan
*/
protected abstract CloseableIterable<ManifestFile> manifests();
-
- private CloseableIterable<ManifestFile> filterManifests(CloseableIterable<ManifestFile> manifests,
- Expression rowFilter,
- boolean caseSensitive) {
- // use an inclusive projection to remove the partition name prefix and filter out any non-partition expressions
- PartitionSpec spec = transformSpec(fileSchema, table().spec(), PARTITION_FIELD_PREFIX);
- Expression partitionFilter = Projections.inclusive(spec, caseSensitive).project(rowFilter);
-
- ManifestEvaluator manifestEval = ManifestEvaluator.forPartitionFilter(
- partitionFilter, table().spec(), caseSensitive);
-
- return CloseableIterable.filter(manifests, manifestEval::eval);
- }
}
static class ManifestReadTask extends BaseFileScanTask implements DataTask {
@@ -133,11 +130,11 @@ abstract class BaseFilesTable extends BaseMetadataTable {
private final ManifestFile manifest;
private final Schema schema;
- ManifestReadTask(FileIO io, Map<Integer, PartitionSpec> specsById, ManifestFile manifest,
+ ManifestReadTask(Table table, ManifestFile manifest,
Schema schema, String schemaString, String specString, ResidualEvaluator residuals) {
super(DataFiles.fromManifest(manifest), null, schemaString, specString, residuals);
- this.io = io;
- this.specsById = specsById;
+ this.io = table.io();
+ this.specsById = Maps.newHashMap(table.specs());
this.manifest = manifest;
this.schema = schema;
}
diff --git a/core/src/main/java/org/apache/iceberg/BaseMetadataTable.java b/core/src/main/java/org/apache/iceberg/BaseMetadataTable.java
index d9b2f797a..f4966352e 100644
--- a/core/src/main/java/org/apache/iceberg/BaseMetadataTable.java
+++ b/core/src/main/java/org/apache/iceberg/BaseMetadataTable.java
@@ -35,7 +35,6 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
* deserialization.
*/
abstract class BaseMetadataTable implements Table, HasTableOperations, Serializable {
- protected static final String PARTITION_FIELD_PREFIX = "partition.";
private final PartitionSpec spec = PartitionSpec.unpartitioned();
private final SortOrder sortOrder = SortOrder.unsorted();
private final TableOperations ops;
@@ -52,18 +51,17 @@ abstract class BaseMetadataTable implements Table, HasTableOperations, Serializa
* This method transforms the table's partition spec to a spec that is used to rewrite the user-provided filter
* expression against the given metadata table.
* <p>
- * The resulting partition spec maps $partitionPrefix.X fields to partition X using an identity partition transform.
+ * The resulting partition spec maps partition.X fields to partition X using an identity partition transform.
* When this spec is used to project an expression for the given metadata table, the projection will remove
- * predicates for non-partition fields (not in the spec) and will remove the "$partitionPrefix." prefix from fields.
+ * predicates for non-partition fields (not in the spec) and will remove the "partition." prefix from fields.
*
* @param metadataTableSchema schema of the metadata table
* @param spec spec on which the metadata table schema is based
- * @param partitionPrefix prefix to remove from each field in the partition spec
* @return a spec used to rewrite the metadata table filters to partition filters using an inclusive projection
*/
- static PartitionSpec transformSpec(Schema metadataTableSchema, PartitionSpec spec, String partitionPrefix) {
+ static PartitionSpec transformSpec(Schema metadataTableSchema, PartitionSpec spec) {
PartitionSpec.Builder identitySpecBuilder = PartitionSpec.builderFor(metadataTableSchema).checkConflicts(false);
- spec.fields().forEach(pf -> identitySpecBuilder.identity(partitionPrefix + pf.name(), pf.name()));
+ spec.fields().forEach(pf -> identitySpecBuilder.add(pf.fieldId(), pf.name(), "identity"));
return identitySpecBuilder.build();
}
diff --git a/core/src/main/java/org/apache/iceberg/PartitionsTable.java b/core/src/main/java/org/apache/iceberg/PartitionsTable.java
index ca81b19f1..45f7581ac 100644
--- a/core/src/main/java/org/apache/iceberg/PartitionsTable.java
+++ b/core/src/main/java/org/apache/iceberg/PartitionsTable.java
@@ -111,7 +111,7 @@ public class PartitionsTable extends BaseMetadataTable {
// use an inclusive projection to remove the partition name prefix and filter out any non-partition expressions
Expression partitionFilter = Projections
- .inclusive(transformSpec(scan.schema(), table.spec(), PARTITION_FIELD_PREFIX), caseSensitive)
+ .inclusive(transformSpec(scan.schema(), table.spec()), caseSensitive)
.project(scan.filter());
ManifestGroup manifestGroup = new ManifestGroup(table.io(), snapshot.dataManifests(), snapshot.deleteManifests())
diff --git a/core/src/test/java/org/apache/iceberg/TestMetadataTableFilters.java b/core/src/test/java/org/apache/iceberg/TestMetadataTableFilters.java
index 81210abf3..9edc379d4 100644
--- a/core/src/test/java/org/apache/iceberg/TestMetadataTableFilters.java
+++ b/core/src/test/java/org/apache/iceberg/TestMetadataTableFilters.java
@@ -28,6 +28,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
import org.apache.iceberg.types.Conversions;
import org.apache.iceberg.types.Types;
import org.junit.Assert;
+import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -63,6 +64,9 @@ public class TestMetadataTableFilters extends TableTestBase {
@Override
public void setupTable() throws Exception {
super.setupTable();
+ table.updateProperties()
+ .set(TableProperties.MANIFEST_MERGE_ENABLED, "false")
+ .commit();
table.newFastAppend()
.appendFile(FILE_A)
.commit();
@@ -252,6 +256,277 @@ public class TestMetadataTableFilters extends TableTestBase {
validateCombinedScanTasks(tasks, 0);
}
+ @Test
+ public void testPartitionSpecEvolutionRemovalV1() {
+ Assume.assumeTrue(formatVersion == 1);
+
+ // Change spec and add two data files
+ table.updateSpec()
+ .removeField(Expressions.bucket("data", 16))
+ .addField("id")
+ .commit();
+ PartitionSpec newSpec = table.spec();
+
+ // Add two data files with new spec
+ PartitionKey data10Key = new PartitionKey(newSpec, table.schema());
+ data10Key.set(1, 10);
+ DataFile data10 = DataFiles.builder(newSpec)
+ .withPath("/path/to/data-10.parquet")
+ .withRecordCount(10)
+ .withFileSizeInBytes(10)
+ .withPartition(data10Key)
+ .build();
+ PartitionKey data11Key = new PartitionKey(newSpec, table.schema());
+ data10Key.set(1, 11);
+ DataFile data11 = DataFiles.builder(newSpec)
+ .withPath("/path/to/data-11.parquet")
+ .withRecordCount(10)
+ .withFileSizeInBytes(10)
+ .withPartition(data11Key)
+ .build();
+
+ table.newFastAppend().appendFile(data10).commit();
+ table.newFastAppend().appendFile(data11).commit();
+
+ if (type.equals(MetadataTableType.ALL_DATA_FILES)) {
+ // Clear all files from current snapshot to test whether 'all' Files tables scans previous files
+ table.newDelete().deleteFromRowFilter(Expressions.alwaysTrue()).commit(); // Moves file entries to DELETED state
+ table.newDelete().deleteFromRowFilter(Expressions.alwaysTrue()).commit(); // Removes all entries
+ Assert.assertEquals("Current snapshot should be made empty",
+ 0, table.currentSnapshot().allManifests().size());
+ }
+
+ Table metadataTable = createMetadataTable();
+ Expression filter = Expressions.and(
+ Expressions.equal("partition.id", 10),
+ Expressions.greaterThan("record_count", 0));
+ TableScan scan = metadataTable.newScan().filter(filter);
+ CloseableIterable<FileScanTask> tasks = scan.planFiles();
+
+ // All 4 original data files written by old spec, plus one data file written by new spec
+ Assert.assertEquals(expectedScanTaskCount(5), Iterables.size(tasks));
+
+ filter = Expressions.and(
+ Expressions.equal("partition.data_bucket", 0),
+ Expressions.greaterThan("record_count", 0));
+ scan = metadataTable.newScan().filter(filter);
+ tasks = scan.planFiles();
+
+ // 1 original data file written by old spec (V1 filters out new specs which don't have this value)
+ Assert.assertEquals(expectedScanTaskCount(1), Iterables.size(tasks));
+ }
+
+ @Test
+ public void testPartitionSpecEvolutionRemovalV2() {
+ Assume.assumeTrue(formatVersion == 2);
+
+ // Change spec and add two data and delete files each
+ table.updateSpec()
+ .removeField(Expressions.bucket("data", 16))
+ .addField("id").commit();
+ PartitionSpec newSpec = table.spec();
+
+ // Add two data files and two delete files with new spec
+ DataFile data10 = DataFiles.builder(newSpec)
+ .withPath("/path/to/data-10.parquet")
+ .withRecordCount(10)
+ .withFileSizeInBytes(10)
+ .withPartitionPath("id=10")
+ .build();
+ DataFile data11 = DataFiles.builder(newSpec)
+ .withPath("/path/to/data-11.parquet")
+ .withRecordCount(10)
+ .withFileSizeInBytes(10)
+ .withPartitionPath("id=11")
+ .build();
+
+ DeleteFile delete10 = FileMetadata.deleteFileBuilder(newSpec)
+ .ofPositionDeletes()
+ .withPath("/path/to/data-10-deletes.parquet")
+ .withFileSizeInBytes(10)
+ .withPartitionPath("id=10")
+ .withRecordCount(1)
+ .build();
+ DeleteFile delete11 = FileMetadata.deleteFileBuilder(newSpec)
+ .ofPositionDeletes()
+ .withPath("/path/to/data-11-deletes.parquet")
+ .withFileSizeInBytes(10)
+ .withPartitionPath("id=11")
+ .withRecordCount(1)
+ .build();
+
+ table.newFastAppend().appendFile(data10).commit();
+ table.newFastAppend().appendFile(data11).commit();
+
+ if (formatVersion == 2) {
+ table.newRowDelta().addDeletes(delete10).commit();
+ table.newRowDelta().addDeletes(delete11).commit();
+ }
+
+ if (type.equals(MetadataTableType.ALL_DATA_FILES)) {
+ // Clear all files from current snapshot to test whether 'all' Files tables scans previous files
+ table.newDelete().deleteFromRowFilter(Expressions.alwaysTrue()).commit(); // Moves file entries to DELETED state
+ table.newDelete().deleteFromRowFilter(Expressions.alwaysTrue()).commit(); // Removes all entries
+ Assert.assertEquals("Current snapshot should be made empty",
+ 0, table.currentSnapshot().allManifests().size());
+ }
+
+ Table metadataTable = createMetadataTable();
+ Expression filter = Expressions.and(
+ Expressions.equal("partition.id", 10),
+ Expressions.greaterThan("record_count", 0));
+ TableScan scan = metadataTable.newScan().filter(filter);
+ CloseableIterable<FileScanTask> tasks = scan.planFiles();
+
+ // All 4 original data/delete files written by old spec, plus one new data file/delete file written by new spec
+ Assert.assertEquals(expectedScanTaskCount(5), Iterables.size(tasks));
+
+ filter = Expressions.and(
+ Expressions.equal("partition.data_bucket", 0),
+ Expressions.greaterThan("record_count", 0));
+ scan = metadataTable.newScan().filter(filter);
+ tasks = scan.planFiles();
+
+ // 1 original data/delete files written by old spec, plus both of new data file/delete file written by new spec
+ Assert.assertEquals(expectedScanTaskCount(3), Iterables.size(tasks));
+ }
+
+ @Test
+ public void testPartitionSpecEvolutionAdditiveV1() {
+ Assume.assumeTrue(formatVersion == 1);
+
+ // Change spec and add two data files
+ table.updateSpec()
+ .addField("id")
+ .commit();
+ PartitionSpec newSpec = table.spec();
+
+ // Add two data files with new spec
+ PartitionKey data10Key = new PartitionKey(newSpec, table.schema());
+ data10Key.set(0, 0); // data=0
+ data10Key.set(1, 10); // id=10
+ DataFile data10 = DataFiles.builder(newSpec)
+ .withPath("/path/to/data-10.parquet")
+ .withRecordCount(10)
+ .withFileSizeInBytes(10)
+ .withPartition(data10Key)
+ .build();
+ PartitionKey data11Key = new PartitionKey(newSpec, table.schema());
+ data11Key.set(0, 1); // data=0
+ data10Key.set(1, 11); // id=11
+ DataFile data11 = DataFiles.builder(newSpec)
+ .withPath("/path/to/data-11.parquet")
+ .withRecordCount(10)
+ .withFileSizeInBytes(10)
+ .withPartition(data11Key)
+ .build();
+
+ table.newFastAppend().appendFile(data10).commit();
+ table.newFastAppend().appendFile(data11).commit();
+
+ if (type.equals(MetadataTableType.ALL_DATA_FILES)) {
+ // Clear all files from current snapshot to test whether 'all' Files tables scans previous files
+ table.newDelete().deleteFromRowFilter(Expressions.alwaysTrue()).commit(); // Moves file entries to DELETED state
+ table.newDelete().deleteFromRowFilter(Expressions.alwaysTrue()).commit(); // Removes all entries
+ Assert.assertEquals("Current snapshot should be made empty",
+ 0, table.currentSnapshot().allManifests().size());
+ }
+
+ Table metadataTable = createMetadataTable();
+ Expression filter = Expressions.and(
+ Expressions.equal("partition.id", 10),
+ Expressions.greaterThan("record_count", 0));
+ TableScan scan = metadataTable.newScan().filter(filter);
+ CloseableIterable<FileScanTask> tasks = scan.planFiles();
+
+ // All 4 original data/delete files written by old spec, plus one new data file written by new spec
+ Assert.assertEquals(expectedScanTaskCount(5), Iterables.size(tasks));
+
+ filter = Expressions.and(
+ Expressions.equal("partition.data_bucket", 0),
+ Expressions.greaterThan("record_count", 0));
+ scan = metadataTable.newScan().filter(filter);
+ tasks = scan.planFiles();
+
+ // 1 original data file written by old spec, plus 1 new data file written by new spec
+ Assert.assertEquals(expectedScanTaskCount(2), Iterables.size(tasks));
+ }
+
+ @Test
+ public void testPartitionSpecEvolutionAdditiveV2() {
+ Assume.assumeTrue(formatVersion == 2);
+
+ // Change spec and add two data and delete files each
+ table.updateSpec()
+ .addField("id")
+ .commit();
+ PartitionSpec newSpec = table.spec();
+
+ // Add two data files and two delete files with new spec
+ DataFile data10 = DataFiles.builder(newSpec)
+ .withPath("/path/to/data-10.parquet")
+ .withRecordCount(10)
+ .withFileSizeInBytes(10)
+ .withPartitionPath("data_bucket=0/id=10")
+ .build();
+ DataFile data11 = DataFiles.builder(newSpec)
+ .withPath("/path/to/data-11.parquet")
+ .withRecordCount(10)
+ .withFileSizeInBytes(10)
+ .withPartitionPath("data_bucket=1/id=11")
+ .build();
+
+ DeleteFile delete10 = FileMetadata.deleteFileBuilder(newSpec)
+ .ofPositionDeletes()
+ .withPath("/path/to/data-10-deletes.parquet")
+ .withFileSizeInBytes(10)
+ .withPartitionPath("data_bucket=0/id=10")
+ .withRecordCount(1)
+ .build();
+ DeleteFile delete11 = FileMetadata.deleteFileBuilder(newSpec)
+ .ofPositionDeletes()
+ .withPath("/path/to/data-11-deletes.parquet")
+ .withFileSizeInBytes(10)
+ .withPartitionPath("data_bucket=1/id=11")
+ .withRecordCount(1)
+ .build();
+
+ table.newFastAppend().appendFile(data10).commit();
+ table.newFastAppend().appendFile(data11).commit();
+
+ if (formatVersion == 2) {
+ table.newRowDelta().addDeletes(delete10).commit();
+ table.newRowDelta().addDeletes(delete11).commit();
+ }
+
+ if (type.equals(MetadataTableType.ALL_DATA_FILES)) {
+ // Clear all files from current snapshot to test whether 'all' Files tables scans previous files
+ table.newDelete().deleteFromRowFilter(Expressions.alwaysTrue()).commit(); // Moves file entries to DELETED state
+ table.newDelete().deleteFromRowFilter(Expressions.alwaysTrue()).commit(); // Removes all entries
+ Assert.assertEquals("Current snapshot should be made empty",
+ 0, table.currentSnapshot().allManifests().size());
+ }
+
+ Table metadataTable = createMetadataTable();
+ Expression filter = Expressions.and(
+ Expressions.equal("partition.id", 10),
+ Expressions.greaterThan("record_count", 0));
+ TableScan scan = metadataTable.newScan().filter(filter);
+ CloseableIterable<FileScanTask> tasks = scan.planFiles();
+
+ // All 4 original data/delete files written by old spec, plus one new data file/delete file written by new spec
+ Assert.assertEquals(expectedScanTaskCount(5), Iterables.size(tasks));
+
+ filter = Expressions.and(
+ Expressions.equal("partition.data_bucket", 0),
+ Expressions.greaterThan("record_count", 0));
+ scan = metadataTable.newScan().filter(filter);
+ tasks = scan.planFiles();
+
+ // 1 original data/delete files written by old spec, plus 1 of new data file/delete file written by new spec
+ Assert.assertEquals(expectedScanTaskCount(2), Iterables.size(tasks));
+ }
+
private void validateFileScanTasks(CloseableIterable<FileScanTask> fileScanTasks, int partValue) {
Assert.assertTrue("File scan tasks do not include correct file",
StreamSupport.stream(fileScanTasks.spliterator(), false).anyMatch(t -> {
diff --git a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java
index 0e690bd27..a93fef0ba 100644
--- a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java
+++ b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java
@@ -22,8 +22,6 @@ package org.apache.iceberg.spark.source;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
-import java.util.Random;
-import java.util.concurrent.ThreadLocalRandom;
import org.apache.iceberg.AssertHelpers;
import org.apache.iceberg.FileFormat;
import org.apache.iceberg.HasTableOperations;
@@ -74,14 +72,40 @@ public class TestMetadataTablesWithPartitionEvolution extends SparkCatalogTestBa
"default-namespace", "default"
),
ORC,
- formatVersion()
+ 1
+ },
+ { "testhive", SparkCatalog.class.getName(),
+ ImmutableMap.of(
+ "type", "hive",
+ "default-namespace", "default"
+ ),
+ ORC,
+ 2
+ },
+ { "testhadoop", SparkCatalog.class.getName(),
+ ImmutableMap.of(
+ "type", "hadoop"
+ ),
+ PARQUET,
+ 1
},
{ "testhadoop", SparkCatalog.class.getName(),
ImmutableMap.of(
"type", "hadoop"
),
PARQUET,
- formatVersion()
+ 2
+ },
+ { "spark_catalog", SparkSessionCatalog.class.getName(),
+ ImmutableMap.of(
+ "type", "hive",
+ "default-namespace", "default",
+ "clients", "1",
+ "parquet-enabled", "false",
+ "cache-enabled", "false" // Spark will delete tables using v1, leaving the cache out of sync
+ ),
+ AVRO,
+ 1
},
{ "spark_catalog", SparkSessionCatalog.class.getName(),
ImmutableMap.of(
@@ -92,17 +116,11 @@ public class TestMetadataTablesWithPartitionEvolution extends SparkCatalogTestBa
"cache-enabled", "false" // Spark will delete tables using v1, leaving the cache out of sync
),
AVRO,
- formatVersion()
+ 2
}
};
}
- private static int formatVersion() {
- return RANDOM.nextInt(2) + 1;
- }
-
- private static final Random RANDOM = ThreadLocalRandom.current();
-
private final FileFormat fileFormat;
private final int formatVersion;
@@ -189,6 +207,106 @@ public class TestMetadataTablesWithPartitionEvolution extends SparkCatalogTestBa
}
}
+ @Test
+ public void testFilesMetadataTableFilter() throws ParseException {
+ sql("CREATE TABLE %s (id bigint NOT NULL, category string, data string) USING iceberg " +
+ "TBLPROPERTIES ('commit.manifest-merge.enabled' 'false')", tableName);
+ initTable();
+
+ sql("INSERT INTO TABLE %s VALUES (1, 'c1', 'd1')", tableName);
+ sql("INSERT INTO TABLE %s VALUES (2, 'c2', 'd2')", tableName);
+
+ // verify the metadata tables while the current spec is still unpartitioned
+ for (MetadataTableType tableType : Arrays.asList(FILES, ALL_DATA_FILES)) {
+ Dataset<Row> df = loadMetadataTable(tableType);
+ Assert.assertTrue("Partition must be skipped", df.schema().getFieldIndex("partition").isEmpty());
+ }
+
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ table.updateSpec()
+ .addField("data")
+ .commit();
+ sql("REFRESH TABLE %s", tableName);
+ sql("INSERT INTO TABLE %s VALUES (1, 'c1', 'd1')", tableName);
+ sql("INSERT INTO TABLE %s VALUES (2, 'c2', 'd2')", tableName);
+
+ // verify the metadata tables after adding the first partition column
+ for (MetadataTableType tableType : Arrays.asList(FILES, ALL_DATA_FILES)) {
+ assertPartitions(
+ ImmutableList.of(row("d2")),
+ "STRUCT<data:STRING>",
+ tableType,
+ "partition.data = 'd2'");
+ }
+
+ table.updateSpec()
+ .addField("category")
+ .commit();
+ sql("REFRESH TABLE %s", tableName);
+ sql("INSERT INTO TABLE %s VALUES (1, 'c1', 'd1')", tableName);
+ sql("INSERT INTO TABLE %s VALUES (2, 'c2', 'd2')", tableName);
+
+ // verify the metadata tables after adding the second partition column
+ for (MetadataTableType tableType : Arrays.asList(FILES, ALL_DATA_FILES)) {
+ assertPartitions(ImmutableList.of(row("d2", null), row("d2", "c2")),
+ "STRUCT<data:STRING,category:STRING>",
+ tableType,
+ "partition.data = 'd2'");
+ }
+ for (MetadataTableType tableType : Arrays.asList(FILES, ALL_DATA_FILES)) {
+ assertPartitions(
+ ImmutableList.of(row("d2", "c2")),
+ "STRUCT<data:STRING,category:STRING>",
+ tableType,
+ "partition.category = 'c2'");
+ }
+
+ table.updateSpec()
+ .removeField("data")
+ .commit();
+ sql("REFRESH TABLE %s", tableName);
+
+ // Verify new partitions do not show up for removed 'partition.data=d2' query
+ sql("INSERT INTO TABLE %s VALUES (3, 'c3', 'd2')", tableName);
+ sql("INSERT INTO TABLE %s VALUES (4, 'c4', 'd2')", tableName);
+
+ // Verify new partitions do show up for 'partition.category=c2' query
+ sql("INSERT INTO TABLE %s VALUES (5, 'c2', 'd5')", tableName);
+
+ // no new partition should show up for 'data' partition query as partition field has been removed
+ for (MetadataTableType tableType : Arrays.asList(FILES, ALL_DATA_FILES)) {
+ assertPartitions(
+ ImmutableList.of(row("d2", null), row("d2", "c2")),
+ "STRUCT<data:STRING,category:STRING>",
+ tableType,
+ "partition.data = 'd2'");
+ }
+ // new partition shows up from 'category' partition field query
+ for (MetadataTableType tableType : Arrays.asList(FILES, ALL_DATA_FILES)) {
+ assertPartitions(
+ ImmutableList.of(row(null, "c2"), row("d2", "c2")),
+ "STRUCT<data:STRING,category:STRING>",
+ tableType,
+ "partition.category = 'c2'");
+ }
+
+ table.updateSpec()
+ .renameField("category", "category_another_name")
+ .commit();
+ sql("REFRESH TABLE %s", tableName);
+
+ // Verify new partitions do show up for 'category=c2' query
+ sql("INSERT INTO TABLE %s VALUES (6, 'c2', 'd6')", tableName);
+ for (MetadataTableType tableType : Arrays.asList(FILES, ALL_DATA_FILES)) {
+ assertPartitions(
+ ImmutableList.of(row(null, "c2"), row(null, "c2"), row("d2", "c2")),
+ "STRUCT<data:STRING,category_another_name:STRING>",
+ tableType,
+ "partition.category_another_name = 'c2'");
+ }
+ }
+
@Test
public void testEntriesMetadataTable() throws ParseException {
sql("CREATE TABLE %s (id bigint NOT NULL, category string, data string) USING iceberg", tableName);
@@ -299,7 +417,15 @@ public class TestMetadataTablesWithPartitionEvolution extends SparkCatalogTestBa
private void assertPartitions(List<Object[]> expectedPartitions, String expectedTypeAsString,
MetadataTableType tableType) throws ParseException {
+ assertPartitions(expectedPartitions, expectedTypeAsString, tableType, null);
+ }
+
+ private void assertPartitions(List<Object[]> expectedPartitions, String expectedTypeAsString,
+ MetadataTableType tableType, String filter) throws ParseException {
Dataset<Row> df = loadMetadataTable(tableType);
+ if (filter != null) {
+ df = df.filter(filter);
+ }
DataType expectedType = spark.sessionState().sqlParser().parseDataType(expectedTypeAsString);
switch (tableType) {