You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ao...@apache.org on 2020/04/23 17:43:14 UTC
[incubator-iceberg] branch master updated: Spark: Implement an
action to rewrite manifests (#875)
This is an automated email from the ASF dual-hosted git repository.
aokolnychyi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 0c8ab3d Spark: Implement an action to rewrite manifests (#875)
0c8ab3d is described below
commit 0c8ab3d856998fee0b03f403630ccb58fcbf9fdd
Author: Anton Okolnychyi <ao...@apple.com>
AuthorDate: Thu Apr 23 10:43:05 2020 -0700
Spark: Implement an action to rewrite manifests (#875)
---
.../java/org/apache/iceberg/actions/Actions.java | 4 +
.../actions/{Actions.java => BaseAction.java} | 32 +-
...{Actions.java => BaseSnapshotUpdateAction.java} | 30 +-
.../iceberg/actions/RemoveOrphanFilesAction.java | 19 +-
.../iceberg/actions/RewriteManifestsAction.java | 375 ++++++++++++++++++
...ions.java => RewriteManifestsActionResult.java} | 32 +-
.../{Actions.java => SnapshotUpdateAction.java} | 26 +-
.../org/apache/iceberg/spark/SparkDataFile.java | 71 ++--
.../apache/iceberg/TestRewriteManifestsAction.java | 440 +++++++++++++++++++++
9 files changed, 918 insertions(+), 111 deletions(-)
diff --git a/spark/src/main/java/org/apache/iceberg/actions/Actions.java b/spark/src/main/java/org/apache/iceberg/actions/Actions.java
index 549d541..f2d386d 100644
--- a/spark/src/main/java/org/apache/iceberg/actions/Actions.java
+++ b/spark/src/main/java/org/apache/iceberg/actions/Actions.java
@@ -43,4 +43,8 @@ public class Actions {
public RemoveOrphanFilesAction removeOrphanFiles() {
return new RemoveOrphanFilesAction(spark, table);
}
+
+ public RewriteManifestsAction rewriteManifests() {
+ return new RewriteManifestsAction(spark, table);
+ }
}
diff --git a/spark/src/main/java/org/apache/iceberg/actions/Actions.java b/spark/src/main/java/org/apache/iceberg/actions/BaseAction.java
similarity index 58%
copy from spark/src/main/java/org/apache/iceberg/actions/Actions.java
copy to spark/src/main/java/org/apache/iceberg/actions/BaseAction.java
index 549d541..67d9714 100644
--- a/spark/src/main/java/org/apache/iceberg/actions/Actions.java
+++ b/spark/src/main/java/org/apache/iceberg/actions/BaseAction.java
@@ -19,28 +19,22 @@
package org.apache.iceberg.actions;
+import org.apache.iceberg.MetadataTableType;
import org.apache.iceberg.Table;
-import org.apache.spark.sql.SparkSession;
-public class Actions {
+abstract class BaseAction<R> implements Action<R> {
- private SparkSession spark;
- private Table table;
+ protected abstract Table table();
- private Actions(SparkSession spark, Table table) {
- this.spark = spark;
- this.table = table;
- }
-
- public static Actions forTable(SparkSession spark, Table table) {
- return new Actions(spark, table);
- }
-
- public static Actions forTable(Table table) {
- return new Actions(SparkSession.active(), table);
- }
-
- public RemoveOrphanFilesAction removeOrphanFiles() {
- return new RemoveOrphanFilesAction(spark, table);
+ protected String metadataTableName(MetadataTableType type) {
+ String tableName = table().toString();
+ if (tableName.contains("/")) {
+ return tableName + "#" + type;
+ } else if (tableName.startsWith("hadoop.") || tableName.startsWith("hive.")) {
+ // HiveCatalog and HadoopCatalog prepend a logical name which we need to drop for Spark 2.4
+ return tableName.replaceFirst("(hadoop\\.)|(hive\\.)", "") + "." + type;
+ } else {
+ return tableName + "." + type;
+ }
}
}
diff --git a/spark/src/main/java/org/apache/iceberg/actions/Actions.java b/spark/src/main/java/org/apache/iceberg/actions/BaseSnapshotUpdateAction.java
similarity index 59%
copy from spark/src/main/java/org/apache/iceberg/actions/Actions.java
copy to spark/src/main/java/org/apache/iceberg/actions/BaseSnapshotUpdateAction.java
index 549d541..b9a8c94 100644
--- a/spark/src/main/java/org/apache/iceberg/actions/Actions.java
+++ b/spark/src/main/java/org/apache/iceberg/actions/BaseSnapshotUpdateAction.java
@@ -19,28 +19,24 @@
package org.apache.iceberg.actions;
-import org.apache.iceberg.Table;
-import org.apache.spark.sql.SparkSession;
+import com.google.common.collect.Maps;
+import java.util.Map;
+import org.apache.iceberg.SnapshotUpdate;
-public class Actions {
+abstract class BaseSnapshotUpdateAction<ThisT, R> extends BaseAction<R> implements SnapshotUpdateAction<ThisT, R> {
- private SparkSession spark;
- private Table table;
+ private final Map<String, String> summary = Maps.newHashMap();
- private Actions(SparkSession spark, Table table) {
- this.spark = spark;
- this.table = table;
- }
-
- public static Actions forTable(SparkSession spark, Table table) {
- return new Actions(spark, table);
- }
+ protected abstract ThisT self();
- public static Actions forTable(Table table) {
- return new Actions(SparkSession.active(), table);
+ @Override
+ public ThisT set(String property, String value) {
+ summary.put(property, value);
+ return self();
}
- public RemoveOrphanFilesAction removeOrphanFiles() {
- return new RemoveOrphanFilesAction(spark, table);
+ protected void commit(SnapshotUpdate<?> update) {
+ summary.forEach(update::set);
+ update.commit();
}
}
diff --git a/spark/src/main/java/org/apache/iceberg/actions/RemoveOrphanFilesAction.java b/spark/src/main/java/org/apache/iceberg/actions/RemoveOrphanFilesAction.java
index aff62ef..b782f08 100644
--- a/spark/src/main/java/org/apache/iceberg/actions/RemoveOrphanFilesAction.java
+++ b/spark/src/main/java/org/apache/iceberg/actions/RemoveOrphanFilesAction.java
@@ -66,7 +66,7 @@ import org.slf4j.LoggerFactory;
* <em>Note:</em> It is dangerous to call this action with a short retention interval as it might corrupt
* the state of the table if another operation is writing at the same time.
*/
-public class RemoveOrphanFilesAction implements Action<List<String>> {
+public class RemoveOrphanFilesAction extends BaseAction<List<String>> {
private static final Logger LOG = LoggerFactory.getLogger(RemoveOrphanFilesAction.class);
@@ -96,6 +96,11 @@ public class RemoveOrphanFilesAction implements Action<List<String>> {
this.location = table.location();
}
+ @Override
+ protected Table table() {
+ return table;
+ }
+
/**
* Removes orphan files in the given location.
*
@@ -250,18 +255,6 @@ public class RemoveOrphanFilesAction implements Action<List<String>> {
}
}
- private String metadataTableName(MetadataTableType type) {
- String tableName = table.toString();
- if (tableName.contains("/")) {
- return tableName + "#" + type;
- } else if (tableName.startsWith("hadoop.") || tableName.startsWith("hive.")) {
- // HiveCatalog and HadoopCatalog prepend a logical name which we need to drop for Spark 2.4
- return tableName.replaceFirst("(hadoop\\.)|(hive\\.)", "") + "." + type;
- } else {
- return tableName + "." + type;
- }
- }
-
private static FlatMapFunction<Iterator<String>, String> listDirsRecursively(
Broadcast<SerializableConfiguration> conf,
long olderThanTimestamp) {
diff --git a/spark/src/main/java/org/apache/iceberg/actions/RewriteManifestsAction.java b/spark/src/main/java/org/apache/iceberg/actions/RewriteManifestsAction.java
new file mode 100644
index 0000000..b381746
--- /dev/null
+++ b/spark/src/main/java/org/apache/iceberg/actions/RewriteManifestsAction.java
@@ -0,0 +1,375 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.actions;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import org.apache.hadoop.fs.Path;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.ManifestFile;
+import org.apache.iceberg.ManifestFiles;
+import org.apache.iceberg.ManifestWriter;
+import org.apache.iceberg.MetadataTableType;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.RewriteManifests;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.HadoopFileIO;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.spark.SparkDataFile;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.api.java.function.MapFunction;
+import org.apache.spark.api.java.function.MapPartitionsFunction;
+import org.apache.spark.broadcast.Broadcast;
+import org.apache.spark.sql.Column;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Encoder;
+import org.apache.spark.sql.Encoders;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.internal.SQLConf;
+import org.apache.spark.sql.types.StructType;
+import org.apache.spark.util.SerializableConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * An action that rewrites manifests in a distributed manner and co-locates metadata for partitions.
+ * <p>
+ * By default, this action rewrites all manifests for the current partition spec. The behavior can
+ * be modified by passing a custom predicate to {@link #rewriteIf(Predicate)} and a custom spec id
+ * to {@link #specId(int)}. In addition, this action requires a staging location for new manifests
+ * that should be configured via {@link #stagingLocation}.
+ */
+public class RewriteManifestsAction
+ extends BaseSnapshotUpdateAction<RewriteManifestsAction, RewriteManifestsActionResult> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RewriteManifestsAction.class);
+
+ private final SparkSession spark;
+ private final JavaSparkContext sparkContext;
+ private final Encoder<ManifestFile> manifestEncoder;
+ private final Table table;
+ private final FileIO fileIO;
+ private final long targetManifestSizeBytes;
+
+ private PartitionSpec spec = null;
+ private Predicate<ManifestFile> predicate = manifest -> true;
+ private String stagingLocation = null;
+ private boolean useCaching = true;
+
+ RewriteManifestsAction(SparkSession spark, Table table) {
+ this.spark = spark;
+ this.sparkContext = new JavaSparkContext(spark.sparkContext());
+ this.manifestEncoder = Encoders.javaSerialization(ManifestFile.class);
+ this.table = table;
+ this.spec = table.spec();
+ this.targetManifestSizeBytes = PropertyUtil.propertyAsLong(
+ table.properties(),
+ TableProperties.MANIFEST_TARGET_SIZE_BYTES,
+ TableProperties.MANIFEST_TARGET_SIZE_BYTES_DEFAULT);
+
+ if (table.io() instanceof HadoopFileIO) {
+ // we need to use Spark's SerializableConfiguration to avoid issues with Kryo serialization
+ SerializableConfiguration conf = new SerializableConfiguration(((HadoopFileIO) table.io()).conf());
+ this.fileIO = new HadoopFileIO(conf::value);
+ } else {
+ this.fileIO = table.io();
+ }
+ }
+
+ @Override
+ protected RewriteManifestsAction self() {
+ return this;
+ }
+
+ @Override
+ protected Table table() {
+ return table;
+ }
+
+ public RewriteManifestsAction specId(int specId) {
+ Preconditions.checkArgument(table.specs().containsKey(specId), "Invalid spec id %d", specId);
+ this.spec = table.specs().get(specId);
+ return this;
+ }
+
+ /**
+ * Rewrites only manifests that match the given predicate.
+ *
+ * @param newPredicate a predicate
+ * @return this for method chaining
+ */
+ public RewriteManifestsAction rewriteIf(Predicate<ManifestFile> newPredicate) {
+ this.predicate = newPredicate;
+ return this;
+ }
+
+ /**
+ * Passes a location where the manifests should be written.
+ *
+ * @param newStagingLocation a staging location
+ * @return this for method chaining
+ */
+ public RewriteManifestsAction stagingLocation(String newStagingLocation) {
+ this.stagingLocation = newStagingLocation;
+ return this;
+ }
+
+ /**
+ * Configures whether the action should cache manifest entries used in multiple jobs.
+ *
+ * @param newUseCaching a flag whether to use caching
+ * @return this for method chaining
+ */
+ public RewriteManifestsAction useCaching(boolean newUseCaching) {
+ this.useCaching = newUseCaching;
+ return this;
+ }
+
+ @Override
+ public RewriteManifestsActionResult execute() {
+ Preconditions.checkArgument(stagingLocation != null, "Staging location must be set");
+
+ List<ManifestFile> matchingManifests = findMatchingManifests();
+ if (matchingManifests.isEmpty()) {
+ return RewriteManifestsActionResult.empty();
+ }
+
+ long totalSizeBytes = 0L;
+ int numEntries = 0;
+
+ for (ManifestFile manifest : matchingManifests) {
+ ValidationException.check(hasFileCounts(manifest), "No file counts in manifest: " + manifest.path());
+
+ totalSizeBytes += manifest.length();
+ numEntries += manifest.addedFilesCount() + manifest.existingFilesCount() + manifest.deletedFilesCount();
+ }
+
+ int targetNumManifests = targetNumManifests(totalSizeBytes);
+ int targetNumManifestEntries = targetNumManifestEntries(numEntries, targetNumManifests);
+
+ Dataset<Row> manifestEntryDF = buildManifestEntryDF(matchingManifests);
+
+ List<ManifestFile> newManifests;
+ if (spec.fields().size() < 1) {
+ newManifests = writeManifestsForUnpartitionedTable(manifestEntryDF, targetNumManifests);
+ } else {
+ newManifests = writeManifestsForPartitionedTable(manifestEntryDF, targetNumManifests, targetNumManifestEntries);
+ }
+
+ replaceManifests(matchingManifests, newManifests);
+
+ return new RewriteManifestsActionResult(matchingManifests, newManifests);
+ }
+
+ private Dataset<Row> buildManifestEntryDF(List<ManifestFile> manifests) {
+ Dataset<Row> manifestDF = spark
+ .createDataset(Lists.transform(manifests, ManifestFile::path), Encoders.STRING())
+ .toDF("manifest");
+
+ String entriesMetadataTable = metadataTableName(MetadataTableType.ENTRIES);
+ Dataset<Row> manifestEntryDF = spark.read().format("iceberg")
+ .load(entriesMetadataTable)
+ .filter("status < 2") // select only live entries
+ .selectExpr("input_file_name() as manifest", "snapshot_id", "sequence_number", "data_file");
+
+ Column joinCond = manifestDF.col("manifest").equalTo(manifestEntryDF.col("manifest"));
+ return manifestEntryDF
+ .join(manifestDF, joinCond, "left_semi")
+ .select("snapshot_id", "sequence_number", "data_file");
+ }
+
+ private List<ManifestFile> writeManifestsForUnpartitionedTable(Dataset<Row> manifestEntryDF, int numManifests) {
+ Broadcast<FileIO> io = sparkContext.broadcast(fileIO);
+ StructType sparkType = (StructType) manifestEntryDF.schema().apply("data_file").dataType();
+
+ // we rely only on the target number of manifests for unpartitioned tables
+ // as we should not worry about having too much metadata per partition
+ long maxNumManifestEntries = Long.MAX_VALUE;
+
+ return manifestEntryDF
+ .repartition(numManifests)
+ .mapPartitions(toManifests(io, maxNumManifestEntries, stagingLocation, spec, sparkType), manifestEncoder)
+ .collectAsList();
+ }
+
+ private List<ManifestFile> writeManifestsForPartitionedTable(
+ Dataset<Row> manifestEntryDF, int numManifests,
+ int targetNumManifestEntries) {
+
+ Broadcast<FileIO> io = sparkContext.broadcast(fileIO);
+ StructType sparkType = (StructType) manifestEntryDF.schema().apply("data_file").dataType();
+
+ // we allow the actual size of manifests to be 10% higher if the estimation is not precise enough
+ long maxNumManifestEntries = (long) (1.1 * targetNumManifestEntries);
+
+ return withReusableDS(manifestEntryDF, df -> {
+ Column partitionColumn = df.col("data_file.partition");
+ return df.repartitionByRange(numManifests, partitionColumn)
+ .sortWithinPartitions(partitionColumn)
+ .mapPartitions(toManifests(io, maxNumManifestEntries, stagingLocation, spec, sparkType), manifestEncoder)
+ .collectAsList();
+ });
+ }
+
+ private <T, U> U withReusableDS(Dataset<T> ds, Function<Dataset<T>, U> func) {
+ Dataset<T> reusableDS;
+ if (useCaching) {
+ reusableDS = ds.cache();
+ } else {
+ int parallelism = SQLConf.get().numShufflePartitions();
+ reusableDS = ds.repartition(parallelism).map((MapFunction<T, T>) value -> value, ds.exprEnc());
+ }
+
+ try {
+ return func.apply(reusableDS);
+ } finally {
+ if (useCaching) {
+ reusableDS.unpersist(false);
+ }
+ }
+ }
+
+ private List<ManifestFile> findMatchingManifests() {
+ Snapshot currentSnapshot = table.currentSnapshot();
+
+ if (currentSnapshot == null) {
+ return ImmutableList.of();
+ }
+
+ return currentSnapshot.manifests().stream()
+ .filter(manifest -> manifest.partitionSpecId() == spec.specId() && predicate.test(manifest))
+ .collect(Collectors.toList());
+ }
+
+ private int targetNumManifests(long totalSizeBytes) {
+ return (int) ((totalSizeBytes + targetManifestSizeBytes - 1) / targetManifestSizeBytes);
+ }
+
+ private int targetNumManifestEntries(int numEntries, int numManifests) {
+ return (numEntries + numManifests - 1) / numManifests;
+ }
+
+ private boolean hasFileCounts(ManifestFile manifest) {
+ return manifest.addedFilesCount() != null &&
+ manifest.existingFilesCount() != null &&
+ manifest.deletedFilesCount() != null;
+ }
+
+ private void replaceManifests(Iterable<ManifestFile> deletedManifests, Iterable<ManifestFile> addedManifests) {
+ try {
+ boolean snapshotIdInheritanceEnabled = PropertyUtil.propertyAsBoolean(
+ table.properties(),
+ TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED,
+ TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED_DEFAULT);
+
+ RewriteManifests rewriteManifests = table.rewriteManifests();
+ deletedManifests.forEach(rewriteManifests::deleteManifest);
+ addedManifests.forEach(rewriteManifests::addManifest);
+ commit(rewriteManifests);
+
+ if (!snapshotIdInheritanceEnabled) {
+ // delete new manifests as they were rewritten before the commit
+ deleteFiles(Iterables.transform(addedManifests, ManifestFile::path));
+ }
+ } catch (Exception e) {
+ // delete all new manifests because the rewrite failed
+ deleteFiles(Iterables.transform(addedManifests, ManifestFile::path));
+ throw e;
+ }
+ }
+
+ private void deleteFiles(Iterable<String> locations) {
+ Tasks.foreach(locations)
+ .noRetry()
+ .suppressFailureWhenFinished()
+ .onFailure((location, exc) -> LOG.warn("Failed to delete: {}", location, exc))
+ .run(fileIO::deleteFile);
+ }
+
+ private static ManifestFile writeManifest(
+ List<Row> rows, int startIndex, int endIndex, Broadcast<FileIO> io,
+ String stagingLocation, PartitionSpec spec, StructType sparkType) throws IOException {
+
+ String manifestName = "optimized-m-" + UUID.randomUUID();
+ Path manifestPath = new Path(stagingLocation, manifestName);
+ OutputFile outputFile = io.value().newOutputFile(FileFormat.AVRO.addExtension(manifestPath.toString()));
+
+ Types.StructType dataFileType = DataFile.getType(spec.partitionType());
+ SparkDataFile wrapper = new SparkDataFile(dataFileType, sparkType);
+
+ ManifestWriter writer = ManifestFiles.write(spec, outputFile);
+
+ try {
+ for (int index = startIndex; index < endIndex; index++) {
+ Row row = rows.get(index);
+ long snapshotId = row.getLong(0);
+ long sequenceNumber = row.getLong(1);
+ Row file = row.getStruct(2);
+ writer.existing(wrapper.wrap(file), snapshotId, sequenceNumber);
+ }
+ } finally {
+ writer.close();
+ }
+
+ return writer.toManifestFile();
+ }
+
+ private static MapPartitionsFunction<Row, ManifestFile> toManifests(
+ Broadcast<FileIO> io, long maxNumManifestEntries, String stagingLocation,
+ PartitionSpec spec, StructType sparkType) {
+
+ return (MapPartitionsFunction<Row, ManifestFile>) rows -> {
+ List<Row> rowsAsList = Lists.newArrayList(rows);
+
+ if (rowsAsList.isEmpty()) {
+ return Collections.emptyIterator();
+ }
+
+ List<ManifestFile> manifests = Lists.newArrayList();
+ if (rowsAsList.size() <= maxNumManifestEntries) {
+ manifests.add(writeManifest(rowsAsList, 0, rowsAsList.size(), io, stagingLocation, spec, sparkType));
+ } else {
+ int midIndex = rowsAsList.size() / 2;
+ manifests.add(writeManifest(rowsAsList, 0, midIndex, io, stagingLocation, spec, sparkType));
+ manifests.add(writeManifest(rowsAsList, midIndex, rowsAsList.size(), io, stagingLocation, spec, sparkType));
+ }
+
+ return manifests.iterator();
+ };
+ }
+}
diff --git a/spark/src/main/java/org/apache/iceberg/actions/Actions.java b/spark/src/main/java/org/apache/iceberg/actions/RewriteManifestsActionResult.java
similarity index 51%
copy from spark/src/main/java/org/apache/iceberg/actions/Actions.java
copy to spark/src/main/java/org/apache/iceberg/actions/RewriteManifestsActionResult.java
index 549d541..b122450 100644
--- a/spark/src/main/java/org/apache/iceberg/actions/Actions.java
+++ b/spark/src/main/java/org/apache/iceberg/actions/RewriteManifestsActionResult.java
@@ -19,28 +19,32 @@
package org.apache.iceberg.actions;
-import org.apache.iceberg.Table;
-import org.apache.spark.sql.SparkSession;
+import com.google.common.collect.ImmutableList;
+import java.util.List;
+import org.apache.iceberg.ManifestFile;
-public class Actions {
+public class RewriteManifestsActionResult {
- private SparkSession spark;
- private Table table;
+ private static final RewriteManifestsActionResult EMPTY =
+ new RewriteManifestsActionResult(ImmutableList.of(), ImmutableList.of());
- private Actions(SparkSession spark, Table table) {
- this.spark = spark;
- this.table = table;
+ private List<ManifestFile> deletedManifests;
+ private List<ManifestFile> addedManifests;
+
+ public RewriteManifestsActionResult(List<ManifestFile> deletedManifests, List<ManifestFile> addedManifests) {
+ this.deletedManifests = deletedManifests;
+ this.addedManifests = addedManifests;
}
- public static Actions forTable(SparkSession spark, Table table) {
- return new Actions(spark, table);
+ static RewriteManifestsActionResult empty() {
+ return EMPTY;
}
- public static Actions forTable(Table table) {
- return new Actions(SparkSession.active(), table);
+ public List<ManifestFile> deletedManifests() {
+ return deletedManifests;
}
- public RemoveOrphanFilesAction removeOrphanFiles() {
- return new RemoveOrphanFilesAction(spark, table);
+ public List<ManifestFile> addedManifests() {
+ return addedManifests;
}
}
diff --git a/spark/src/main/java/org/apache/iceberg/actions/Actions.java b/spark/src/main/java/org/apache/iceberg/actions/SnapshotUpdateAction.java
similarity index 58%
copy from spark/src/main/java/org/apache/iceberg/actions/Actions.java
copy to spark/src/main/java/org/apache/iceberg/actions/SnapshotUpdateAction.java
index 549d541..7bf6c7f 100644
--- a/spark/src/main/java/org/apache/iceberg/actions/Actions.java
+++ b/spark/src/main/java/org/apache/iceberg/actions/SnapshotUpdateAction.java
@@ -19,28 +19,6 @@
package org.apache.iceberg.actions;
-import org.apache.iceberg.Table;
-import org.apache.spark.sql.SparkSession;
-
-public class Actions {
-
- private SparkSession spark;
- private Table table;
-
- private Actions(SparkSession spark, Table table) {
- this.spark = spark;
- this.table = table;
- }
-
- public static Actions forTable(SparkSession spark, Table table) {
- return new Actions(spark, table);
- }
-
- public static Actions forTable(Table table) {
- return new Actions(SparkSession.active(), table);
- }
-
- public RemoveOrphanFilesAction removeOrphanFiles() {
- return new RemoveOrphanFilesAction(spark, table);
- }
+public interface SnapshotUpdateAction<ThisT, R> extends Action<R> {
+ ThisT set(String property, String value);
}
diff --git a/spark/src/main/java/org/apache/iceberg/spark/SparkDataFile.java b/spark/src/main/java/org/apache/iceberg/spark/SparkDataFile.java
index 1875c3d..e1866ad 100644
--- a/spark/src/main/java/org/apache/iceberg/spark/SparkDataFile.java
+++ b/spark/src/main/java/org/apache/iceberg/spark/SparkDataFile.java
@@ -19,6 +19,7 @@
package org.apache.iceberg.spark;
+import com.google.common.collect.Maps;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.Locale;
@@ -33,11 +34,23 @@ import org.apache.spark.sql.types.StructType;
public class SparkDataFile implements DataFile {
+ private final int filePathPosition;
+ private final int fileFormatPosition;
+ private final int partitionPosition;
+ private final int recordCountPosition;
+ private final int fileSizeInBytesPosition;
+ private final int columnSizesPosition;
+ private final int valueCountsPosition;
+ private final int nullValueCountsPosition;
+ private final int lowerBoundsPosition;
+ private final int upperBoundsPosition;
+ private final int keyMetadataPosition;
+ private final int splitOffsetsPosition;
private final Type lowerBoundsType;
private final Type upperBoundsType;
private final Type keyMetadataType;
+
private final SparkStructLike wrappedPartition;
- private final int[] fieldPositions;
private Row wrapped;
public SparkDataFile(Types.StructType type, StructType sparkType) {
@@ -45,25 +58,43 @@ public class SparkDataFile implements DataFile {
this.upperBoundsType = type.fieldType("upper_bounds");
this.keyMetadataType = type.fieldType("key_metadata");
this.wrappedPartition = new SparkStructLike(type.fieldType("partition").asStructType());
- this.fieldPositions = indexFields(type, sparkType);
+
+ Map<String, Integer> positions = Maps.newHashMap();
+ type.fields().forEach(field -> {
+ String fieldName = field.name();
+ positions.put(fieldName, fieldPosition(fieldName, sparkType));
+ });
+
+ filePathPosition = positions.get("file_path");
+ fileFormatPosition = positions.get("file_format");
+ partitionPosition = positions.get("partition");
+ recordCountPosition = positions.get("record_count");
+ fileSizeInBytesPosition = positions.get("file_size_in_bytes");
+ columnSizesPosition = positions.get("column_sizes");
+ valueCountsPosition = positions.get("value_counts");
+ nullValueCountsPosition = positions.get("null_value_counts");
+ lowerBoundsPosition = positions.get("lower_bounds");
+ upperBoundsPosition = positions.get("upper_bounds");
+ keyMetadataPosition = positions.get("key_metadata");
+ splitOffsetsPosition = positions.get("split_offsets");
}
public SparkDataFile wrap(Row row) {
this.wrapped = row;
if (wrappedPartition.size() > 0) {
- this.wrappedPartition.wrap(row.getAs(fieldPositions[2]));
+ this.wrappedPartition.wrap(row.getAs(partitionPosition));
}
return this;
}
@Override
public CharSequence path() {
- return wrapped.getAs(fieldPositions[0]);
+ return wrapped.getAs(filePathPosition);
}
@Override
public FileFormat format() {
- String formatAsString = wrapped.getString(fieldPositions[1]).toUpperCase(Locale.ROOT);
+ String formatAsString = wrapped.getString(fileFormatPosition).toUpperCase(Locale.ROOT);
return FileFormat.valueOf(formatAsString);
}
@@ -74,42 +105,44 @@ public class SparkDataFile implements DataFile {
@Override
public long recordCount() {
- return wrapped.getAs(fieldPositions[3]);
+ return wrapped.getAs(recordCountPosition);
}
@Override
public long fileSizeInBytes() {
- return wrapped.getAs(fieldPositions[4]);
+ return wrapped.getAs(fileSizeInBytesPosition);
}
@Override
public Map<Integer, Long> columnSizes() {
- return wrapped.getJavaMap(fieldPositions[6]);
+ return !wrapped.isNullAt(columnSizesPosition) ? wrapped.getJavaMap(columnSizesPosition) : null;
}
@Override
public Map<Integer, Long> valueCounts() {
- return wrapped.getJavaMap(fieldPositions[7]);
+ return !wrapped.isNullAt(valueCountsPosition) ? wrapped.getJavaMap(valueCountsPosition) : null;
}
@Override
public Map<Integer, Long> nullValueCounts() {
- return wrapped.getJavaMap(fieldPositions[8]);
+ return !wrapped.isNullAt(nullValueCountsPosition) ? wrapped.getJavaMap(nullValueCountsPosition) : null;
}
@Override
public Map<Integer, ByteBuffer> lowerBounds() {
- return convert(lowerBoundsType, wrapped.getJavaMap(fieldPositions[9]));
+ Map<?, ?> lowerBounds = !wrapped.isNullAt(lowerBoundsPosition) ? wrapped.getJavaMap(lowerBoundsPosition) : null;
+ return convert(lowerBoundsType, lowerBounds);
}
@Override
public Map<Integer, ByteBuffer> upperBounds() {
- return convert(upperBoundsType, wrapped.getJavaMap(fieldPositions[10]));
+ Map<?, ?> upperBounds = !wrapped.isNullAt(upperBoundsPosition) ? wrapped.getJavaMap(upperBoundsPosition) : null;
+ return convert(upperBoundsType, upperBounds);
}
@Override
public ByteBuffer keyMetadata() {
- return convert(keyMetadataType, wrapped.get(fieldPositions[11]));
+ return convert(keyMetadataType, wrapped.get(keyMetadataPosition));
}
@Override
@@ -124,17 +157,7 @@ public class SparkDataFile implements DataFile {
@Override
public List<Long> splitOffsets() {
- return wrapped.getList(fieldPositions[12]);
- }
-
- private int[] indexFields(Types.StructType type, StructType sparkType) {
- List<Types.NestedField> fields = type.fields();
- int[] positions = new int[fields.size()];
- for (int index = 0; index < fields.size(); index++) {
- Types.NestedField field = fields.get(index);
- positions[index] = fieldPosition(field.name(), sparkType);
- }
- return positions;
+ return !wrapped.isNullAt(splitOffsetsPosition) ? wrapped.getList(splitOffsetsPosition) : null;
}
private int fieldPosition(String name, StructType sparkType) {
diff --git a/spark/src/test/java/org/apache/iceberg/TestRewriteManifestsAction.java b/spark/src/test/java/org/apache/iceberg/TestRewriteManifestsAction.java
new file mode 100644
index 0000000..df35590
--- /dev/null
+++ b/spark/src/test/java/org/apache/iceberg/TestRewriteManifestsAction.java
@@ -0,0 +1,440 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.actions.Actions;
+import org.apache.iceberg.actions.RewriteManifestsActionResult;
+import org.apache.iceberg.hadoop.HadoopTables;
+import org.apache.iceberg.spark.SparkTableUtil;
+import org.apache.iceberg.spark.source.ThreeColumnRecord;
+import org.apache.iceberg.types.Types;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Encoders;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.catalyst.TableIdentifier;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static org.apache.iceberg.types.Types.NestedField.optional;
+
+@RunWith(Parameterized.class)
+public class TestRewriteManifestsAction {
+
+ private static final HadoopTables TABLES = new HadoopTables(new Configuration());
+ private static final Schema SCHEMA = new Schema(
+ optional(1, "c1", Types.IntegerType.get()),
+ optional(2, "c2", Types.StringType.get()),
+ optional(3, "c3", Types.StringType.get())
+ );
+
+ private static SparkSession spark;
+
+ @Parameterized.Parameters
+ public static Object[][] parameters() {
+ return new Object[][] {
+ new Object[] { "true" },
+ new Object[] { "false" }
+ };
+ }
+
+ @BeforeClass
+ public static void startSpark() {
+ TestRewriteManifestsAction.spark = SparkSession.builder()
+ .master("local[2]")
+ .getOrCreate();
+ }
+
+ @AfterClass
+ public static void stopSpark() {
+ SparkSession currentSpark = TestRewriteManifestsAction.spark;
+ TestRewriteManifestsAction.spark = null;
+ currentSpark.stop();
+ }
+
+ @Rule
+ public TemporaryFolder temp = new TemporaryFolder();
+
+ private final String snapshotIdInheritanceEnabled;
+ private String tableLocation = null;
+
+ public TestRewriteManifestsAction(String snapshotIdInheritanceEnabled) {
+ this.snapshotIdInheritanceEnabled = snapshotIdInheritanceEnabled;
+ }
+
+ @Before
+ public void setupTableLocation() throws Exception {
+ File tableDir = temp.newFolder();
+ this.tableLocation = tableDir.toURI().toString();
+ }
+
+ @Test
+ public void testRewriteManifestsEmptyTable() throws IOException {
+ PartitionSpec spec = PartitionSpec.unpartitioned();
+ Map<String, String> options = Maps.newHashMap();
+ options.put(TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED, snapshotIdInheritanceEnabled);
+ Table table = TABLES.create(SCHEMA, spec, options, tableLocation);
+
+ Assert.assertNull("Table must be empty", table.currentSnapshot());
+
+ Actions actions = Actions.forTable(table);
+
+ actions.rewriteManifests()
+ .rewriteIf(manifest -> true)
+ .stagingLocation(temp.newFolder().toString())
+ .execute();
+
+ Assert.assertNull("Table must stay empty", table.currentSnapshot());
+ }
+
+ @Test
+ public void testRewriteSmallManifestsNonPartitionedTable() throws IOException {
+ PartitionSpec spec = PartitionSpec.unpartitioned();
+ Map<String, String> options = Maps.newHashMap();
+ options.put(TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED, snapshotIdInheritanceEnabled);
+ Table table = TABLES.create(SCHEMA, spec, options, tableLocation);
+
+ List<ThreeColumnRecord> records1 = Lists.newArrayList(
+ new ThreeColumnRecord(1, null, "AAAA"),
+ new ThreeColumnRecord(1, "BBBBBBBBBB", "BBBB")
+ );
+ writeRecords(records1);
+
+ List<ThreeColumnRecord> records2 = Lists.newArrayList(
+ new ThreeColumnRecord(2, "CCCCCCCCCC", "CCCC"),
+ new ThreeColumnRecord(2, "DDDDDDDDDD", "DDDD")
+ );
+ writeRecords(records2);
+
+ table.refresh();
+
+ List<ManifestFile> manifests = table.currentSnapshot().manifests();
+ Assert.assertEquals("Should have 2 manifests before rewrite", 2, manifests.size());
+
+ Actions actions = Actions.forTable(table);
+
+ RewriteManifestsActionResult result = actions.rewriteManifests()
+ .rewriteIf(manifest -> true)
+ .stagingLocation(temp.newFolder().toString())
+ .execute();
+
+ Assert.assertEquals("Action should rewrite 2 manifests", 2, result.deletedManifests().size());
+ Assert.assertEquals("Action should add 1 manifests", 1, result.addedManifests().size());
+
+ table.refresh();
+
+ List<ManifestFile> newManifests = table.currentSnapshot().manifests();
+ Assert.assertEquals("Should have 1 manifests after rewrite", 1, newManifests.size());
+
+ Assert.assertEquals(4, (long) newManifests.get(0).existingFilesCount());
+ Assert.assertFalse(newManifests.get(0).hasAddedFiles());
+ Assert.assertFalse(newManifests.get(0).hasDeletedFiles());
+
+ List<ThreeColumnRecord> expectedRecords = Lists.newArrayList();
+ expectedRecords.addAll(records1);
+ expectedRecords.addAll(records2);
+
+ Dataset<Row> resultDF = spark.read().format("iceberg").load(tableLocation);
+ List<ThreeColumnRecord> actualRecords = resultDF.sort("c1", "c2")
+ .as(Encoders.bean(ThreeColumnRecord.class))
+ .collectAsList();
+
+ Assert.assertEquals("Rows must match", expectedRecords, actualRecords);
+ }
+
+ @Test
+ public void testRewriteSmallManifestsPartitionedTable() throws IOException {
+ PartitionSpec spec = PartitionSpec.builderFor(SCHEMA)
+ .identity("c1")
+ .truncate("c2", 2)
+ .build();
+ Map<String, String> options = Maps.newHashMap();
+ options.put(TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED, snapshotIdInheritanceEnabled);
+ Table table = TABLES.create(SCHEMA, spec, options, tableLocation);
+
+ List<ThreeColumnRecord> records1 = Lists.newArrayList(
+ new ThreeColumnRecord(1, null, "AAAA"),
+ new ThreeColumnRecord(1, "BBBBBBBBBB", "BBBB")
+ );
+ writeRecords(records1);
+
+ List<ThreeColumnRecord> records2 = Lists.newArrayList(
+ new ThreeColumnRecord(2, "CCCCCCCCCC", "CCCC"),
+ new ThreeColumnRecord(2, "DDDDDDDDDD", "DDDD")
+ );
+ writeRecords(records2);
+
+ List<ThreeColumnRecord> records3 = Lists.newArrayList(
+ new ThreeColumnRecord(3, "EEEEEEEEEE", "EEEE"),
+ new ThreeColumnRecord(3, "FFFFFFFFFF", "FFFF")
+ );
+ writeRecords(records3);
+
+ List<ThreeColumnRecord> records4 = Lists.newArrayList(
+ new ThreeColumnRecord(4, "GGGGGGGGGG", "GGGG"),
+ new ThreeColumnRecord(4, "HHHHHHHHHG", "HHHH")
+ );
+ writeRecords(records4);
+
+ table.refresh();
+
+ List<ManifestFile> manifests = table.currentSnapshot().manifests();
+ Assert.assertEquals("Should have 4 manifests before rewrite", 4, manifests.size());
+
+ Actions actions = Actions.forTable(table);
+
+ // we will expect to have 2 manifests with 4 entries in each after rewrite
+ long manifestEntrySizeBytes = computeManifestEntrySizeBytes(manifests);
+ long targetManifestSizeBytes = (long) (1.05 * 4 * manifestEntrySizeBytes);
+
+ table.updateProperties()
+ .set(TableProperties.MANIFEST_TARGET_SIZE_BYTES, String.valueOf(targetManifestSizeBytes))
+ .commit();
+
+ RewriteManifestsActionResult result = actions.rewriteManifests()
+ .rewriteIf(manifest -> true)
+ .stagingLocation(temp.newFolder().toString())
+ .execute();
+
+ Assert.assertEquals("Action should rewrite 4 manifests", 4, result.deletedManifests().size());
+ Assert.assertEquals("Action should add 2 manifests", 2, result.addedManifests().size());
+
+ table.refresh();
+
+ List<ManifestFile> newManifests = table.currentSnapshot().manifests();
+ Assert.assertEquals("Should have 2 manifests after rewrite", 2, newManifests.size());
+
+ Assert.assertEquals(4, (long) newManifests.get(0).existingFilesCount());
+ Assert.assertFalse(newManifests.get(0).hasAddedFiles());
+ Assert.assertFalse(newManifests.get(0).hasDeletedFiles());
+
+ Assert.assertEquals(4, (long) newManifests.get(1).existingFilesCount());
+ Assert.assertFalse(newManifests.get(1).hasAddedFiles());
+ Assert.assertFalse(newManifests.get(1).hasDeletedFiles());
+
+ List<ThreeColumnRecord> expectedRecords = Lists.newArrayList();
+ expectedRecords.addAll(records1);
+ expectedRecords.addAll(records2);
+ expectedRecords.addAll(records3);
+ expectedRecords.addAll(records4);
+
+ Dataset<Row> resultDF = spark.read().format("iceberg").load(tableLocation);
+ List<ThreeColumnRecord> actualRecords = resultDF.sort("c1", "c2")
+ .as(Encoders.bean(ThreeColumnRecord.class))
+ .collectAsList();
+
+ Assert.assertEquals("Rows must match", expectedRecords, actualRecords);
+ }
+
+ @Test
+ public void testRewriteImportedManifests() throws IOException {
+ PartitionSpec spec = PartitionSpec.builderFor(SCHEMA)
+ .identity("c3")
+ .build();
+ Map<String, String> options = Maps.newHashMap();
+ options.put(TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED, snapshotIdInheritanceEnabled);
+ Table table = TABLES.create(SCHEMA, spec, options, tableLocation);
+
+ List<ThreeColumnRecord> records = Lists.newArrayList(
+ new ThreeColumnRecord(1, null, "AAAA"),
+ new ThreeColumnRecord(1, "BBBBBBBBBB", "BBBB")
+ );
+ File parquetTableDir = temp.newFolder("parquet_table");
+ String parquetTableLocation = parquetTableDir.toURI().toString();
+
+ try {
+ Dataset<Row> inputDF = spark.createDataFrame(records, ThreeColumnRecord.class);
+ inputDF.select("c1", "c2", "c3")
+ .write()
+ .format("parquet")
+ .mode("overwrite")
+ .option("path", parquetTableLocation)
+ .partitionBy("c3")
+ .saveAsTable("parquet_table");
+
+ File stagingDir = temp.newFolder("staging-dir");
+ SparkTableUtil.importSparkTable(spark, new TableIdentifier("parquet_table"), table, stagingDir.toString());
+
+ Snapshot snapshot = table.currentSnapshot();
+
+ Actions actions = Actions.forTable(table);
+
+ RewriteManifestsActionResult result = actions.rewriteManifests()
+ .rewriteIf(manifest -> true)
+ .stagingLocation(temp.newFolder().toString())
+ .execute();
+
+ Assert.assertEquals("Action should rewrite all manifests", snapshot.manifests(), result.deletedManifests());
+ Assert.assertEquals("Action should add 1 manifest", 1, result.addedManifests().size());
+
+ } finally {
+ spark.sql("DROP TABLE parquet_table");
+ }
+ }
+
+ @Test
+ public void testRewriteLargeManifestsPartitionedTable() throws IOException {
+ PartitionSpec spec = PartitionSpec.builderFor(SCHEMA)
+ .identity("c3")
+ .build();
+ Map<String, String> options = Maps.newHashMap();
+ options.put(TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED, snapshotIdInheritanceEnabled);
+ Table table = TABLES.create(SCHEMA, spec, options, tableLocation);
+
+ // all records belong to the same partition
+ List<ThreeColumnRecord> records = Lists.newArrayList();
+ for (int i = 0; i < 50; i++) {
+ records.add(new ThreeColumnRecord(i, String.valueOf(i), "0"));
+ }
+ Dataset<Row> df = spark.createDataFrame(records, ThreeColumnRecord.class);
+ // repartition to create separate files
+ writeDF(df.repartition(50, df.col("c1")));
+
+ table.refresh();
+
+ List<ManifestFile> manifests = table.currentSnapshot().manifests();
+ Assert.assertEquals("Should have 1 manifests before rewrite", 1, manifests.size());
+
+ // set the target manifest size to a small value to force splitting records into multiple files
+ table.updateProperties()
+ .set(TableProperties.MANIFEST_TARGET_SIZE_BYTES, String.valueOf(manifests.get(0).length() / 2))
+ .commit();
+
+ Actions actions = Actions.forTable(table);
+
+ RewriteManifestsActionResult result = actions.rewriteManifests()
+ .rewriteIf(manifest -> true)
+ .stagingLocation(temp.newFolder().toString())
+ .execute();
+
+ Assert.assertEquals("Action should rewrite 1 manifest", 1, result.deletedManifests().size());
+ Assert.assertEquals("Action should add 2 manifests", 2, result.addedManifests().size());
+
+ table.refresh();
+
+ List<ManifestFile> newManifests = table.currentSnapshot().manifests();
+ Assert.assertEquals("Should have 2 manifests after rewrite", 2, newManifests.size());
+
+ Dataset<Row> resultDF = spark.read().format("iceberg").load(tableLocation);
+ List<ThreeColumnRecord> actualRecords = resultDF.sort("c1", "c2")
+ .as(Encoders.bean(ThreeColumnRecord.class))
+ .collectAsList();
+
+ Assert.assertEquals("Rows must match", records, actualRecords);
+ }
+
+ @Test
+ public void testRewriteManifestsWithPredicate() throws IOException {
+ PartitionSpec spec = PartitionSpec.builderFor(SCHEMA)
+ .identity("c1")
+ .truncate("c2", 2)
+ .build();
+ Map<String, String> options = Maps.newHashMap();
+ options.put(TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED, snapshotIdInheritanceEnabled);
+ Table table = TABLES.create(SCHEMA, spec, options, tableLocation);
+
+ List<ThreeColumnRecord> records1 = Lists.newArrayList(
+ new ThreeColumnRecord(1, null, "AAAA"),
+ new ThreeColumnRecord(1, "BBBBBBBBBB", "BBBB")
+ );
+ writeRecords(records1);
+
+ List<ThreeColumnRecord> records2 = Lists.newArrayList(
+ new ThreeColumnRecord(2, "CCCCCCCCCC", "CCCC"),
+ new ThreeColumnRecord(2, "DDDDDDDDDD", "DDDD")
+ );
+ writeRecords(records2);
+
+ table.refresh();
+
+ List<ManifestFile> manifests = table.currentSnapshot().manifests();
+ Assert.assertEquals("Should have 2 manifests before rewrite", 2, manifests.size());
+
+ Actions actions = Actions.forTable(table);
+
+ // rewrite only the first manifest without caching
+ RewriteManifestsActionResult result = actions.rewriteManifests()
+ .rewriteIf(manifest -> manifest.path().equals(manifests.get(0).path()))
+ .stagingLocation(temp.newFolder().toString())
+ .useCaching(false)
+ .execute();
+
+ Assert.assertEquals("Action should rewrite 1 manifest", 1, result.deletedManifests().size());
+ Assert.assertEquals("Action should add 1 manifests", 1, result.addedManifests().size());
+
+ table.refresh();
+
+ List<ManifestFile> newManifests = table.currentSnapshot().manifests();
+ Assert.assertEquals("Should have 2 manifests after rewrite", 2, newManifests.size());
+
+ Assert.assertFalse("First manifest must be rewritten", newManifests.contains(manifests.get(0)));
+ Assert.assertTrue("Second manifest must not be rewritten", newManifests.contains(manifests.get(1)));
+
+ List<ThreeColumnRecord> expectedRecords = Lists.newArrayList();
+ expectedRecords.addAll(records1);
+ expectedRecords.addAll(records2);
+
+ Dataset<Row> resultDF = spark.read().format("iceberg").load(tableLocation);
+ List<ThreeColumnRecord> actualRecords = resultDF.sort("c1", "c2")
+ .as(Encoders.bean(ThreeColumnRecord.class))
+ .collectAsList();
+
+ Assert.assertEquals("Rows must match", expectedRecords, actualRecords);
+ }
+
+ private void writeRecords(List<ThreeColumnRecord> records) {
+ Dataset<Row> df = spark.createDataFrame(records, ThreeColumnRecord.class);
+ writeDF(df);
+ }
+
+ private void writeDF(Dataset<Row> df) {
+ df.select("c1", "c2", "c3")
+ .write()
+ .format("iceberg")
+ .mode("append")
+ .save(tableLocation);
+ }
+
+ private long computeManifestEntrySizeBytes(List<ManifestFile> manifests) {
+ long totalSize = 0L;
+ int numEntries = 0;
+
+ for (ManifestFile manifest : manifests) {
+ totalSize += manifest.length();
+ numEntries += manifest.addedFilesCount() + manifest.existingFilesCount() + manifest.deletedFilesCount();
+ }
+
+ return totalSize / numEntries;
+ }
+}