You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by pv...@apache.org on 2025/03/29 08:15:21 UTC
(iceberg) branch main updated: Core: FileRewritePlanner implementation (#12493)
This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new d5971429ea Core: FileRewritePlanner implementation (#12493)
d5971429ea is described below
commit d5971429ea903be873b5884c64a3dd41076179ea
Author: pvary <pe...@gmail.com>
AuthorDate: Sat Mar 29 09:15:14 2025 +0100
Core: FileRewritePlanner implementation (#12493)
---
.../iceberg/actions/BinPackRewriteFilePlanner.java | 312 ++++++++++++++
.../BinPackRewritePositionDeletePlanner.java | 222 ++++++++++
.../org/apache/iceberg/actions/FileRewriter.java | 3 +
.../apache/iceberg/actions/RewriteFileGroup.java | 13 +-
.../actions/RewritePositionDeletesGroup.java | 13 +-
.../iceberg/actions/SizeBasedDataRewriter.java | 23 +-
...riter.java => SizeBasedFileRewritePlanner.java} | 75 ++--
.../iceberg/actions/SizeBasedFileRewriter.java | 4 +
.../actions/SizeBasedPositionDeletesRewriter.java | 5 +
.../actions/TestBinPackRewriteFilePlanner.java | 478 +++++++++++++++++++++
.../TestBinPackRewritePositionDeletePlanner.java | 294 +++++++++++++
.../actions/TestSizeBasedFileRewritePlanner.java | 185 ++++++++
.../iceberg/actions/TestSizeBasedRewriter.java | 4 +
13 files changed, 1584 insertions(+), 47 deletions(-)
diff --git a/core/src/main/java/org/apache/iceberg/actions/BinPackRewriteFilePlanner.java b/core/src/main/java/org/apache/iceberg/actions/BinPackRewriteFilePlanner.java
new file mode 100644
index 0000000000..b12bba21b2
--- /dev/null
+++ b/core/src/main/java/org/apache/iceberg/actions/BinPackRewriteFilePlanner.java
@@ -0,0 +1,312 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.actions;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.ContentFile;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.RewriteJobOrder;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.TableScan;
+import org.apache.iceberg.actions.RewriteDataFiles.FileGroupInfo;
+import org.apache.iceberg.data.GenericRecord;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
+import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.ContentFileUtil;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.StructLikeMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Groups specified data files in the {@link Table} into {@link RewriteFileGroup}s. The files are
+ * grouped by partitions based on their size using fix sized bins. Extends {@link
+ * SizeBasedFileRewritePlanner} with delete file number and delete ratio thresholds and job {@link
+ * RewriteDataFiles#REWRITE_JOB_ORDER} handling.
+ */
+public class BinPackRewriteFilePlanner
+ extends SizeBasedFileRewritePlanner<FileGroupInfo, FileScanTask, DataFile, RewriteFileGroup> {
+ /**
+ * The minimum number of deletes that needs to be associated with a data file for it to be
+ * considered for rewriting. If a data file has this number of deletes or more, it will be
+ * rewritten regardless of its file size determined by {@link #MIN_FILE_SIZE_BYTES} and {@link
+ * #MAX_FILE_SIZE_BYTES}. If a file group contains a file that satisfies this condition, the file
+ * group will be rewritten regardless of the number of files in the file group determined by
+ * {@link #MIN_INPUT_FILES}.
+ *
+ * <p>Defaults to Integer.MAX_VALUE, which means this feature is not enabled by default.
+ */
+ public static final String DELETE_FILE_THRESHOLD = "delete-file-threshold";
+
+ public static final int DELETE_FILE_THRESHOLD_DEFAULT = Integer.MAX_VALUE;
+
+ /**
+ * The ratio of the deleted rows in a data file for it to be considered for rewriting. If the
+ * deletion ratio of a data file is greater than or equal to this value, it will be rewritten
+ * regardless of its file size determined by {@link #MIN_FILE_SIZE_BYTES} and {@link
+ * #MAX_FILE_SIZE_BYTES}. If a file group contains a file that satisfies this condition, the file
+ * group will be rewritten regardless of the number of files in the file group determined by
+ * {@link #MIN_INPUT_FILES}.
+ *
+ * <p>Defaults to 0.3, which means that if the number of deleted records in a file reaches or
+ * exceeds 30%, it will trigger the rewriting operation.
+ */
+ public static final String DELETE_RATIO_THRESHOLD = "delete-ratio-threshold";
+
+ public static final double DELETE_RATIO_THRESHOLD_DEFAULT = 0.3;
+
+ private static final Logger LOG = LoggerFactory.getLogger(BinPackRewriteFilePlanner.class);
+
+ private final Expression filter;
+ private final Long snapshotId;
+ private final boolean caseSensitive;
+
+ private int deleteFileThreshold;
+ private double deleteRatioThreshold;
+ private RewriteJobOrder rewriteJobOrder;
+
+ public BinPackRewriteFilePlanner(Table table) {
+ this(table, Expressions.alwaysTrue());
+ }
+
+ public BinPackRewriteFilePlanner(Table table, Expression filter) {
+ this(
+ table,
+ filter,
+ table.currentSnapshot() != null ? table.currentSnapshot().snapshotId() : null,
+ false);
+ }
+
+ /**
+ * Creates the planner for the given table.
+ *
+ * @param table to plan for
+ * @param filter used to remove files from the plan
+ * @param snapshotId a snapshot ID used for planning and as the starting snapshot id for commit
+ * validation when replacing the files
+ * @param caseSensitive property used for scanning
+ */
+ public BinPackRewriteFilePlanner(
+ Table table, Expression filter, Long snapshotId, boolean caseSensitive) {
+ super(table);
+ this.filter = filter;
+ this.snapshotId = snapshotId;
+ this.caseSensitive = caseSensitive;
+ }
+
+ @Override
+ public Set<String> validOptions() {
+ return ImmutableSet.<String>builder()
+ .addAll(super.validOptions())
+ .add(DELETE_FILE_THRESHOLD)
+ .add(DELETE_RATIO_THRESHOLD)
+ .add(RewriteDataFiles.REWRITE_JOB_ORDER)
+ .build();
+ }
+
+ @Override
+ public void init(Map<String, String> options) {
+ super.init(options);
+ this.deleteFileThreshold = deleteFileThreshold(options);
+ this.deleteRatioThreshold = deleteRatioThreshold(options);
+ this.rewriteJobOrder =
+ RewriteJobOrder.fromName(
+ PropertyUtil.propertyAsString(
+ options,
+ RewriteDataFiles.REWRITE_JOB_ORDER,
+ RewriteDataFiles.REWRITE_JOB_ORDER_DEFAULT));
+ }
+
+ private int deleteFileThreshold(Map<String, String> options) {
+ int value =
+ PropertyUtil.propertyAsInt(options, DELETE_FILE_THRESHOLD, DELETE_FILE_THRESHOLD_DEFAULT);
+ Preconditions.checkArgument(
+ value >= 0, "'%s' is set to %s but must be >= 0", DELETE_FILE_THRESHOLD, value);
+ return value;
+ }
+
+ private double deleteRatioThreshold(Map<String, String> options) {
+ double value =
+ PropertyUtil.propertyAsDouble(
+ options, DELETE_RATIO_THRESHOLD, DELETE_RATIO_THRESHOLD_DEFAULT);
+ Preconditions.checkArgument(
+ value > 0, "'%s' is set to %s but must be > 0", DELETE_RATIO_THRESHOLD, value);
+ Preconditions.checkArgument(
+ value <= 1, "'%s' is set to %s but must be <= 1", DELETE_RATIO_THRESHOLD, value);
+ return value;
+ }
+
+ @Override
+ protected Iterable<FileScanTask> filterFiles(Iterable<FileScanTask> tasks) {
+ return Iterables.filter(
+ tasks,
+ task ->
+ outsideDesiredFileSizeRange(task) || tooManyDeletes(task) || tooHighDeleteRatio(task));
+ }
+
+ @Override
+ protected Iterable<List<FileScanTask>> filterFileGroups(List<List<FileScanTask>> groups) {
+ return Iterables.filter(
+ groups,
+ group ->
+ enoughInputFiles(group)
+ || enoughContent(group)
+ || tooMuchContent(group)
+ || group.stream().anyMatch(this::tooManyDeletes)
+ || group.stream().anyMatch(this::tooHighDeleteRatio));
+ }
+
+ @Override
+ protected long defaultTargetFileSize() {
+ return PropertyUtil.propertyAsLong(
+ table().properties(),
+ TableProperties.WRITE_TARGET_FILE_SIZE_BYTES,
+ TableProperties.WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT);
+ }
+
+ @Override
+ public FileRewritePlan<FileGroupInfo, FileScanTask, DataFile, RewriteFileGroup> plan() {
+ StructLikeMap<List<List<FileScanTask>>> plan = planFileGroups();
+ RewriteExecutionContext ctx = new RewriteExecutionContext();
+ Stream<RewriteFileGroup> groups =
+ plan.entrySet().stream()
+ .filter(e -> !e.getValue().isEmpty())
+ .flatMap(
+ e -> {
+ StructLike partition = e.getKey();
+ List<List<FileScanTask>> scanGroups = e.getValue();
+ return scanGroups.stream()
+ .map(
+ tasks -> {
+ long inputSize = inputSize(tasks);
+ return newRewriteGroup(
+ ctx,
+ partition,
+ tasks,
+ inputSplitSize(inputSize),
+ expectedOutputFiles(inputSize));
+ });
+ })
+ .sorted(RewriteFileGroup.comparator(rewriteJobOrder));
+ Map<StructLike, Integer> groupsInPartition = plan.transformValues(List::size);
+ int totalGroupCount = groupsInPartition.values().stream().reduce(Integer::sum).orElse(0);
+ return new FileRewritePlan<>(
+ CloseableIterable.of(groups.collect(Collectors.toList())),
+ totalGroupCount,
+ groupsInPartition);
+ }
+
+ private boolean tooManyDeletes(FileScanTask task) {
+ return task.deletes() != null && task.deletes().size() >= deleteFileThreshold;
+ }
+
+ private boolean tooHighDeleteRatio(FileScanTask task) {
+ if (task.deletes() == null || task.deletes().isEmpty()) {
+ return false;
+ }
+
+ long knownDeletedRecordCount =
+ task.deletes().stream()
+ .filter(ContentFileUtil::isFileScoped)
+ .mapToLong(ContentFile::recordCount)
+ .sum();
+
+ double deletedRecords = (double) Math.min(knownDeletedRecordCount, task.file().recordCount());
+ double deleteRatio = deletedRecords / task.file().recordCount();
+ return deleteRatio >= deleteRatioThreshold;
+ }
+
+ private StructLikeMap<List<List<FileScanTask>>> planFileGroups() {
+ TableScan scan =
+ table().newScan().filter(filter).caseSensitive(caseSensitive).ignoreResiduals();
+
+ if (snapshotId != null) {
+ scan = scan.useSnapshot(snapshotId);
+ }
+
+ CloseableIterable<FileScanTask> fileScanTasks = scan.planFiles();
+
+ try {
+ Types.StructType partitionType = table().spec().partitionType();
+ StructLikeMap<List<FileScanTask>> filesByPartition =
+ groupByPartition(table(), partitionType, fileScanTasks);
+ return filesByPartition.transformValues(tasks -> ImmutableList.copyOf(planFileGroups(tasks)));
+ } finally {
+ try {
+ fileScanTasks.close();
+ } catch (IOException io) {
+ LOG.error("Cannot properly close file iterable while planning for rewrite", io);
+ }
+ }
+ }
+
+ private StructLikeMap<List<FileScanTask>> groupByPartition(
+ Table table, Types.StructType partitionType, Iterable<FileScanTask> tasks) {
+ StructLikeMap<List<FileScanTask>> filesByPartition = StructLikeMap.create(partitionType);
+ StructLike emptyStruct = GenericRecord.create(partitionType);
+
+ for (FileScanTask task : tasks) {
+ // If a task uses an incompatible partition spec the data inside could contain values
+ // which belong to multiple partitions in the current spec. Treating all such files as
+ // un-partitioned and grouping them together helps to minimize new files made.
+ StructLike taskPartition =
+ task.file().specId() == table.spec().specId() ? task.file().partition() : emptyStruct;
+
+ filesByPartition.computeIfAbsent(taskPartition, unused -> Lists.newArrayList()).add(task);
+ }
+
+ return filesByPartition;
+ }
+
+ private RewriteFileGroup newRewriteGroup(
+ RewriteExecutionContext ctx,
+ StructLike partition,
+ List<FileScanTask> tasks,
+ long inputSplitSize,
+ int expectedOutputFiles) {
+ FileGroupInfo info =
+ ImmutableRewriteDataFiles.FileGroupInfo.builder()
+ .globalIndex(ctx.currentGlobalIndex())
+ .partitionIndex(ctx.currentPartitionIndex(partition))
+ .partition(partition)
+ .build();
+ return new RewriteFileGroup(
+ info,
+ Lists.newArrayList(tasks),
+ outputSpecId(),
+ writeMaxFileSize(),
+ inputSplitSize,
+ expectedOutputFiles);
+ }
+}
diff --git a/core/src/main/java/org/apache/iceberg/actions/BinPackRewritePositionDeletePlanner.java b/core/src/main/java/org/apache/iceberg/actions/BinPackRewritePositionDeletePlanner.java
new file mode 100644
index 0000000000..8e080ee30b
--- /dev/null
+++ b/core/src/main/java/org/apache/iceberg/actions/BinPackRewritePositionDeletePlanner.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.actions;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.MetadataTableType;
+import org.apache.iceberg.MetadataTableUtils;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.PositionDeletesScanTask;
+import org.apache.iceberg.PositionDeletesTable;
+import org.apache.iceberg.RewriteJobOrder;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.actions.RewritePositionDeleteFiles.FileGroupInfo;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
+import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.PartitionUtil;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.StructLikeMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Groups specified delete files in the {@link Table} into {@link RewritePositionDeletesGroup}s.
+ * These will be grouped by partitions based on their size using fix sized bins. Extends the {@link
+ * SizeBasedFileRewritePlanner} with {@link RewritePositionDeleteFiles#REWRITE_JOB_ORDER} handling.
+ */
+public class BinPackRewritePositionDeletePlanner
+ extends SizeBasedFileRewritePlanner<
+ FileGroupInfo, PositionDeletesScanTask, DeleteFile, RewritePositionDeletesGroup> {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(BinPackRewritePositionDeletePlanner.class);
+
+ private final Expression filter;
+ private final boolean caseSensitive;
+ private RewriteJobOrder rewriteJobOrder;
+
+ public BinPackRewritePositionDeletePlanner(Table table) {
+ this(table, Expressions.alwaysTrue(), false);
+ }
+
+ /**
+ * Creates the planner for the given table.
+ *
+ * @param table to plan for
+ * @param filter used to remove files from the plan
+ * @param caseSensitive property used for scanning
+ */
+ public BinPackRewritePositionDeletePlanner(
+ Table table, Expression filter, boolean caseSensitive) {
+ super(table);
+ this.caseSensitive = caseSensitive;
+ this.filter = filter;
+ }
+
+ @Override
+ public Set<String> validOptions() {
+ return ImmutableSet.<String>builder()
+ .addAll(super.validOptions())
+ .add(RewritePositionDeleteFiles.REWRITE_JOB_ORDER)
+ .build();
+ }
+
+ @Override
+ public void init(Map<String, String> options) {
+ super.init(options);
+ this.rewriteJobOrder =
+ RewriteJobOrder.fromName(
+ PropertyUtil.propertyAsString(
+ options,
+ RewritePositionDeleteFiles.REWRITE_JOB_ORDER,
+ RewritePositionDeleteFiles.REWRITE_JOB_ORDER_DEFAULT));
+ }
+
+ @Override
+ public FileRewritePlan<
+ FileGroupInfo, PositionDeletesScanTask, DeleteFile, RewritePositionDeletesGroup>
+ plan() {
+ StructLikeMap<List<List<PositionDeletesScanTask>>> plan = planFileGroups();
+ RewriteExecutionContext ctx = new RewriteExecutionContext();
+ Stream<RewritePositionDeletesGroup> groups =
+ plan.entrySet().stream()
+ .filter(e -> !e.getValue().isEmpty())
+ .flatMap(
+ e -> {
+ StructLike partition = e.getKey();
+ List<List<PositionDeletesScanTask>> scanGroups = e.getValue();
+ return scanGroups.stream()
+ .map(
+ tasks -> {
+ long inputSize = inputSize(tasks);
+ return newRewriteGroup(
+ ctx,
+ partition,
+ tasks,
+ inputSplitSize(inputSize),
+ expectedOutputFiles(inputSize));
+ });
+ })
+ .sorted(RewritePositionDeletesGroup.comparator(rewriteJobOrder));
+ Map<StructLike, Integer> groupsInPartition = plan.transformValues(List::size);
+ int totalGroupCount = groupsInPartition.values().stream().reduce(Integer::sum).orElse(0);
+ return new FileRewritePlan<>(
+ CloseableIterable.of(groups.collect(Collectors.toList())),
+ totalGroupCount,
+ groupsInPartition);
+ }
+
+ @Override
+ protected Iterable<PositionDeletesScanTask> filterFiles(Iterable<PositionDeletesScanTask> tasks) {
+ return Iterables.filter(tasks, this::outsideDesiredFileSizeRange);
+ }
+
+ @Override
+ protected Iterable<List<PositionDeletesScanTask>> filterFileGroups(
+ List<List<PositionDeletesScanTask>> groups) {
+ return Iterables.filter(
+ groups, group -> enoughInputFiles(group) || enoughContent(group) || tooMuchContent(group));
+ }
+
+ @Override
+ protected long defaultTargetFileSize() {
+ return PropertyUtil.propertyAsLong(
+ table().properties(),
+ TableProperties.DELETE_TARGET_FILE_SIZE_BYTES,
+ TableProperties.DELETE_TARGET_FILE_SIZE_BYTES_DEFAULT);
+ }
+
+ private StructLikeMap<List<List<PositionDeletesScanTask>>> planFileGroups() {
+ Table deletesTable =
+ MetadataTableUtils.createMetadataTableInstance(table(), MetadataTableType.POSITION_DELETES);
+ CloseableIterable<PositionDeletesScanTask> fileTasks = planFiles(deletesTable);
+
+ try {
+ Types.StructType partitionType = Partitioning.partitionType(deletesTable);
+ StructLikeMap<List<PositionDeletesScanTask>> fileTasksByPartition =
+ groupByPartition(partitionType, fileTasks);
+ return fileTasksByPartition.transformValues(
+ tasks -> ImmutableList.copyOf(planFileGroups(tasks)));
+ } finally {
+ try {
+ fileTasks.close();
+ } catch (IOException io) {
+ LOG.error("Cannot properly close file iterable while planning for rewrite", io);
+ }
+ }
+ }
+
+ private CloseableIterable<PositionDeletesScanTask> planFiles(Table deletesTable) {
+ PositionDeletesTable.PositionDeletesBatchScan scan =
+ (PositionDeletesTable.PositionDeletesBatchScan) deletesTable.newBatchScan();
+ return CloseableIterable.transform(
+ scan.baseTableFilter(filter).caseSensitive(caseSensitive).ignoreResiduals().planFiles(),
+ PositionDeletesScanTask.class::cast);
+ }
+
+ private StructLikeMap<List<PositionDeletesScanTask>> groupByPartition(
+ Types.StructType partitionType, Iterable<PositionDeletesScanTask> tasks) {
+ StructLikeMap<List<PositionDeletesScanTask>> filesByPartition =
+ StructLikeMap.create(partitionType);
+
+ for (PositionDeletesScanTask task : tasks) {
+ StructLike coerced =
+ PartitionUtil.coercePartition(partitionType, task.spec(), task.partition());
+
+ List<PositionDeletesScanTask> partitionTasks = filesByPartition.get(coerced);
+ if (partitionTasks == null) {
+ partitionTasks = Lists.newArrayList();
+ }
+
+ partitionTasks.add(task);
+ filesByPartition.put(coerced, partitionTasks);
+ }
+
+ return filesByPartition;
+ }
+
+ private RewritePositionDeletesGroup newRewriteGroup(
+ RewriteExecutionContext ctx,
+ StructLike partition,
+ List<PositionDeletesScanTask> tasks,
+ long inputSplitSize,
+ int expectedOutputFiles) {
+ ImmutableRewritePositionDeleteFiles.FileGroupInfo info =
+ ImmutableRewritePositionDeleteFiles.FileGroupInfo.builder()
+ .globalIndex(ctx.currentGlobalIndex())
+ .partitionIndex(ctx.currentPartitionIndex(partition))
+ .partition(partition)
+ .build();
+ return new RewritePositionDeletesGroup(
+ info, Lists.newArrayList(tasks), writeMaxFileSize(), inputSplitSize, expectedOutputFiles);
+ }
+}
diff --git a/core/src/main/java/org/apache/iceberg/actions/FileRewriter.java b/core/src/main/java/org/apache/iceberg/actions/FileRewriter.java
index 7c6b4e8d7e..3aabb63c15 100644
--- a/core/src/main/java/org/apache/iceberg/actions/FileRewriter.java
+++ b/core/src/main/java/org/apache/iceberg/actions/FileRewriter.java
@@ -34,7 +34,10 @@ import org.apache.iceberg.ContentScanTask;
*
* @param <T> the Java type of tasks to read content files
* @param <F> the Java type of content files
+ * @deprecated since 1.9.0, will be removed in 1.10.0; use {@link FileRewritePlanner} and {@link
+ * FileRewriteRunner}
*/
+@Deprecated
public interface FileRewriter<T extends ContentScanTask<F>, F extends ContentFile<F>> {
/** Returns a description for this rewriter. */
diff --git a/core/src/main/java/org/apache/iceberg/actions/RewriteFileGroup.java b/core/src/main/java/org/apache/iceberg/actions/RewriteFileGroup.java
index c31ff327b6..e5982bb447 100644
--- a/core/src/main/java/org/apache/iceberg/actions/RewriteFileGroup.java
+++ b/core/src/main/java/org/apache/iceberg/actions/RewriteFileGroup.java
@@ -38,6 +38,9 @@ public class RewriteFileGroup extends RewriteGroupBase<FileGroupInfo, FileScanTa
private final int outputSpecId;
private DataFileSet addedFiles = DataFileSet.create();
+ /**
+ * @deprecated since 1.9.0, will be removed in 1.10.0
+ */
@Deprecated
public RewriteFileGroup(FileGroupInfo info, List<FileScanTask> fileScanTasks) {
this(info, fileScanTasks, 0, 0L, 0L, 0);
@@ -48,14 +51,14 @@ public class RewriteFileGroup extends RewriteGroupBase<FileGroupInfo, FileScanTa
List<FileScanTask> fileScanTasks,
int outputSpecId,
long writeMaxFileSize,
- long splitSize,
+ long inputSplitSize,
int expectedOutputFiles) {
- super(info, fileScanTasks, writeMaxFileSize, splitSize, expectedOutputFiles);
+ super(info, fileScanTasks, writeMaxFileSize, inputSplitSize, expectedOutputFiles);
this.outputSpecId = outputSpecId;
}
/**
- * @deprecated use {@link #fileScanTasks()}
+ * @deprecated since 1.9.0, will be removed in 1.10.0; use {@link #fileScanTasks()}
*/
@Deprecated
public List<FileScanTask> fileScans() {
@@ -103,7 +106,7 @@ public class RewriteFileGroup extends RewriteGroupBase<FileGroupInfo, FileScanTa
}
/**
- * @deprecated use {@link #inputFilesSizeInBytes()}
+ * @deprecated since 1.9.0, will be removed in 1.10.0; use {@link #inputFilesSizeInBytes()}
*/
@Deprecated
public long sizeInBytes() {
@@ -111,7 +114,7 @@ public class RewriteFileGroup extends RewriteGroupBase<FileGroupInfo, FileScanTa
}
/**
- * @deprecated use {@link #inputFileNum()}
+ * @deprecated since 1.9.0, will be removed in 1.10.0; use {@link #inputFileNum()}
*/
@Deprecated
public int numFiles() {
diff --git a/core/src/main/java/org/apache/iceberg/actions/RewritePositionDeletesGroup.java b/core/src/main/java/org/apache/iceberg/actions/RewritePositionDeletesGroup.java
index 382409629f..f3b3a6a81f 100644
--- a/core/src/main/java/org/apache/iceberg/actions/RewritePositionDeletesGroup.java
+++ b/core/src/main/java/org/apache/iceberg/actions/RewritePositionDeletesGroup.java
@@ -42,6 +42,9 @@ public class RewritePositionDeletesGroup
private DeleteFileSet addedDeleteFiles = DeleteFileSet.create();
+ /**
+ * @deprecated since 1.9.0, will be removed in 1.10.0
+ */
@Deprecated
public RewritePositionDeletesGroup(FileGroupInfo info, List<PositionDeletesScanTask> tasks) {
this(info, tasks, 0L, 0L, 0);
@@ -51,16 +54,16 @@ public class RewritePositionDeletesGroup
FileGroupInfo info,
List<PositionDeletesScanTask> tasks,
long writeMaxFileSize,
- long splitSize,
+ long inputSplitSize,
int expectedOutputFiles) {
- super(info, tasks, writeMaxFileSize, splitSize, expectedOutputFiles);
+ super(info, tasks, writeMaxFileSize, inputSplitSize, expectedOutputFiles);
Preconditions.checkArgument(!tasks.isEmpty(), "Tasks must not be empty");
this.maxRewrittenDataSequenceNumber =
tasks.stream().mapToLong(t -> t.file().dataSequenceNumber()).max().getAsLong();
}
/**
- * @deprecated use {@link #fileScanTasks()}
+ * @deprecated since 1.9.0, will be removed in 1.10.0; use {@link #fileScanTasks()}
*/
@Deprecated
public List<PositionDeletesScanTask> tasks() {
@@ -117,7 +120,7 @@ public class RewritePositionDeletesGroup
}
/**
- * @deprecated use {@link #inputFilesSizeInBytes()}
+ * @deprecated since 1.9.0, will be removed in 1.10.0; use {@link #inputFilesSizeInBytes()}
*/
@Deprecated
public long rewrittenBytes() {
@@ -129,7 +132,7 @@ public class RewritePositionDeletesGroup
}
/**
- * @deprecated use {@link #inputFileNum()}
+ * @deprecated since 1.9.0, will be removed in 1.10.0; use {@link #inputFileNum()}
*/
@Deprecated
public int numRewrittenDeleteFiles() {
diff --git a/core/src/main/java/org/apache/iceberg/actions/SizeBasedDataRewriter.java b/core/src/main/java/org/apache/iceberg/actions/SizeBasedDataRewriter.java
index 0c55b2892a..a4304ee22c 100644
--- a/core/src/main/java/org/apache/iceberg/actions/SizeBasedDataRewriter.java
+++ b/core/src/main/java/org/apache/iceberg/actions/SizeBasedDataRewriter.java
@@ -32,6 +32,13 @@ import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
import org.apache.iceberg.util.ContentFileUtil;
import org.apache.iceberg.util.PropertyUtil;
+/**
+ * Deprecated {@link SizeBasedDataRewriter} abstract class.
+ *
+ * @deprecated since 1.9.0, will be removed in 1.10.0; use {@link BinPackRewriteFilePlanner} and
+ * {@link FileRewriteRunner}
+ */
+@Deprecated
public abstract class SizeBasedDataRewriter extends SizeBasedFileRewriter<FileScanTask, DataFile> {
/**
@@ -49,15 +56,15 @@ public abstract class SizeBasedDataRewriter extends SizeBasedFileRewriter<FileSc
public static final int DELETE_FILE_THRESHOLD_DEFAULT = Integer.MAX_VALUE;
/**
- * The minimum deletion ratio that needs to be associated with a data file for it to be considered
- * for rewriting. If the deletion ratio of a data file is greater than or equal to this value, it
- * will be rewritten regardless of its file size determined by {@link #MIN_FILE_SIZE_BYTES} and
- * {@link #MAX_FILE_SIZE_BYTES}. If a file group contains a file that satisfies this condition,
- * the file group will be rewritten regardless of the number of files in the file group determined
- * by {@link #MIN_INPUT_FILES}.
+ * The ratio of the deleted rows in a data file for it to be considered for rewriting. If the
+ * deletion ratio of a data file is greater than or equal to this value, it will be rewritten
+ * regardless of its file size determined by {@link #MIN_FILE_SIZE_BYTES} and {@link
+ * #MAX_FILE_SIZE_BYTES}. If a file group contains a file that satisfies this condition, the file
+ * group will be rewritten regardless of the number of files in the file group determined by
+ * {@link #MIN_INPUT_FILES}.
*
- * <p>Defaults to 0.3, which means that if the deletion ratio of a file reaches or exceeds 30%, it
- * may trigger the rewriting operation.
+ * <p>Defaults to 0.3, which means that if the number of deleted records in a file reaches or
+ * exceeds 30%, it will trigger the rewriting operation.
*/
public static final String DELETE_RATIO_THRESHOLD = "delete-ratio-threshold";
diff --git a/core/src/main/java/org/apache/iceberg/actions/SizeBasedFileRewriter.java b/core/src/main/java/org/apache/iceberg/actions/SizeBasedFileRewritePlanner.java
similarity index 88%
copy from core/src/main/java/org/apache/iceberg/actions/SizeBasedFileRewriter.java
copy to core/src/main/java/org/apache/iceberg/actions/SizeBasedFileRewritePlanner.java
index cea7003c1a..fad139078b 100644
--- a/core/src/main/java/org/apache/iceberg/actions/SizeBasedFileRewriter.java
+++ b/core/src/main/java/org/apache/iceberg/actions/SizeBasedFileRewritePlanner.java
@@ -22,9 +22,10 @@ import java.math.RoundingMode;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.iceberg.ContentFile;
import org.apache.iceberg.ContentScanTask;
-import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.StructLike;
import org.apache.iceberg.Table;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
@@ -36,7 +37,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * A file rewriter that determines which files to rewrite based on their size.
+ * A file rewrite planner that determines which files to rewrite based on their size.
*
* <p>If files are smaller than the {@link #MIN_FILE_SIZE_BYTES} threshold or larger than the {@link
* #MAX_FILE_SIZE_BYTES} threshold, they are considered targets for being rewritten.
@@ -48,10 +49,14 @@ import org.slf4j.LoggerFactory;
*
* <p>Note that implementations may add extra conditions for selecting files or filtering groups.
*/
-public abstract class SizeBasedFileRewriter<T extends ContentScanTask<F>, F extends ContentFile<F>>
- implements FileRewriter<T, F> {
+public abstract class SizeBasedFileRewritePlanner<
+ I,
+ T extends ContentScanTask<F>,
+ F extends ContentFile<F>,
+ G extends RewriteGroupBase<I, T, F>>
+ implements FileRewritePlanner<I, T, F, G> {
- private static final Logger LOG = LoggerFactory.getLogger(SizeBasedFileRewriter.class);
+ private static final Logger LOG = LoggerFactory.getLogger(SizeBasedFileRewritePlanner.class);
/** The target output file size that this file rewriter will attempt to generate. */
public static final String TARGET_FILE_SIZE_BYTES = "target-file-size-bytes";
@@ -102,7 +107,7 @@ public abstract class SizeBasedFileRewriter<T extends ContentScanTask<F>, F exte
public static final long MAX_FILE_GROUP_SIZE_BYTES_DEFAULT = 100L * 1024 * 1024 * 1024; // 100 GB
- private static final long SPLIT_OVERHEAD = 5 * 1024;
+ private static final long SPLIT_OVERHEAD = 5L * 1024;
private final Table table;
private long targetFileSize;
@@ -111,23 +116,21 @@ public abstract class SizeBasedFileRewriter<T extends ContentScanTask<F>, F exte
private int minInputFiles;
private boolean rewriteAll;
private long maxGroupSize;
-
private int outputSpecId;
- protected SizeBasedFileRewriter(Table table) {
+ protected SizeBasedFileRewritePlanner(Table table) {
this.table = table;
}
+ /** Expected target file size before configuration. */
protected abstract long defaultTargetFileSize();
+ /** Additional filter for tasks before grouping. */
protected abstract Iterable<T> filterFiles(Iterable<T> tasks);
+ /** Additional filter for groups. */
protected abstract Iterable<List<T>> filterFileGroups(List<List<T>> groups);
- protected Table table() {
- return table;
- }
-
@Override
public Set<String> validOptions() {
return ImmutableSet.of(
@@ -145,7 +148,6 @@ public abstract class SizeBasedFileRewriter<T extends ContentScanTask<F>, F exte
this.targetFileSize = sizeThresholds.get(TARGET_FILE_SIZE_BYTES);
this.minFileSize = sizeThresholds.get(MIN_FILE_SIZE_BYTES);
this.maxFileSize = sizeThresholds.get(MAX_FILE_SIZE_BYTES);
-
this.minInputFiles = minInputFiles(options);
this.rewriteAll = rewriteAll(options);
this.maxGroupSize = maxGroupSize(options);
@@ -156,12 +158,15 @@ public abstract class SizeBasedFileRewriter<T extends ContentScanTask<F>, F exte
}
}
- protected boolean wronglySized(T task) {
+ protected Table table() {
+ return table;
+ }
+
+ protected boolean outsideDesiredFileSizeRange(T task) {
return task.length() < minFileSize || task.length() > maxFileSize;
}
- @Override
- public Iterable<List<T>> planFileGroups(Iterable<T> tasks) {
+ protected Iterable<List<T>> planFileGroups(Iterable<T> tasks) {
Iterable<T> filteredTasks = rewriteAll ? tasks : filterFiles(tasks);
BinPacking.ListPacker<T> packer = new BinPacking.ListPacker<>(maxGroupSize, 1, false);
List<List<T>> groups = packer.pack(filteredTasks, ContentScanTask::length);
@@ -191,14 +196,12 @@ public abstract class SizeBasedFileRewriter<T extends ContentScanTask<F>, F exte
* of output files. The final split size is adjusted to be at least as big as the target file size
* but less than the max write file size.
*/
- protected long splitSize(long inputSize) {
- long estimatedSplitSize = (inputSize / numOutputFiles(inputSize)) + SPLIT_OVERHEAD;
+ protected long inputSplitSize(long inputSize) {
+ long estimatedSplitSize = (inputSize / expectedOutputFiles(inputSize)) + SPLIT_OVERHEAD;
if (estimatedSplitSize < targetFileSize) {
return targetFileSize;
- } else if (estimatedSplitSize > writeMaxFileSize()) {
- return writeMaxFileSize();
} else {
- return estimatedSplitSize;
+ return Math.min(estimatedSplitSize, writeMaxFileSize());
}
}
@@ -216,7 +219,7 @@ public abstract class SizeBasedFileRewriter<T extends ContentScanTask<F>, F exte
* @param inputSize a total input size for a file group
* @return the number of files this rewriter should create
*/
- protected long numOutputFiles(long inputSize) {
+ protected int expectedOutputFiles(long inputSize) {
if (inputSize < targetFileSize) {
return 1;
}
@@ -227,18 +230,18 @@ public abstract class SizeBasedFileRewriter<T extends ContentScanTask<F>, F exte
if (LongMath.mod(inputSize, targetFileSize) > minFileSize) {
// the remainder file is of a valid size for this rewrite so keep it
- return numFilesWithRemainder;
+ return (int) numFilesWithRemainder;
} else if (avgFileSizeWithoutRemainder
< Math.min(1.1 * targetFileSize, (double) writeMaxFileSize())) {
// if the reminder is distributed amongst other files,
// the average file size will be no more than 10% bigger than the target file size
// so round down and distribute remainder amongst other files
- return numFilesWithoutRemainder;
+ return (int) numFilesWithoutRemainder;
} else {
// keep the remainder file as it is not OK to distribute it amongst other files
- return numFilesWithRemainder;
+ return (int) numFilesWithRemainder;
}
}
@@ -263,10 +266,6 @@ public abstract class SizeBasedFileRewriter<T extends ContentScanTask<F>, F exte
return (long) (targetFileSize + ((maxFileSize - targetFileSize) * 0.5));
}
- protected PartitionSpec outputSpec() {
- return table.specs().get(outputSpecId);
- }
-
protected int outputSpecId() {
return outputSpecId;
}
@@ -341,4 +340,22 @@ public abstract class SizeBasedFileRewriter<T extends ContentScanTask<F>, F exte
private boolean rewriteAll(Map<String, String> options) {
return PropertyUtil.propertyAsBoolean(options, REWRITE_ALL, REWRITE_ALL_DEFAULT);
}
+
+ protected static class RewriteExecutionContext {
+ private final Map<StructLike, Integer> partitionIndexMap;
+ private final AtomicInteger groupIndex;
+
+ protected RewriteExecutionContext() {
+ this.partitionIndexMap = Maps.newConcurrentMap();
+ this.groupIndex = new AtomicInteger(1);
+ }
+
+ protected int currentGlobalIndex() {
+ return groupIndex.getAndIncrement();
+ }
+
+ protected int currentPartitionIndex(StructLike partition) {
+ return partitionIndexMap.merge(partition, 1, Integer::sum);
+ }
+ }
}
diff --git a/core/src/main/java/org/apache/iceberg/actions/SizeBasedFileRewriter.java b/core/src/main/java/org/apache/iceberg/actions/SizeBasedFileRewriter.java
index cea7003c1a..785360c75f 100644
--- a/core/src/main/java/org/apache/iceberg/actions/SizeBasedFileRewriter.java
+++ b/core/src/main/java/org/apache/iceberg/actions/SizeBasedFileRewriter.java
@@ -47,7 +47,11 @@ import org.slf4j.LoggerFactory;
* {@link #TARGET_FILE_SIZE_BYTES}.
*
* <p>Note that implementations may add extra conditions for selecting files or filtering groups.
+ *
+ * @deprecated since 1.9.0, will be removed in 1.10.0; use {@link SizeBasedFileRewritePlanner} and
+ * {@link FileRewriteRunner}
*/
+@Deprecated
public abstract class SizeBasedFileRewriter<T extends ContentScanTask<F>, F extends ContentFile<F>>
implements FileRewriter<T, F> {
diff --git a/core/src/main/java/org/apache/iceberg/actions/SizeBasedPositionDeletesRewriter.java b/core/src/main/java/org/apache/iceberg/actions/SizeBasedPositionDeletesRewriter.java
index c08a31a731..5c2f5c3bfb 100644
--- a/core/src/main/java/org/apache/iceberg/actions/SizeBasedPositionDeletesRewriter.java
+++ b/core/src/main/java/org/apache/iceberg/actions/SizeBasedPositionDeletesRewriter.java
@@ -26,6 +26,11 @@ import org.apache.iceberg.TableProperties;
import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
import org.apache.iceberg.util.PropertyUtil;
+/**
+ * @deprecated since 1.9.0, will be removed in 1.10.0; use {@link
+ * BinPackRewritePositionDeletePlanner} and {@link FileRewriteRunner}
+ */
+@Deprecated
public abstract class SizeBasedPositionDeletesRewriter
extends SizeBasedFileRewriter<PositionDeletesScanTask, DeleteFile> {
diff --git a/core/src/test/java/org/apache/iceberg/actions/TestBinPackRewriteFilePlanner.java b/core/src/test/java/org/apache/iceberg/actions/TestBinPackRewriteFilePlanner.java
new file mode 100644
index 0000000000..885e1d62b9
--- /dev/null
+++ b/core/src/test/java/org/apache/iceberg/actions/TestBinPackRewriteFilePlanner.java
@@ -0,0 +1,478 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.actions;
+
+import static org.apache.iceberg.actions.BinPackRewriteFilePlanner.MAX_FILE_SIZE_DEFAULT_RATIO;
+import static org.apache.iceberg.actions.RewriteDataFiles.REWRITE_JOB_ORDER;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DataFiles;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.MockFileScanTask;
+import org.apache.iceberg.RewriteJobOrder;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.TestBase;
+import org.apache.iceberg.TestTables;
+import org.apache.iceberg.actions.RewriteDataFiles.FileGroupInfo;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
+import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+class TestBinPackRewriteFilePlanner {
+ private static final Map<String, String> REWRITE_ALL =
+ ImmutableMap.of(BinPackRewriteFilePlanner.REWRITE_ALL, "true");
+
+ private static final DataFile FILE_1 = newDataFile("data_bucket=0", 10);
+ private static final DataFile FILE_2 = newDataFile("data_bucket=0", 10);
+ private static final DataFile FILE_3 = newDataFile("data_bucket=0", 10);
+ private static final DataFile FILE_4 = newDataFile("data_bucket=1", 11);
+ private static final DataFile FILE_5 = newDataFile("data_bucket=1", 11);
+ private static final DataFile FILE_6 = newDataFile("data_bucket=2", 50);
+
+ private static final Map<RewriteJobOrder, List<StructLike>> EXPECTED =
+ ImmutableMap.of(
+ RewriteJobOrder.FILES_DESC,
+ ImmutableList.of(FILE_1.partition(), FILE_4.partition(), FILE_6.partition()),
+ RewriteJobOrder.FILES_ASC,
+ ImmutableList.of(FILE_6.partition(), FILE_4.partition(), FILE_1.partition()),
+ RewriteJobOrder.BYTES_DESC,
+ ImmutableList.of(FILE_6.partition(), FILE_1.partition(), FILE_4.partition()),
+ RewriteJobOrder.BYTES_ASC,
+ ImmutableList.of(FILE_4.partition(), FILE_1.partition(), FILE_6.partition()));
+
+ @TempDir private File tableDir = null;
+ private TestTables.TestTable table = null;
+
+ @BeforeEach
+ public void setupTable() throws Exception {
+ this.table = TestTables.create(tableDir, "test", TestBase.SCHEMA, TestBase.SPEC, 3);
+ }
+
+ @AfterEach
+ public void cleanupTables() {
+ TestTables.clearTables();
+ }
+
+ @Test
+ void testPartitionedTable() {
+ addFiles();
+ BinPackRewriteFilePlanner planner = new BinPackRewriteFilePlanner(table);
+ planner.init(ImmutableMap.of(BinPackRewriteFilePlanner.REWRITE_ALL, "true"));
+
+ FileRewritePlan<FileGroupInfo, FileScanTask, DataFile, RewriteFileGroup> plan = planner.plan();
+
+ List<RewriteFileGroup> groups = Lists.newArrayList(plan.groups().iterator());
+ assertThat(groups).hasSize(3);
+ assertThat(groups.stream().mapToInt(RewriteGroupBase::inputFileNum).sum()).isEqualTo(6);
+ }
+
+ @Test
+ void testUnpartitionedTable() {
+ table.updateSpec().removeField("data_bucket").commit();
+ table.refresh();
+ table
+ .newAppend()
+ .appendFile(newDataFile("", 10))
+ .appendFile(newDataFile("", 20))
+ .appendFile(newDataFile("", 30))
+ .commit();
+ BinPackRewriteFilePlanner planner = new BinPackRewriteFilePlanner(table);
+ planner.init(
+ ImmutableMap.of(
+ BinPackRewriteFilePlanner.MIN_INPUT_FILES,
+ "1",
+ BinPackRewriteFilePlanner.MIN_FILE_SIZE_BYTES,
+ "30"));
+
+ FileRewritePlan<FileGroupInfo, FileScanTask, DataFile, RewriteFileGroup> plan = planner.plan();
+
+ assertThat(plan.totalGroupCount()).isEqualTo(1);
+ assertThat(plan.groups().iterator().next().inputFileNum()).isEqualTo(2);
+ }
+
+ @Test
+ void testEmptyTable() {
+ BinPackRewriteFilePlanner planner = new BinPackRewriteFilePlanner(table);
+ planner.init(REWRITE_ALL);
+
+ FileRewritePlan<FileGroupInfo, FileScanTask, DataFile, RewriteFileGroup> plan = planner.plan();
+
+ assertThat(table.currentSnapshot()).as("Table must be empty").isNull();
+ assertThat(plan.totalGroupCount()).isZero();
+ }
+
+ @ParameterizedTest
+ @EnumSource(
+ value = RewriteJobOrder.class,
+ names = {"FILES_DESC", "FILES_ASC", "BYTES_DESC", "BYTES_ASC"})
+ void testJobOrder(RewriteJobOrder order) {
+ addFiles();
+ BinPackRewriteFilePlanner planner = new BinPackRewriteFilePlanner(table);
+ planner.init(
+ ImmutableMap.of(
+ BinPackRewriteFilePlanner.REWRITE_ALL, "true", REWRITE_JOB_ORDER, order.name()));
+
+ FileRewritePlan<FileGroupInfo, FileScanTask, DataFile, RewriteFileGroup> plan = planner.plan();
+
+ List<RewriteFileGroup> groups = Lists.newArrayList(plan.groups().iterator());
+ assertThat(groups.stream().map(group -> group.info().partition()).collect(Collectors.toList()))
+ .isEqualTo(EXPECTED.get(order));
+ assertThat(plan.totalGroupCount()).isEqualTo(3);
+ EXPECTED.get(order).forEach(s -> assertThat(plan.groupsInPartition(s)).isEqualTo(1));
+ }
+
+ @Test
+ void testMaxGroupSize() {
+ addFiles();
+ BinPackRewriteFilePlanner planner = new BinPackRewriteFilePlanner(table);
+ planner.init(
+ ImmutableMap.of(
+ BinPackRewriteFilePlanner.REWRITE_ALL,
+ "true",
+ BinPackRewriteFilePlanner.MAX_FILE_GROUP_SIZE_BYTES,
+ "10"));
+
+ FileRewritePlan<FileGroupInfo, FileScanTask, DataFile, RewriteFileGroup> plan = planner.plan();
+
+ assertThat(plan.totalGroupCount()).isEqualTo(6);
+ assertThat(plan.groupsInPartition(FILE_1.partition())).isEqualTo(3);
+ assertThat(plan.groupsInPartition(FILE_4.partition())).isEqualTo(2);
+ assertThat(plan.groupsInPartition(FILE_6.partition())).isEqualTo(1);
+ }
+
+ @Test
+ void testFilter() {
+ addFiles();
+ BinPackRewriteFilePlanner planner =
+ new BinPackRewriteFilePlanner(
+ table,
+ Expressions.or(
+ Expressions.equal(Expressions.bucket("data", 16), 0),
+ Expressions.equal(Expressions.bucket("data", 16), 2)));
+ planner.init(REWRITE_ALL);
+
+ FileRewritePlan<FileGroupInfo, FileScanTask, DataFile, RewriteFileGroup> plan = planner.plan();
+
+ List<RewriteFileGroup> groups = Lists.newArrayList(plan.groups().iterator());
+ assertThat(plan.totalGroupCount()).isEqualTo(2);
+ assertThat(groups).as("Only partitions for bucket 0 and 2 should be in the plan").hasSize(2);
+ assertThat(groups.stream().mapToLong(RewriteFileGroup::inputFileNum).sum())
+ .as("Only files matching bucket 0 and 2 should be in the plan")
+ .isEqualTo(4);
+ }
+
+ @Test
+ void testMaxOutputFileSize() {
+ addFiles();
+ int targetFileSize = 10;
+ BinPackRewriteFilePlanner planner = new BinPackRewriteFilePlanner(table);
+ planner.init(
+ ImmutableMap.of(
+ BinPackRewriteFilePlanner.REWRITE_ALL,
+ "true",
+ BinPackRewriteFilePlanner.TARGET_FILE_SIZE_BYTES,
+ String.valueOf(targetFileSize)));
+
+ FileRewritePlan<FileGroupInfo, FileScanTask, DataFile, RewriteFileGroup> plan = planner.plan();
+
+ plan.groups()
+ .forEach(
+ group ->
+ assertThat(group.maxOutputFileSize())
+ .isGreaterThan(targetFileSize)
+ .isLessThan((long) (targetFileSize * MAX_FILE_SIZE_DEFAULT_RATIO)));
+ }
+
+ @Test
+ void testExpectedOutputFiles() {
+ addFiles();
+ BinPackRewriteFilePlanner planner = new BinPackRewriteFilePlanner(table);
+ planner.init(
+ ImmutableMap.of(
+ BinPackRewriteFilePlanner.REWRITE_ALL,
+ "true",
+ BinPackRewriteFilePlanner.TARGET_FILE_SIZE_BYTES,
+ "21"));
+
+ FileRewritePlan<FileGroupInfo, FileScanTask, DataFile, RewriteFileGroup> plan = planner.plan();
+
+ List<RewriteFileGroup> groups = Lists.newArrayList(plan.groups().iterator());
+ assertThat(groups).hasSize(3);
+ for (RewriteFileGroup group : groups) {
+ if (FILE_1.partition().equals(group.info().partition())) {
+ assertThat(group.expectedOutputFiles()).isEqualTo(2);
+ } else if (FILE_4.partition().equals(group.info().partition())) {
+ // The result file size is slightly bigger than target, but still below the 10% threshold
+ assertThat(group.expectedOutputFiles())
+ .as("Should be 1, because the size is below target * 1.1")
+ .isEqualTo(1);
+ } else if (FILE_6.partition().equals(group.info().partition())) {
+ assertThat(group.expectedOutputFiles()).isEqualTo(3);
+ } else {
+ throw new IllegalStateException("Unexpected partition: " + group.info().partition());
+ }
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void testOutputSpec(boolean useOldSpecId) {
+ addFiles();
+ int oldSpecId = table.spec().specId();
+ table.updateSpec().removeField("data_bucket").commit();
+ table.newAppend().appendFile(newDataFile("", 10)).commit();
+ table.refresh();
+ int newSpecId = table.spec().specId();
+
+ Map<String, String> options = Maps.newHashMap(REWRITE_ALL);
+ if (useOldSpecId) {
+ options.put(RewriteDataFiles.OUTPUT_SPEC_ID, String.valueOf(oldSpecId));
+ }
+
+ BinPackRewriteFilePlanner planner = new BinPackRewriteFilePlanner(table);
+ planner.init(options);
+
+ FileRewritePlan<FileGroupInfo, FileScanTask, DataFile, RewriteFileGroup> plan = planner.plan();
+
+ plan.groups()
+ .forEach(
+ group ->
+ assertThat(group.outputSpecId()).isEqualTo(useOldSpecId ? oldSpecId : newSpecId));
+ }
+
+ @Test
+ void testValidOptions() {
+ BinPackRewriteFilePlanner planner = new BinPackRewriteFilePlanner(table);
+
+ assertThat(planner.validOptions())
+ .as("Planner must report all supported options")
+ .isEqualTo(
+ ImmutableSet.of(
+ BinPackRewriteFilePlanner.TARGET_FILE_SIZE_BYTES,
+ BinPackRewriteFilePlanner.MIN_FILE_SIZE_BYTES,
+ BinPackRewriteFilePlanner.MAX_FILE_SIZE_BYTES,
+ BinPackRewriteFilePlanner.MIN_INPUT_FILES,
+ BinPackRewriteFilePlanner.REWRITE_ALL,
+ BinPackRewriteFilePlanner.MAX_FILE_GROUP_SIZE_BYTES,
+ BinPackRewriteFilePlanner.DELETE_FILE_THRESHOLD,
+ BinPackRewriteFilePlanner.DELETE_RATIO_THRESHOLD,
+ RewriteDataFiles.REWRITE_JOB_ORDER));
+ }
+
+ @Test
+ void testInvalidOption() {
+ BinPackRewriteFilePlanner planner = new BinPackRewriteFilePlanner(table);
+
+ Map<String, String> invalidRewriteJobOrderOptions =
+ ImmutableMap.of(RewriteDataFiles.REWRITE_JOB_ORDER, "foo");
+ assertThatThrownBy(() -> planner.init(invalidRewriteJobOrderOptions))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid rewrite job order name: foo");
+
+ Map<String, String> invalidOutputSpecIdOptions =
+ ImmutableMap.of(RewriteDataFiles.OUTPUT_SPEC_ID, String.valueOf(1234));
+ assertThatThrownBy(() -> planner.init(invalidOutputSpecIdOptions))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage(
+ "Cannot use output spec id 1234 because the table does not contain a reference to this spec-id.");
+
+ Map<String, String> invalidDeleteFileThresholdOptions =
+ ImmutableMap.of(BinPackRewriteFilePlanner.DELETE_FILE_THRESHOLD, "-1");
+ assertThatThrownBy(() -> planner.init(invalidDeleteFileThresholdOptions))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("'delete-file-threshold' is set to -1 but must be >= 0");
+
+ Map<String, String> negativeDeleteRatioThresholdOptions =
+ ImmutableMap.of(BinPackRewriteFilePlanner.DELETE_RATIO_THRESHOLD, "-1");
+ assertThatThrownBy(() -> planner.init(negativeDeleteRatioThresholdOptions))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("'delete-ratio-threshold' is set to -1.0 but must be > 0");
+
+ Map<String, String> invalidDeleteRatioThresholdOptions =
+ ImmutableMap.of(BinPackRewriteFilePlanner.DELETE_RATIO_THRESHOLD, "127");
+ assertThatThrownBy(() -> planner.init(invalidDeleteRatioThresholdOptions))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("'delete-ratio-threshold' is set to 127.0 but must be <= 1");
+ }
+
+ @Test
+ void testSizeFiltering() {
+ BinPackRewriteFilePlanner planner = new BinPackRewriteFilePlanner(table);
+ FileScanTask tooSmallTask = new MockFileScanTask(100L);
+ FileScanTask optimal = new MockFileScanTask(450);
+ FileScanTask tooBigTask = new MockFileScanTask(1000L);
+ List<FileScanTask> tasks = ImmutableList.of(tooSmallTask, optimal, tooBigTask);
+
+ Map<String, String> options =
+ ImmutableMap.of(
+ BinPackRewriteFilePlanner.MIN_FILE_SIZE_BYTES, "250",
+ BinPackRewriteFilePlanner.TARGET_FILE_SIZE_BYTES, "500",
+ BinPackRewriteFilePlanner.MAX_FILE_SIZE_BYTES, "750");
+ planner.init(options);
+
+ Iterable<List<FileScanTask>> groups = planner.planFileGroups(tasks);
+ assertThat(groups).as("Must have 1 group").hasSize(1);
+ List<FileScanTask> group = Iterables.getOnlyElement(groups);
+ assertThat(group).as("Must rewrite 2 files").hasSize(2);
+ assertThat(group).containsExactlyInAnyOrder(tooSmallTask, tooBigTask);
+ }
+
+ @Test
+ void testGroupWithEnoughFiles() {
+ BinPackRewriteFilePlanner planner = new BinPackRewriteFilePlanner(table);
+ List<FileScanTask> tasks =
+ ImmutableList.of(
+ new MockFileScanTask(100L),
+ new MockFileScanTask(100L),
+ new MockFileScanTask(100L),
+ new MockFileScanTask(100L));
+
+ Map<String, String> options =
+ ImmutableMap.of(
+ BinPackRewriteFilePlanner.MIN_INPUT_FILES, "3",
+ BinPackRewriteFilePlanner.MIN_FILE_SIZE_BYTES, "150",
+ BinPackRewriteFilePlanner.TARGET_FILE_SIZE_BYTES, "1000",
+ BinPackRewriteFilePlanner.MAX_FILE_SIZE_BYTES, "5000");
+ planner.init(options);
+
+ Iterable<List<FileScanTask>> groups = planner.planFileGroups(tasks);
+ assertThat(groups).as("Must have 1 group").hasSize(1);
+ List<FileScanTask> group = Iterables.getOnlyElement(groups);
+ assertThat(group).as("Must rewrite 4 files").hasSize(4);
+ }
+
+ @Test
+ void testGroupWithEnoughData() {
+ BinPackRewriteFilePlanner planner = new BinPackRewriteFilePlanner(table);
+ List<FileScanTask> tasks =
+ ImmutableList.of(
+ new MockFileScanTask(100L), new MockFileScanTask(100L), new MockFileScanTask(100L));
+
+ Map<String, String> options =
+ ImmutableMap.of(
+ BinPackRewriteFilePlanner.MIN_FILE_SIZE_BYTES, "200",
+ BinPackRewriteFilePlanner.TARGET_FILE_SIZE_BYTES, "250",
+ BinPackRewriteFilePlanner.MAX_FILE_SIZE_BYTES, "500");
+ planner.init(options);
+
+ Iterable<List<FileScanTask>> groups = planner.planFileGroups(tasks);
+ assertThat(groups).as("Must have 1 group").hasSize(1);
+ List<FileScanTask> group = Iterables.getOnlyElement(groups);
+ assertThat(group).as("Must rewrite 3 files").hasSize(3);
+ }
+
+ @Test
+ void testFileSizeAboveTarget() {
+ BinPackRewriteFilePlanner planner = new BinPackRewriteFilePlanner(table);
+ List<FileScanTask> tasks = ImmutableList.of(new MockFileScanTask(2000L));
+
+ Map<String, String> options =
+ ImmutableMap.of(
+ BinPackRewriteFilePlanner.MIN_FILE_SIZE_BYTES, "200",
+ BinPackRewriteFilePlanner.TARGET_FILE_SIZE_BYTES, "250",
+ BinPackRewriteFilePlanner.MAX_FILE_SIZE_BYTES, "500");
+ planner.init(options);
+
+ Iterable<List<FileScanTask>> groups = planner.planFileGroups(tasks);
+ assertThat(groups).as("Must have 1 group").hasSize(1);
+ List<FileScanTask> group = Iterables.getOnlyElement(groups);
+ assertThat(group).as("Must rewrite big file").hasSize(1);
+ }
+
+ @Test
+ void testDeleteFileThreshold() {
+ BinPackRewriteFilePlanner planner = new BinPackRewriteFilePlanner(table);
+ FileScanTask tooManyDeletesTask = MockFileScanTask.mockTaskWithDeletes(1000L, 3);
+ FileScanTask optimalTask = MockFileScanTask.mockTaskWithDeletes(1001L, 1);
+ List<FileScanTask> tasks = ImmutableList.of(tooManyDeletesTask, optimalTask);
+
+ Map<String, String> options =
+ ImmutableMap.of(
+ BinPackRewriteFilePlanner.MIN_FILE_SIZE_BYTES, "0",
+ BinPackRewriteFilePlanner.DELETE_FILE_THRESHOLD, "2");
+ planner.init(options);
+
+ Iterable<List<FileScanTask>> groups = planner.planFileGroups(tasks);
+ assertThat(groups).as("Must have 1 group").hasSize(1);
+ List<FileScanTask> group = Iterables.getOnlyElement(groups);
+ assertThat(group).as("Must rewrite 1 file").hasSize(1);
+ assertThat(group).containsExactly(tooManyDeletesTask);
+ }
+
+ @Test
+ void testDeleteRatioThreshold() {
+ BinPackRewriteFilePlanner planner = new BinPackRewriteFilePlanner(table);
+ FileScanTask tooManyDeletesTask =
+ MockFileScanTask.mockTaskWithFileScopedDeleteRecords(1000L, 100, 1, 20);
+ FileScanTask optimalTask =
+ MockFileScanTask.mockTaskWithFileScopedDeleteRecords(1001L, 100, 1, 19);
+ List<FileScanTask> tasks = ImmutableList.of(tooManyDeletesTask, optimalTask);
+
+ Map<String, String> options =
+ ImmutableMap.of(
+ BinPackRewriteFilePlanner.MIN_FILE_SIZE_BYTES, "0",
+ BinPackRewriteFilePlanner.DELETE_RATIO_THRESHOLD, "0.2");
+ planner.init(options);
+
+ Iterable<List<FileScanTask>> groups = planner.planFileGroups(tasks);
+ assertThat(groups).as("Must have 1 group").hasSize(1);
+ List<FileScanTask> group = Iterables.getOnlyElement(groups);
+ assertThat(group).as("Must rewrite 1 file").hasSize(1);
+ assertThat(group).containsExactly(tooManyDeletesTask);
+ }
+
+ private void addFiles() {
+ table
+ .newAppend()
+ .appendFile(FILE_1)
+ .appendFile(FILE_2)
+ .appendFile(FILE_3)
+ .appendFile(FILE_4)
+ .appendFile(FILE_5)
+ .appendFile(FILE_6)
+ .commit();
+ }
+
+ private static DataFile newDataFile(String partitionPath, long fileSize) {
+ return DataFiles.builder(TestBase.SPEC)
+ .withPath("/path/to/data-" + UUID.randomUUID() + ".parquet")
+ .withFileSizeInBytes(fileSize)
+ .withPartitionPath(partitionPath)
+ .withRecordCount(1)
+ .build();
+ }
+}
diff --git a/core/src/test/java/org/apache/iceberg/actions/TestBinPackRewritePositionDeletePlanner.java b/core/src/test/java/org/apache/iceberg/actions/TestBinPackRewritePositionDeletePlanner.java
new file mode 100644
index 0000000000..c527b658de
--- /dev/null
+++ b/core/src/test/java/org/apache/iceberg/actions/TestBinPackRewritePositionDeletePlanner.java
@@ -0,0 +1,294 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.actions;
+
+import static org.apache.iceberg.actions.BinPackRewritePositionDeletePlanner.MAX_FILE_SIZE_DEFAULT_RATIO;
+import static org.apache.iceberg.actions.RewritePositionDeleteFiles.REWRITE_JOB_ORDER;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DataFiles;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileMetadata;
+import org.apache.iceberg.PartitionData;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PositionDeletesScanTask;
+import org.apache.iceberg.RewriteJobOrder;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.TestBase;
+import org.apache.iceberg.TestTables;
+import org.apache.iceberg.actions.RewritePositionDeleteFiles.FileGroupInfo;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+class TestBinPackRewritePositionDeletePlanner {
+ private static final Map<String, String> REWRITE_ALL =
+ ImmutableMap.of(BinPackRewritePositionDeletePlanner.REWRITE_ALL, "true");
+
+ private static final DataFile FILE_1 = newDataFile("data_bucket=0");
+ private static final DataFile FILE_2 = newDataFile("data_bucket=1");
+ private static final DataFile FILE_3 = newDataFile("data_bucket=2");
+ private static final Map<RewriteJobOrder, List<StructLike>> EXPECTED =
+ ImmutableMap.of(
+ RewriteJobOrder.FILES_DESC,
+ ImmutableList.of(FILE_1.partition(), FILE_2.partition(), FILE_3.partition()),
+ RewriteJobOrder.FILES_ASC,
+ ImmutableList.of(FILE_3.partition(), FILE_2.partition(), FILE_1.partition()),
+ RewriteJobOrder.BYTES_DESC,
+ ImmutableList.of(FILE_3.partition(), FILE_1.partition(), FILE_2.partition()),
+ RewriteJobOrder.BYTES_ASC,
+ ImmutableList.of(FILE_2.partition(), FILE_1.partition(), FILE_3.partition()));
+
+ @TempDir private File tableDir = null;
+ private TestTables.TestTable table = null;
+
+ @BeforeEach
+ public void setupTable() throws Exception {
+ this.table = TestTables.create(tableDir, "test", TestBase.SCHEMA, TestBase.SPEC, 2);
+ }
+
+ @AfterEach
+ public void cleanupTables() {
+ TestTables.clearTables();
+ }
+
+ @Test
+ void testPartitionedTable() {
+ addFiles();
+ BinPackRewritePositionDeletePlanner planner = new BinPackRewritePositionDeletePlanner(table);
+ planner.init(ImmutableMap.of(BinPackRewritePositionDeletePlanner.REWRITE_ALL, "true"));
+
+ FileRewritePlan<FileGroupInfo, PositionDeletesScanTask, DeleteFile, RewritePositionDeletesGroup>
+ plan = planner.plan();
+
+ List<RewritePositionDeletesGroup> groups = Lists.newArrayList(plan.groups().iterator());
+ assertThat(groups).hasSize(3);
+ assertThat(groups.stream().mapToInt(RewriteGroupBase::inputFileNum).sum()).isEqualTo(6);
+ }
+
+ @Test
+ void testUnpartitionedTable() {
+ table.updateSpec().removeField("data_bucket").commit();
+ table.refresh();
+ table
+ .newRowDelta()
+ .addRows(newDataFile(""))
+ .addDeletes(newDeleteFile(10))
+ .addDeletes(newDeleteFile(20))
+ .addDeletes(newDeleteFile(30))
+ .commit();
+
+ BinPackRewritePositionDeletePlanner planner = new BinPackRewritePositionDeletePlanner(table);
+ planner.init(
+ ImmutableMap.of(
+ BinPackRewriteFilePlanner.MIN_INPUT_FILES,
+ "1",
+ BinPackRewriteFilePlanner.MIN_FILE_SIZE_BYTES,
+ "30"));
+
+ FileRewritePlan<FileGroupInfo, PositionDeletesScanTask, DeleteFile, RewritePositionDeletesGroup>
+ plan = planner.plan();
+
+ assertThat(plan.totalGroupCount()).isEqualTo(1);
+ assertThat(plan.groups().iterator().next().inputFileNum()).isEqualTo(2);
+ }
+
+ @Test
+ void testEmptyTable() {
+ BinPackRewritePositionDeletePlanner planner = new BinPackRewritePositionDeletePlanner(table);
+ planner.init(REWRITE_ALL);
+
+ FileRewritePlan<FileGroupInfo, PositionDeletesScanTask, DeleteFile, RewritePositionDeletesGroup>
+ plan = planner.plan();
+
+ assertThat(table.currentSnapshot()).as("Table must be empty").isNull();
+ assertThat(plan.totalGroupCount()).isZero();
+ }
+
+ @ParameterizedTest
+ @EnumSource(
+ value = RewriteJobOrder.class,
+ names = {"FILES_DESC", "FILES_ASC", "BYTES_DESC", "BYTES_ASC"})
+ void testJobOrder(RewriteJobOrder order) {
+ addFiles();
+ BinPackRewritePositionDeletePlanner planner = new BinPackRewritePositionDeletePlanner(table);
+ planner.init(
+ ImmutableMap.of(
+ BinPackRewriteFilePlanner.REWRITE_ALL, "true", REWRITE_JOB_ORDER, order.name()));
+
+ FileRewritePlan<FileGroupInfo, PositionDeletesScanTask, DeleteFile, RewritePositionDeletesGroup>
+ plan = planner.plan();
+
+ List<RewritePositionDeletesGroup> groups = Lists.newArrayList(plan.groups().iterator());
+ assertThat(
+ groups.stream()
+ .map(
+ group ->
+ new PartitionData(TestBase.SPEC.partitionType())
+ .copyFor(group.info().partition()))
+ .collect(Collectors.toList()))
+ .isEqualTo(EXPECTED.get(order));
+ assertThat(plan.totalGroupCount()).isEqualTo(3);
+ EXPECTED.get(order).forEach(s -> assertThat(plan.groupsInPartition(s)).isEqualTo(1));
+ }
+
+ @Test
+ void testMaxGroupSize() {
+ addFiles();
+ BinPackRewritePositionDeletePlanner planner = new BinPackRewritePositionDeletePlanner(table);
+ planner.init(
+ ImmutableMap.of(
+ BinPackRewritePositionDeletePlanner.REWRITE_ALL,
+ "true",
+ BinPackRewritePositionDeletePlanner.MAX_FILE_GROUP_SIZE_BYTES,
+ "10"));
+
+ FileRewritePlan<FileGroupInfo, PositionDeletesScanTask, DeleteFile, RewritePositionDeletesGroup>
+ plan = planner.plan();
+
+ assertThat(plan.totalGroupCount()).isEqualTo(6);
+ assertThat(plan.groupsInPartition(FILE_1.partition())).isEqualTo(3);
+ assertThat(plan.groupsInPartition(FILE_2.partition())).isEqualTo(2);
+ assertThat(plan.groupsInPartition(FILE_3.partition())).isEqualTo(1);
+ }
+
+ @Test
+ void testFilter() {
+ addFiles();
+ BinPackRewritePositionDeletePlanner planner =
+ new BinPackRewritePositionDeletePlanner(
+ table,
+ Expressions.or(
+ Expressions.equal(Expressions.bucket("data", 16), 0),
+ Expressions.equal(Expressions.bucket("data", 16), 2)),
+ false);
+ planner.init(REWRITE_ALL);
+
+ FileRewritePlan<FileGroupInfo, PositionDeletesScanTask, DeleteFile, RewritePositionDeletesGroup>
+ plan = planner.plan();
+
+ List<RewritePositionDeletesGroup> groups = Lists.newArrayList(plan.groups().iterator());
+ assertThat(plan.totalGroupCount()).isEqualTo(2);
+ assertThat(groups).hasSize(2);
+ assertThat(groups.stream().mapToLong(RewritePositionDeletesGroup::inputFileNum).sum())
+ .isEqualTo(4);
+ }
+
+ @Test
+ void testMaxOutputFileSize() {
+ addFiles();
+ int targetFileSize = 10;
+ BinPackRewritePositionDeletePlanner planner = new BinPackRewritePositionDeletePlanner(table);
+ planner.init(
+ ImmutableMap.of(
+ BinPackRewritePositionDeletePlanner.REWRITE_ALL,
+ "true",
+ BinPackRewritePositionDeletePlanner.TARGET_FILE_SIZE_BYTES,
+ String.valueOf(targetFileSize)));
+
+ FileRewritePlan<FileGroupInfo, PositionDeletesScanTask, DeleteFile, RewritePositionDeletesGroup>
+ plan = planner.plan();
+
+ assertThat(plan.groups().iterator().next().maxOutputFileSize())
+ .isGreaterThan(targetFileSize)
+ .isLessThan((long) (targetFileSize * MAX_FILE_SIZE_DEFAULT_RATIO));
+ }
+
+ @Test
+ void testValidOptions() {
+ BinPackRewritePositionDeletePlanner planner = new BinPackRewritePositionDeletePlanner(table);
+
+ assertThat(planner.validOptions())
+ .as("Planner must report all supported options")
+ .isEqualTo(
+ ImmutableSet.of(
+ BinPackRewritePositionDeletePlanner.TARGET_FILE_SIZE_BYTES,
+ BinPackRewritePositionDeletePlanner.MIN_FILE_SIZE_BYTES,
+ BinPackRewritePositionDeletePlanner.MAX_FILE_SIZE_BYTES,
+ BinPackRewritePositionDeletePlanner.MIN_INPUT_FILES,
+ BinPackRewritePositionDeletePlanner.REWRITE_ALL,
+ BinPackRewritePositionDeletePlanner.MAX_FILE_GROUP_SIZE_BYTES,
+ RewriteDataFiles.REWRITE_JOB_ORDER));
+ }
+
+ @Test
+ void testInvalidOption() {
+ BinPackRewritePositionDeletePlanner planner = new BinPackRewritePositionDeletePlanner(table);
+
+ Map<String, String> invalidRewriteJobOrderOptions =
+ ImmutableMap.of(RewritePositionDeleteFiles.REWRITE_JOB_ORDER, "foo");
+ assertThatThrownBy(() -> planner.init(invalidRewriteJobOrderOptions))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid rewrite job order name: foo");
+ }
+
+ private void addFiles() {
+ table
+ .newRowDelta()
+ .addRows(FILE_1)
+ .addDeletes(newDeleteFile(FILE_1.partition(), 10))
+ .addDeletes(newDeleteFile(FILE_1.partition(), 10))
+ .addDeletes(newDeleteFile(FILE_1.partition(), 10))
+ .addRows(FILE_2)
+ .addDeletes(newDeleteFile(FILE_2.partition(), 11))
+ .addDeletes(newDeleteFile(FILE_2.partition(), 11))
+ .addRows(FILE_3)
+ .addDeletes(newDeleteFile(FILE_3.partition(), 50))
+ .commit();
+ }
+
+ private static DataFile newDataFile(String partitionPath) {
+ return DataFiles.builder(TestBase.SPEC)
+ .withPath("/path/to/data-" + UUID.randomUUID() + ".parquet")
+ .withFileSizeInBytes(10)
+ .withPartitionPath(partitionPath)
+ .withRecordCount(1)
+ .build();
+ }
+
+ private static DeleteFile newDeleteFile(long fileSize) {
+ return newDeleteFile(
+ new PartitionData(PartitionSpec.unpartitioned().partitionType()), fileSize);
+ }
+
+ private static DeleteFile newDeleteFile(StructLike partition, long fileSize) {
+ return FileMetadata.deleteFileBuilder(TestBase.SPEC)
+ .ofPositionDeletes()
+ .withPath("/path/to/delete-" + UUID.randomUUID() + ".parquet")
+ .withFileSizeInBytes(fileSize)
+ .withPartition(partition)
+ .withRecordCount(1)
+ .build();
+ }
+}
diff --git a/core/src/test/java/org/apache/iceberg/actions/TestSizeBasedFileRewritePlanner.java b/core/src/test/java/org/apache/iceberg/actions/TestSizeBasedFileRewritePlanner.java
new file mode 100644
index 0000000000..7d24e92338
--- /dev/null
+++ b/core/src/test/java/org/apache/iceberg/actions/TestSizeBasedFileRewritePlanner.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.actions;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.when;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.MockFileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.TestBase;
+import org.apache.iceberg.TestTables;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.Mockito;
+
+class TestSizeBasedFileRewritePlanner {
+ @TempDir private File tableDir = null;
+ private TestTables.TestTable table = null;
+
+ @BeforeEach
+ public void setupTable() throws Exception {
+ this.table = TestTables.create(tableDir, "test", TestBase.SCHEMA, TestBase.SPEC, 3);
+ }
+
+ @AfterEach
+ public void cleanupTables() {
+ TestTables.clearTables();
+ }
+
+ @Test
+ void testInputSplitSizeLowerBound() {
+ FileScanTask task1 = new MockFileScanTask(mockDataFile());
+ FileScanTask task2 = new MockFileScanTask(mockDataFile());
+ FileScanTask task3 = new MockFileScanTask(mockDataFile());
+ FileScanTask task4 = new MockFileScanTask(mockDataFile());
+ List<FileScanTask> tasks = ImmutableList.of(task1, task2, task3, task4);
+
+ TestingPlanner planner = new TestingPlanner(table);
+
+ long minFileSize = 256L * 1024 * 1024;
+ long targetFileSize = 512L * 1024 * 1024;
+ long maxFileSize = 768L * 1024 * 1024;
+
+ Map<String, String> options =
+ ImmutableMap.of(
+ BinPackRewriteFilePlanner.MIN_FILE_SIZE_BYTES, String.valueOf(minFileSize),
+ BinPackRewriteFilePlanner.TARGET_FILE_SIZE_BYTES, String.valueOf(targetFileSize),
+ BinPackRewriteFilePlanner.MAX_FILE_SIZE_BYTES, String.valueOf(maxFileSize));
+ planner.init(options);
+
+ // the total task size is 580 MB and the target file size is 512 MB
+ // the remainder must be written into a separate file as it exceeds 10%
+ List<List<FileScanTask>> groups = Lists.newArrayList(planner.planFileGroups(tasks).iterator());
+
+ assertThat(groups).hasSize(1);
+
+ List<FileScanTask> group = groups.get(0);
+ // the split size must be >= targetFileSize and < maxFileSize
+ long splitSize = group.stream().mapToLong(FileScanTask::sizeBytes).sum();
+ assertThat(splitSize).isGreaterThanOrEqualTo(targetFileSize).isLessThan(maxFileSize);
+ }
+
+ @Test
+ void testValidOptions() {
+ TestingPlanner planner = new TestingPlanner(table);
+
+ assertThat(planner.validOptions())
+ .as("Planner must report all supported options")
+ .isEqualTo(
+ ImmutableSet.of(
+ BinPackRewriteFilePlanner.TARGET_FILE_SIZE_BYTES,
+ BinPackRewriteFilePlanner.MIN_FILE_SIZE_BYTES,
+ BinPackRewriteFilePlanner.MAX_FILE_SIZE_BYTES,
+ BinPackRewriteFilePlanner.MIN_INPUT_FILES,
+ BinPackRewriteFilePlanner.REWRITE_ALL,
+ BinPackRewriteFilePlanner.MAX_FILE_GROUP_SIZE_BYTES));
+ }
+
+ @Test
+ void testInvalidOption() {
+ TestingPlanner planner = new TestingPlanner(table);
+
+ Map<String, String> invalidTargetSizeOptions =
+ ImmutableMap.of(SizeBasedFileRewritePlanner.TARGET_FILE_SIZE_BYTES, "0");
+ assertThatThrownBy(() -> planner.init(invalidTargetSizeOptions))
+ .hasMessageContaining("'target-file-size-bytes' is set to 0 but must be > 0");
+
+ Map<String, String> invalidMinSizeOptions =
+ ImmutableMap.of(SizeBasedFileRewritePlanner.MIN_FILE_SIZE_BYTES, "-1");
+ assertThatThrownBy(() -> planner.init(invalidMinSizeOptions))
+ .hasMessageContaining("'min-file-size-bytes' is set to -1 but must be >= 0");
+
+ Map<String, String> invalidTargetMinSizeOptions =
+ ImmutableMap.of(
+ SizeBasedFileRewritePlanner.TARGET_FILE_SIZE_BYTES, "3",
+ SizeBasedFileRewritePlanner.MIN_FILE_SIZE_BYTES, "5");
+ assertThatThrownBy(() -> planner.init(invalidTargetMinSizeOptions))
+ .hasMessageContaining("'target-file-size-bytes' (3) must be > 'min-file-size-bytes' (5)")
+ .hasMessageContaining("all new files will be smaller than the min threshold");
+
+ Map<String, String> invalidTargetMaxSizeOptions =
+ ImmutableMap.of(
+ SizeBasedFileRewritePlanner.TARGET_FILE_SIZE_BYTES, "5",
+ SizeBasedFileRewritePlanner.MAX_FILE_SIZE_BYTES, "3");
+ assertThatThrownBy(() -> planner.init(invalidTargetMaxSizeOptions))
+ .hasMessageContaining("'target-file-size-bytes' (5) must be < 'max-file-size-bytes' (3)")
+ .hasMessageContaining("all new files will be larger than the max threshold");
+
+ Map<String, String> invalidMinInputFilesOptions =
+ ImmutableMap.of(SizeBasedFileRewritePlanner.MIN_INPUT_FILES, "0");
+ assertThatThrownBy(() -> planner.init(invalidMinInputFilesOptions))
+ .hasMessageContaining("'min-input-files' is set to 0 but must be > 0");
+
+ Map<String, String> invalidMaxFileGroupSizeOptions =
+ ImmutableMap.of(SizeBasedFileRewritePlanner.MAX_FILE_GROUP_SIZE_BYTES, "0");
+ assertThatThrownBy(() -> planner.init(invalidMaxFileGroupSizeOptions))
+ .hasMessageContaining("'max-file-group-size-bytes' is set to 0 but must be > 0");
+ }
+
+ private static class TestingPlanner
+ extends SizeBasedFileRewritePlanner<
+ RewriteDataFiles.FileGroupInfo, FileScanTask, DataFile, RewriteFileGroup> {
+ protected TestingPlanner(Table table) {
+ super(table);
+ }
+
+ @Override
+ protected long defaultTargetFileSize() {
+ return TableProperties.WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT;
+ }
+
+ @Override
+ protected Iterable<FileScanTask> filterFiles(Iterable<FileScanTask> tasks) {
+ return tasks;
+ }
+
+ @Override
+ protected Iterable<List<FileScanTask>> filterFileGroups(List<List<FileScanTask>> groups) {
+ return groups;
+ }
+
+ @Override
+ public FileRewritePlan<RewriteDataFiles.FileGroupInfo, FileScanTask, DataFile, RewriteFileGroup>
+ plan() {
+ throw new UnsupportedOperationException("Not supported");
+ }
+ }
+
+ private DataFile mockDataFile() {
+ DataFile file = Mockito.mock(DataFile.class);
+ when(file.partition()).thenReturn(Mockito.mock(StructLike.class));
+ when(file.fileSizeInBytes()).thenReturn(145L * 1024 * 1024);
+ return file;
+ }
+}
diff --git a/core/src/test/java/org/apache/iceberg/actions/TestSizeBasedRewriter.java b/core/src/test/java/org/apache/iceberg/actions/TestSizeBasedRewriter.java
index 74cd3ca4fe..5cf78a973f 100644
--- a/core/src/test/java/org/apache/iceberg/actions/TestSizeBasedRewriter.java
+++ b/core/src/test/java/org/apache/iceberg/actions/TestSizeBasedRewriter.java
@@ -34,7 +34,11 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.junit.jupiter.api.TestTemplate;
import org.junit.jupiter.api.extension.ExtendWith;
+/**
+ * @deprecated The tested class is deprecated since 1.9.0, will be removed in 1.10.0
+ */
@ExtendWith(ParameterizedTestExtension.class)
+@Deprecated
public class TestSizeBasedRewriter extends TestBase {
@TestTemplate