You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2021/09/27 22:06:35 UTC
[iceberg] branch master updated: Spark: Avoid calling toGroupStream
if possible (#3173)
This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 7164ca9 Spark: Avoid calling toGroupStream if possible (#3173)
7164ca9 is described below
commit 7164ca93fe0b844dd13d994a253898fbfce7ac75
Author: xloya <98...@qq.com>
AuthorDate: Tue Sep 28 06:06:27 2021 +0800
Spark: Avoid calling toGroupStream if possible (#3173)
Co-authored-by: xiaojiebao <xi...@xiaomi.com>
---
.../apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java b/spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
index c49ac28..7a6a7a9 100644
--- a/spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
+++ b/spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
@@ -125,13 +125,14 @@ abstract class BaseRewriteDataFilesSparkAction
Map<StructLike, List<List<FileScanTask>>> fileGroupsByPartition = planFileGroups(startingSnapshotId);
RewriteExecutionContext ctx = new RewriteExecutionContext(fileGroupsByPartition);
- Stream<RewriteFileGroup> groupStream = toGroupStream(ctx, fileGroupsByPartition);
if (ctx.totalGroupCount() == 0) {
LOG.info("Nothing found to rewrite in {}", table.name());
return new BaseRewriteDataFilesResult(Collections.emptyList());
}
+ Stream<RewriteFileGroup> groupStream = toGroupStream(ctx, fileGroupsByPartition);
+
RewriteDataFilesCommitManager commitManager = commitManager(startingSnapshotId);
if (partialProgressEnabled) {
return doExecuteWithPartialProgress(ctx, groupStream, commitManager);