You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2021/09/27 22:06:35 UTC

[iceberg] branch master updated: Spark: Avoid calling toGroupStream if possible (#3173)

This is an automated email from the ASF dual-hosted git repository.

blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/master by this push:
     new 7164ca9  Spark: Avoid calling toGroupStream if possible (#3173)
7164ca9 is described below

commit 7164ca93fe0b844dd13d994a253898fbfce7ac75
Author: xloya <98...@qq.com>
AuthorDate: Tue Sep 28 06:06:27 2021 +0800

    Spark: Avoid calling toGroupStream if possible (#3173)
    
    Co-authored-by: xiaojiebao <xi...@xiaomi.com>
---
 .../apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java b/spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
index c49ac28..7a6a7a9 100644
--- a/spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
+++ b/spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
@@ -125,13 +125,14 @@ abstract class BaseRewriteDataFilesSparkAction
 
     Map<StructLike, List<List<FileScanTask>>> fileGroupsByPartition = planFileGroups(startingSnapshotId);
     RewriteExecutionContext ctx = new RewriteExecutionContext(fileGroupsByPartition);
-    Stream<RewriteFileGroup> groupStream = toGroupStream(ctx, fileGroupsByPartition);
 
     if (ctx.totalGroupCount() == 0) {
       LOG.info("Nothing found to rewrite in {}", table.name());
       return new BaseRewriteDataFilesResult(Collections.emptyList());
     }
 
+    Stream<RewriteFileGroup> groupStream = toGroupStream(ctx, fileGroupsByPartition);
+
     RewriteDataFilesCommitManager commitManager = commitManager(startingSnapshotId);
     if (partialProgressEnabled) {
       return doExecuteWithPartialProgress(ctx, groupStream, commitManager);