You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by pw...@apache.org on 2023/09/01 11:28:17 UTC
[hudi] 30/30: [HUDI-6763] Optimize collect calls (#9561)
This is an automated email from the ASF dual-hosted git repository.
pwason pushed a commit to branch release-0.14.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit d995bb8262cafa22253fa961557bbfcde6369dfb
Author: Tim Brown <ti...@onehouse.ai>
AuthorDate: Wed Aug 30 20:37:23 2023 -0500
[HUDI-6763] Optimize collect calls (#9561)
---
.../table/action/commit/BaseSparkCommitActionExecutor.java | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java
index 7383f428e0a..040cc798747 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java
@@ -286,7 +286,9 @@ public abstract class BaseSparkCommitActionExecutor<T> extends
@Override
protected void setCommitMetadata(HoodieWriteMetadata<HoodieData<WriteStatus>> result) {
- result.setCommitMetadata(Option.of(CommitUtils.buildMetadata(result.getWriteStatuses().map(WriteStatus::getStat).collectAsList(),
+ List<HoodieWriteStat> writeStats = result.getWriteStatuses().map(WriteStatus::getStat).collectAsList();
+ result.setWriteStats(writeStats);
+ result.setCommitMetadata(Option.of(CommitUtils.buildMetadata(writeStats,
result.getPartitionToReplaceFileIds(),
extraMetadata, operationType, getSchemaToStoreInCommit(), getCommitActionType())));
}
@@ -294,16 +296,14 @@ public abstract class BaseSparkCommitActionExecutor<T> extends
@Override
protected void commit(Option<Map<String, String>> extraMetadata, HoodieWriteMetadata<HoodieData<WriteStatus>> result) {
context.setJobStatus(this.getClass().getSimpleName(), "Commit write status collect: " + config.getTableName());
- commit(extraMetadata, result, result.getWriteStatuses().map(WriteStatus::getStat).collectAsList());
- }
-
- protected void commit(Option<Map<String, String>> extraMetadata, HoodieWriteMetadata<HoodieData<WriteStatus>> result, List<HoodieWriteStat> writeStats) {
String actionType = getCommitActionType();
LOG.info("Committing " + instantTime + ", action Type " + actionType + ", operation Type " + operationType);
result.setCommitted(true);
- result.setWriteStats(writeStats);
+ if (!result.getWriteStats().isPresent()) {
+ result.setWriteStats(result.getWriteStatuses().map(WriteStatus::getStat).collectAsList());
+ }
// Finalize write
- finalizeWrite(instantTime, writeStats, result);
+ finalizeWrite(instantTime, result.getWriteStats().get(), result);
try {
HoodieActiveTimeline activeTimeline = table.getActiveTimeline();
HoodieCommitMetadata metadata = result.getCommitMetadata().get();