You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by da...@apache.org on 2023/08/31 01:37:31 UTC
[hudi] branch master updated: [HUDI-6763] Optimize collect calls (#9561)
This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 2f7e9caebb0 [HUDI-6763] Optimize collect calls (#9561)
2f7e9caebb0 is described below
commit 2f7e9caebb0e7f68a7cc1a9c541cc67440eafa44
Author: Tim Brown <ti...@onehouse.ai>
AuthorDate: Wed Aug 30 20:37:23 2023 -0500
[HUDI-6763] Optimize collect calls (#9561)
---
.../table/action/commit/BaseSparkCommitActionExecutor.java | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java
index 7383f428e0a..040cc798747 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java
@@ -286,7 +286,9 @@ public abstract class BaseSparkCommitActionExecutor<T> extends
@Override
protected void setCommitMetadata(HoodieWriteMetadata<HoodieData<WriteStatus>> result) {
- result.setCommitMetadata(Option.of(CommitUtils.buildMetadata(result.getWriteStatuses().map(WriteStatus::getStat).collectAsList(),
+ List<HoodieWriteStat> writeStats = result.getWriteStatuses().map(WriteStatus::getStat).collectAsList();
+ result.setWriteStats(writeStats);
+ result.setCommitMetadata(Option.of(CommitUtils.buildMetadata(writeStats,
result.getPartitionToReplaceFileIds(),
extraMetadata, operationType, getSchemaToStoreInCommit(), getCommitActionType())));
}
@@ -294,16 +296,14 @@ public abstract class BaseSparkCommitActionExecutor<T> extends
@Override
protected void commit(Option<Map<String, String>> extraMetadata, HoodieWriteMetadata<HoodieData<WriteStatus>> result) {
context.setJobStatus(this.getClass().getSimpleName(), "Commit write status collect: " + config.getTableName());
- commit(extraMetadata, result, result.getWriteStatuses().map(WriteStatus::getStat).collectAsList());
- }
-
- protected void commit(Option<Map<String, String>> extraMetadata, HoodieWriteMetadata<HoodieData<WriteStatus>> result, List<HoodieWriteStat> writeStats) {
String actionType = getCommitActionType();
LOG.info("Committing " + instantTime + ", action Type " + actionType + ", operation Type " + operationType);
result.setCommitted(true);
- result.setWriteStats(writeStats);
+ if (!result.getWriteStats().isPresent()) {
+ result.setWriteStats(result.getWriteStatuses().map(WriteStatus::getStat).collectAsList());
+ }
// Finalize write
- finalizeWrite(instantTime, writeStats, result);
+ finalizeWrite(instantTime, result.getWriteStats().get(), result);
try {
HoodieActiveTimeline activeTimeline = table.getActiveTimeline();
HoodieCommitMetadata metadata = result.getCommitMetadata().get();