You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by fo...@apache.org on 2023/01/04 03:32:53 UTC
[hudi] 43/45: check parquet file does not exist
This is an automated email from the ASF dual-hosted git repository.
forwardxu pushed a commit to branch release-0.12.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit d0b3b36e96d326d9f683f1d557527e4e50d92e78
Author: XuQianJin-Stars <fo...@apache.com>
AuthorDate: Wed Dec 28 10:06:17 2022 +0800
check parquet file does not exist
---
.../src/main/java/org/apache/hudi/io/HoodieMergeHandle.java | 8 +++++++-
.../apache/hudi/io/storage/row/HoodieRowDataParquetWriter.java | 4 +++-
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
index c569acdda6..17bbb2f7f0 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
@@ -445,7 +445,13 @@ public class HoodieMergeHandle<T extends HoodieRecordPayload, I, K, O> extends H
return;
}
- IOUtils.checkParquetFileVaid(hoodieTable.getHadoopConf(), newFilePath);
+ try {
+ if (fs.exists(newFilePath)) {
+ IOUtils.checkParquetFileVaid(hoodieTable.getHadoopConf(), newFilePath);
+ }
+ } catch (IOException e) {
+ throw new HoodieUpsertException("Failed to check for merge data validation", e);
+ }
long oldNumWrites = 0;
try {
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetWriter.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetWriter.java
index fd1edaab84..66a830887a 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetWriter.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetWriter.java
@@ -75,6 +75,8 @@ public class HoodieRowDataParquetWriter extends ParquetWriter<RowData>
@Override
public void close() throws IOException {
super.close();
- IOUtils.checkParquetFileVaid(fs.getConf(), file);
+ if (fs.exists(file)) {
+ IOUtils.checkParquetFileVaid(fs.getConf(), file);
+ }
}
}