You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by da...@apache.org on 2024/12/12 07:22:21 UTC
(hudi) branch master updated: [HUDI-8626] Create consistent-hash-metadata file using #rename (#12394)
This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 5fed226c723 [HUDI-8626] Create consistent-hash-metadata file using #rename (#12394)
5fed226c723 is described below
commit 5fed226c7231b8da4a6e7b978a41300418d79d92
Author: TheR1sing3un <ch...@apache.org>
AuthorDate: Thu Dec 12 15:22:15 2024 +0800
[HUDI-8626] Create consistent-hash-metadata file using #rename (#12394)
---
.../hudi/index/bucket/ConsistentBucketIndexUtils.java | 19 +++++++------------
.../bucket/HoodieSparkConsistentBucketIndex.java | 4 +++-
.../java/org/apache/hudi/storage/HoodieStorage.java | 1 -
3 files changed, 10 insertions(+), 14 deletions(-)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java
index 1cd5d110de5..e5014bb8f13 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java
@@ -40,7 +40,6 @@ import org.slf4j.LoggerFactory;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
-import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
@@ -72,7 +71,6 @@ public class ConsistentBucketIndexUtils {
* @param table Hoodie table
* @param partition Table partition
* @param numBuckets Default bucket number
- *
* @return Consistent hashing metadata
*/
public static HoodieConsistentHashingMetadata loadOrCreateMetadata(HoodieTable table, String partition, int numBuckets) {
@@ -85,7 +83,7 @@ public class ConsistentBucketIndexUtils {
// There is no metadata, so try to create a new one and save it.
HoodieConsistentHashingMetadata metadata = new HoodieConsistentHashingMetadata(partition, numBuckets);
- if (saveMetadata(table, metadata, false)) {
+ if (saveMetadata(table, metadata)) {
return metadata;
}
@@ -177,25 +175,22 @@ public class ConsistentBucketIndexUtils {
/**
* Saves the metadata into storage
*
- * @param table Hoodie table
- * @param metadata Hashing metadata to be saved
- * @param overwrite Whether to overwrite existing metadata
+ * @param table Hoodie table
+ * @param metadata Hashing metadata to be saved
* @return true if the metadata is saved successfully
*/
- public static boolean saveMetadata(HoodieTable table, HoodieConsistentHashingMetadata metadata, boolean overwrite) {
+ public static boolean saveMetadata(HoodieTable table, HoodieConsistentHashingMetadata metadata) {
HoodieStorage storage = table.getStorage();
StoragePath dir = FSUtils.constructAbsolutePath(
table.getMetaClient().getHashingMetadataPath(), metadata.getPartitionPath());
StoragePath fullPath = new StoragePath(dir, metadata.getFilename());
- try (OutputStream out = storage.create(fullPath, overwrite)) {
- byte[] bytes = metadata.toBytes();
- out.write(bytes);
- out.close();
+ try {
+ storage.createImmutableFileInPath(fullPath, Option.of(metadata.toBytes()));
return true;
} catch (IOException e) {
LOG.warn("Failed to update bucket metadata: " + metadata, e);
+ return false;
}
- return false;
}
/***
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bucket/HoodieSparkConsistentBucketIndex.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bucket/HoodieSparkConsistentBucketIndex.java
index 3fc6fe83310..bc8130e38b7 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bucket/HoodieSparkConsistentBucketIndex.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bucket/HoodieSparkConsistentBucketIndex.java
@@ -103,7 +103,9 @@ public class HoodieSparkConsistentBucketIndex extends HoodieConsistentBucketInde
.collect(Collectors.toList());
HoodieConsistentHashingMetadata newMeta = new HoodieConsistentHashingMetadata(meta.getVersion(), meta.getPartitionPath(),
instantTime, meta.getNumBuckets(), seqNo + 1, newNodes);
- ConsistentBucketIndexUtils.saveMetadata(hoodieTable, newMeta, true);
+ if (!ConsistentBucketIndexUtils.saveMetadata(hoodieTable, newMeta)) {
+ throw new HoodieIndexException("Failed to save metadata for partition: " + partition);
+ }
});
return writeStatuses;
diff --git a/hudi-io/src/main/java/org/apache/hudi/storage/HoodieStorage.java b/hudi-io/src/main/java/org/apache/hudi/storage/HoodieStorage.java
index 8de04f6903e..8d5cbb38025 100644
--- a/hudi-io/src/main/java/org/apache/hudi/storage/HoodieStorage.java
+++ b/hudi-io/src/main/java/org/apache/hudi/storage/HoodieStorage.java
@@ -313,7 +313,6 @@ public abstract class HoodieStorage implements Closeable {
StoragePath tmpPath = null;
boolean needTempFile = needCreateTempFile();
-
try {
if (!content.isPresent()) {
fsout = create(path, false);