You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by yi...@apache.org on 2023/03/07 17:48:16 UTC
[hudi] branch master updated: [HUDI-5882] Close table metadata and file system view instances in Metadata Table Validator (#8106)
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new c3bfd129f8e [HUDI-5882] Close table metadata and file system view instances in Metadata Table Validator (#8106)
c3bfd129f8e is described below
commit c3bfd129f8eb2c9ff3866bc1ae1236c61674f67e
Author: Y Ethan Guo <et...@gmail.com>
AuthorDate: Tue Mar 7 09:48:10 2023 -0800
[HUDI-5882] Close table metadata and file system view instances in Metadata Table Validator (#8106)
This commit improves the Metadata Table Validator (`HoodieMetadataTableValidator`) to properly close the table metadata and file system view instances used by the `HoodieMetadataValidationContext`.
---
.../utilities/HoodieMetadataTableValidator.java | 70 ++++++++++++----------
1 file changed, 40 insertions(+), 30 deletions(-)
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
index 5bd54d1e8ca..ccf30cbf37a 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
@@ -445,41 +445,45 @@ public class HoodieMetadataTableValidator implements Serializable {
return true;
}
- HoodieMetadataValidationContext metadataTableBasedContext =
- new HoodieMetadataValidationContext(engineContext, cfg, metaClient, true);
- HoodieMetadataValidationContext fsBasedContext =
- new HoodieMetadataValidationContext(engineContext, cfg, metaClient, false);
+ try (HoodieMetadataValidationContext metadataTableBasedContext =
+ new HoodieMetadataValidationContext(engineContext, cfg, metaClient, true);
+ HoodieMetadataValidationContext fsBasedContext =
+ new HoodieMetadataValidationContext(engineContext, cfg, metaClient, false)) {
+ Set<String> finalBaseFilesForCleaning = baseFilesForCleaning;
+ List<Pair<Boolean, String>> result = engineContext.parallelize(allPartitions, allPartitions.size()).map(partitionPath -> {
+ try {
+ validateFilesInPartition(metadataTableBasedContext, fsBasedContext, partitionPath, finalBaseFilesForCleaning);
+ LOG.info(String.format("Metadata table validation succeeded for partition %s (partition %s)", partitionPath, taskLabels));
+ return Pair.of(true, "");
+ } catch (HoodieValidationException e) {
+ LOG.error(
+ String.format("Metadata table validation failed for partition %s due to HoodieValidationException (partition %s)",
+ partitionPath, taskLabels), e);
+ if (!cfg.ignoreFailed) {
+ throw e;
+ }
+ return Pair.of(false, e.getMessage() + " for partition: " + partitionPath);
+ }
+ }).collectAsList();
- Set<String> finalBaseFilesForCleaning = baseFilesForCleaning;
- List<Pair<Boolean, String>> result = engineContext.parallelize(allPartitions, allPartitions.size()).map(partitionPath -> {
- try {
- validateFilesInPartition(metadataTableBasedContext, fsBasedContext, partitionPath, finalBaseFilesForCleaning);
- LOG.info(String.format("Metadata table validation succeeded for partition %s (partition %s)", partitionPath, taskLabels));
- return Pair.of(true, "");
- } catch (HoodieValidationException e) {
- LOG.error(
- String.format("Metadata table validation failed for partition %s due to HoodieValidationException (partition %s)",
- partitionPath, taskLabels), e);
- if (!cfg.ignoreFailed) {
- throw e;
+ for (Pair<Boolean, String> res : result) {
+ finalResult &= res.getKey();
+ if (res.getKey().equals(false)) {
+ LOG.error("Metadata Validation failed for table: " + cfg.basePath + " with error: " + res.getValue());
}
- return Pair.of(false, e.getMessage() + " for partition: " + partitionPath);
}
- }).collectAsList();
- for (Pair<Boolean, String> res : result) {
- finalResult &= res.getKey();
- if (res.getKey().equals(false)) {
- LOG.error("Metadata Validation failed for table: " + cfg.basePath + " with error: " + res.getValue());
+ if (finalResult) {
+ LOG.info(String.format("Metadata table validation succeeded (%s).", taskLabels));
+ return true;
+ } else {
+ LOG.warn(String.format("Metadata table validation failed (%s).", taskLabels));
+ return false;
}
- }
-
- if (finalResult) {
- LOG.info(String.format("Metadata table validation succeeded (%s).", taskLabels));
+ } catch (Exception e) {
+ LOG.warn("Error closing HoodieMetadataValidationContext, "
+ + "ignoring the error as the validation is successful.", e);
return true;
- } else {
- LOG.warn(String.format("Metadata table validation failed (%s).", taskLabels));
- return false;
}
}
@@ -1020,7 +1024,7 @@ public class HoodieMetadataTableValidator implements Serializable {
* the same information regardless of whether metadata table is enabled, which is
* verified in the {@link HoodieMetadataTableValidator}.
*/
- private static class HoodieMetadataValidationContext implements Serializable {
+ private static class HoodieMetadataValidationContext implements AutoCloseable, Serializable {
private static final Logger LOG = LogManager.getLogger(HoodieMetadataValidationContext.class);
@@ -1147,5 +1151,11 @@ public class HoodieMetadataTableValidator implements Serializable {
.setBloomFilter(ByteBuffer.wrap(bloomFilter.serializeToString().getBytes()))
.build());
}
+
+ @Override
+ public void close() throws Exception {
+ tableMetadata.close();
+ fileSystemView.close();
+ }
}
}