You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by yi...@apache.org on 2024/05/14 23:51:25 UTC
(hudi) 20/22: [HUDI-7619] Removed code duplicates in HoodieTableMetadataUtil (#11022)
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 8a1b6f3845c611dda6a8d5480e383a072bf2759c
Author: Vova Kolmakov <wo...@gmail.com>
AuthorDate: Tue May 14 16:01:09 2024 -0700
[HUDI-7619] Removed code duplicates in HoodieTableMetadataUtil (#11022)
Co-authored-by: Vova Kolmakov <ko...@huawei-partners.com>
---
.../hudi/metadata/HoodieTableMetadataUtil.java | 92 +++++++++-------------
1 file changed, 36 insertions(+), 56 deletions(-)
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
index b25d6741b83..503e3351d8c 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
@@ -73,6 +73,7 @@ import org.apache.hudi.exception.HoodieMetadataException;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieFileReaderFactory;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.util.Lazy;
import org.apache.avro.AvroTypeException;
@@ -1749,26 +1750,7 @@ public class HoodieTableMetadataUtil {
final String instantTime = baseFile.getCommitTime();
HoodieFileReader reader = HoodieFileReaderFactory.getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
.getFileReader(config, configuration.get(), dataFilePath);
- ClosableIterator<String> recordKeyIterator = reader.getRecordKeyIterator();
-
- return new ClosableIterator<HoodieRecord>() {
- @Override
- public void close() {
- recordKeyIterator.close();
- }
-
- @Override
- public boolean hasNext() {
- return recordKeyIterator.hasNext();
- }
-
- @Override
- public HoodieRecord next() {
- return forDelete
- ? HoodieMetadataPayload.createRecordIndexDelete(recordKeyIterator.next())
- : HoodieMetadataPayload.createRecordIndexUpdate(recordKeyIterator.next(), partition, fileId, instantTime, 0);
- }
- };
+ return getHoodieRecordIterator(reader.getRecordKeyIterator(), forDelete, partition, fileId, instantTime);
});
}
@@ -1816,24 +1798,7 @@ public class HoodieTableMetadataUtil {
.withTableMetaClient(metaClient)
.build();
ClosableIterator<String> recordKeyIterator = ClosableIterator.wrap(mergedLogRecordScanner.getRecords().keySet().iterator());
- return new ClosableIterator<HoodieRecord>() {
- @Override
- public void close() {
- recordKeyIterator.close();
- }
-
- @Override
- public boolean hasNext() {
- return recordKeyIterator.hasNext();
- }
-
- @Override
- public HoodieRecord next() {
- return forDelete
- ? HoodieMetadataPayload.createRecordIndexDelete(recordKeyIterator.next())
- : HoodieMetadataPayload.createRecordIndexUpdate(recordKeyIterator.next(), partition, fileSlice.getFileId(), fileSlice.getBaseInstantTime(), 0);
- }
- };
+ return getHoodieRecordIterator(recordKeyIterator, forDelete, partition, fileSlice.getFileId(), fileSlice.getBaseInstantTime());
}
final HoodieBaseFile baseFile = fileSlice.getBaseFile().get();
final String filename = baseFile.getFileName();
@@ -1844,26 +1809,41 @@ public class HoodieTableMetadataUtil {
HoodieConfig hoodieConfig = getReaderConfigs(configuration.get());
HoodieFileReader reader = HoodieFileReaderFactory.getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
.getFileReader(hoodieConfig, configuration.get(), dataFilePath);
- ClosableIterator<String> recordKeyIterator = reader.getRecordKeyIterator();
+ return getHoodieRecordIterator(reader.getRecordKeyIterator(), forDelete, partition, fileId, instantTime);
+ });
+ }
- return new ClosableIterator<HoodieRecord>() {
- @Override
- public void close() {
- recordKeyIterator.close();
- }
+ private static Path filePath(String basePath, String partition, String filename) {
+ if (partition.isEmpty()) {
+ return new Path(basePath, filename);
+ } else {
+ return new Path(basePath, partition + StoragePath.SEPARATOR + filename);
+ }
+ }
- @Override
- public boolean hasNext() {
- return recordKeyIterator.hasNext();
- }
+ private static ClosableIterator<HoodieRecord> getHoodieRecordIterator(ClosableIterator<String> recordKeyIterator,
+ boolean forDelete,
+ String partition,
+ String fileId,
+ String instantTime
+ ) {
+ return new ClosableIterator<HoodieRecord>() {
+ @Override
+ public void close() {
+ recordKeyIterator.close();
+ }
- @Override
- public HoodieRecord next() {
- return forDelete
- ? HoodieMetadataPayload.createRecordIndexDelete(recordKeyIterator.next())
- : HoodieMetadataPayload.createRecordIndexUpdate(recordKeyIterator.next(), partition, fileId, instantTime, 0);
- }
- };
- });
+ @Override
+ public boolean hasNext() {
+ return recordKeyIterator.hasNext();
+ }
+
+ @Override
+ public HoodieRecord next() {
+ return forDelete
+ ? HoodieMetadataPayload.createRecordIndexDelete(recordKeyIterator.next())
+ : HoodieMetadataPayload.createRecordIndexUpdate(recordKeyIterator.next(), partition, fileId, instantTime, 0);
+ }
+ };
}
}