You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by si...@apache.org on 2022/01/07 18:06:40 UTC
[hudi] 18/21: [HUDI-3170] Do not preserve filename when preserveCommitMetadata enabled (#4512)
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.10.1-rc1
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit c4673b228d2da65804272d07809748fa7e61b5f8
Author: Sagar Sumit <sa...@gmail.com>
AuthorDate: Wed Jan 5 18:39:58 2022 +0530
[HUDI-3170] Do not preserve filename when preserveCommitMetadata enabled (#4512)
---
.../src/main/java/org/apache/hudi/io/HoodieCreateHandle.java | 2 ++
.../hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java | 3 +++
2 files changed, 5 insertions(+)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
index 01ad453..a110cd9 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
@@ -139,6 +139,8 @@ public class HoodieCreateHandle<T extends HoodieRecordPayload, I, K, O> extends
// Convert GenericRecord to GenericRecord with hoodie commit metadata in schema
IndexedRecord recordWithMetadataInSchema = rewriteRecord((GenericRecord) avroRecord.get());
if (preserveHoodieMetadata) {
+ // do not preserve FILENAME_METADATA_FIELD
+ recordWithMetadataInSchema.put(HoodieRecord.HOODIE_META_COLUMNS_NAME_TO_POS.get(HoodieRecord.FILENAME_METADATA_FIELD), path.getName());
fileWriter.writeAvro(record.getRecordKey(), recordWithMetadataInSchema);
} else {
fileWriter.writeAvroWithMetadata(recordWithMetadataInSchema, record);
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
index 566db22..aa3ead4 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
@@ -1836,9 +1836,12 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
Map<String, List<GenericRecord>> recordsByCommitTime = records.stream()
.collect(Collectors.groupingBy(r -> r.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString()));
assertTrue(commitTimes.containsAll(recordsByCommitTime.keySet()));
+ Set<String> expectedFileIds = allStatus.stream().map(WriteStatus::getFileId).collect(Collectors.toSet());
for (GenericRecord record : records) {
String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
assertTrue(expectedKeys.contains(recordKey));
+ String fileName = record.get(HoodieRecord.FILENAME_METADATA_FIELD).toString();
+ assertTrue(expectedFileIds.contains(FSUtils.getFileId(fileName)));
}
}