You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by da...@apache.org on 2025/11/07 02:49:41 UTC
(hudi) branch master updated: perf: reduce unnecessary row group metadata loading (#14208)
This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 355f19c03309 perf: reduce unnecessary row group metadata loading (#14208)
355f19c03309 is described below
commit 355f19c03309b32a7e2787e340b2874844d5104c
Author: TheR1sing3un <ch...@apache.org>
AuthorDate: Fri Nov 7 10:49:35 2025 +0800
perf: reduce unnecessary row group metadata loading (#14208)
* perf: reduce unnecessary row group metadata loading
---------
Signed-off-by: TheR1sing3un <ch...@apache.org>
---
.../org/apache/hudi/common/util/ParquetUtils.java | 17 ++++++++++++++---
.../apache/hudi/common/util/TestParquetUtils.java | 20 ++++++++++++++++++++
2 files changed, 34 insertions(+), 3 deletions(-)
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java
index 78c7bdbaa22f..63bc33bdefb7 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.parquet.avro.AvroParquetReader;
import org.apache.parquet.avro.AvroReadSupport;
import org.apache.parquet.column.statistics.Statistics;
+import org.apache.parquet.format.converter.ParquetMetadataConverter;
import org.apache.parquet.hadoop.ParquetFileReader;
import org.apache.parquet.hadoop.ParquetReader;
import org.apache.parquet.hadoop.ParquetWriter;
@@ -86,6 +87,8 @@ import static org.apache.hudi.common.config.HoodieStorageConfig.PARQUET_MAX_FILE
import static org.apache.hudi.common.config.HoodieStorageConfig.PARQUET_PAGE_SIZE;
import static org.apache.hudi.hadoop.fs.HadoopFSUtils.convertToStoragePath;
import static org.apache.parquet.avro.HoodieAvroParquetSchemaConverter.getAvroSchemaConverter;
+import static org.apache.parquet.format.converter.ParquetMetadataConverter.NO_FILTER;
+import static org.apache.parquet.format.converter.ParquetMetadataConverter.SKIP_ROW_GROUPS;
/**
* Utility functions involving with parquet.
@@ -109,12 +112,20 @@ public class ParquetUtils extends FileFormatUtils {
}
public static ParquetMetadata readMetadata(HoodieStorage storage, StoragePath parquetFilePath) {
+ return readMetadata(storage, parquetFilePath, NO_FILTER);
+ }
+
+ public static ParquetMetadata readFileMetadataOnly(HoodieStorage storage, StoragePath parquetFilePath) {
+ return readMetadata(storage, parquetFilePath, SKIP_ROW_GROUPS);
+ }
+
+ private static ParquetMetadata readMetadata(HoodieStorage storage, StoragePath parquetFilePath, ParquetMetadataConverter.MetadataFilter metadataFilter) {
Path parquetFileHadoopPath = new Path(parquetFilePath.toUri());
ParquetMetadata footer;
try {
// TODO(vc): Should we use the parallel reading version here?
footer = ParquetFileReader.readFooter(storage.newInstance(
- parquetFilePath, storage.getConf()).getConf().unwrapAs(Configuration.class), parquetFileHadoopPath);
+ parquetFilePath, storage.getConf()).getConf().unwrapAs(Configuration.class), parquetFileHadoopPath, metadataFilter);
} catch (IOException e) {
throw new HoodieIOException("Failed to read footer for parquet " + parquetFileHadoopPath, e);
}
@@ -233,7 +244,7 @@ public class ParquetUtils extends FileFormatUtils {
* Get the schema of the given parquet file.
*/
public MessageType readSchema(HoodieStorage storage, StoragePath parquetFilePath) {
- return readMetadata(storage, parquetFilePath).getFileMetaData().getSchema();
+ return readFileMetadataOnly(storage, parquetFilePath).getFileMetaData().getSchema();
}
/**
@@ -255,7 +266,7 @@ public class ParquetUtils extends FileFormatUtils {
public Map<String, String> readFooter(HoodieStorage storage, boolean required,
StoragePath filePath, String... footerNames) {
Map<String, String> footerVals = new HashMap<>();
- ParquetMetadata footer = readMetadata(storage, filePath);
+ ParquetMetadata footer = readFileMetadataOnly(storage, filePath);
Map<String, String> metadata = footer.getFileMetaData().getKeyValueMetaData();
for (String footerName : footerNames) {
if (metadata.containsKey(footerName)) {
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java
index c567f3fb4386..dc7681e7fb12 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.parquet.avro.AvroSchemaConverter;
import org.apache.parquet.hadoop.ParquetWriter;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
import org.apache.parquet.schema.MessageType;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -66,6 +67,7 @@ import static org.apache.hudi.avro.AvroSchemaUtils.createNullableSchema;
import static org.apache.hudi.avro.HoodieAvroUtils.METADATA_FIELD_SCHEMA;
import static org.apache.hudi.metadata.HoodieIndexVersion.V1;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
@@ -503,4 +505,22 @@ public class TestParquetUtils extends HoodieCommonTestHarness {
}
writer.close();
}
+
+ @Test
+ public void testReadParquetMetadata() throws Exception {
+ List<String> rowKeys = new ArrayList<>();
+ for (int i = 0; i < 1000; i++) {
+ rowKeys.add(UUID.randomUUID().toString());
+ }
+
+ String filePath = Paths.get(basePath, "test.parquet").toUri().toString();
+ writeParquetFile(BloomFilterTypeCode.SIMPLE.name(), filePath, rowKeys);
+
+ ParquetMetadata metadata = parquetUtils.readMetadata(HoodieTestUtils.getStorage(filePath), new StoragePath(filePath));
+ ParquetMetadata metadataWithSkipRowGroups = parquetUtils.readFileMetadataOnly(HoodieTestUtils.getStorage(filePath), new StoragePath(filePath));
+ assertTrue(metadata.getFileMetaData() != null);
+ assertTrue(metadataWithSkipRowGroups.getFileMetaData() != null);
+ assertFalse(metadata.getBlocks().isEmpty());
+ assertTrue(metadataWithSkipRowGroups.getBlocks().isEmpty());
+ }
}