You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vh...@apache.org on 2025/09/17 09:17:49 UTC

(hudi) 01/01: Minor add configurable hfile config

This is an automated email from the ASF dual-hosted git repository.

vhs pushed a commit to branch minor-add-configurable-hfile-config
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit ddf7f9a9c6a67388febd321e77ee78d86a577173
Author: voon <vo...@gmail.com>
AuthorDate: Wed Sep 17 17:17:29 2025 +0800

    Minor add configurable hfile config
---
 .../apache/hudi/avro/HoodieAvroReaderContext.java  | 30 ++++++++++++++++++----
 .../scala/org/apache/hudi/HoodieBaseRelation.scala | 12 +++++++++
 2 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroReaderContext.java b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroReaderContext.java
index c90dc0186741..4c5bf59dccd1 100644
--- a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroReaderContext.java
+++ b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroReaderContext.java
@@ -66,6 +66,7 @@ import static org.apache.hudi.common.util.ValidationUtils.checkState;
 public class HoodieAvroReaderContext extends HoodieReaderContext<IndexedRecord> {
   private final Map<StoragePath, HoodieAvroFileReader> reusableFileReaders;
   private final boolean isMultiFormat;
+  private final HoodieConfig hoodieConfig;
 
   /**
    * Constructs an instance of the reader context that will read data into Avro records.
@@ -79,7 +80,24 @@ public class HoodieAvroReaderContext extends HoodieReaderContext<IndexedRecord>
       HoodieTableConfig tableConfig,
       Option<InstantRange> instantRangeOpt,
       Option<Predicate> filterOpt) {
-    this(storageConfiguration, tableConfig, instantRangeOpt, filterOpt, Collections.emptyMap());
+    this(storageConfiguration, tableConfig, instantRangeOpt, filterOpt, Collections.emptyMap(), tableConfig.getPayloadClass(), new HoodieConfig());
+  }
+
+  /**
+   * Constructs an instance of the reader context that will read data into Avro records with custom configuration.
+   * @param storageConfiguration the storage configuration to use for reading files
+   * @param tableConfig the configuration of the Hudi table being read
+   * @param instantRangeOpt the set of valid instants for this read
+   * @param filterOpt an optional filter to apply on the record keys
+   * @param hoodieConfig the configuration that contains cache and other settings
+   */
+  public HoodieAvroReaderContext(
+      StorageConfiguration<?> storageConfiguration,
+      HoodieTableConfig tableConfig,
+      Option<InstantRange> instantRangeOpt,
+      Option<Predicate> filterOpt,
+      HoodieConfig hoodieConfig) {
+    this(storageConfiguration, tableConfig, instantRangeOpt, filterOpt, Collections.emptyMap(), tableConfig.getPayloadClass(), hoodieConfig);
   }
 
   /**
@@ -98,7 +116,7 @@ public class HoodieAvroReaderContext extends HoodieReaderContext<IndexedRecord>
       Option<InstantRange> instantRangeOpt,
       Option<Predicate> filterOpt,
       Map<StoragePath, HoodieAvroFileReader> reusableFileReaders) {
-    this(storageConfiguration, tableConfig, instantRangeOpt, filterOpt, reusableFileReaders, tableConfig.getPayloadClass());
+    this(storageConfiguration, tableConfig, instantRangeOpt, filterOpt, reusableFileReaders, tableConfig.getPayloadClass(), new HoodieConfig());
   }
 
   /**
@@ -112,7 +130,7 @@ public class HoodieAvroReaderContext extends HoodieReaderContext<IndexedRecord>
       StorageConfiguration<?> storageConfiguration,
       HoodieTableConfig tableConfig,
       String payloadClassName) {
-    this(storageConfiguration, tableConfig, Option.empty(), Option.empty(), Collections.emptyMap(), payloadClassName);
+    this(storageConfiguration, tableConfig, Option.empty(), Option.empty(), Collections.emptyMap(), payloadClassName, new HoodieConfig());
   }
 
   private HoodieAvroReaderContext(
@@ -121,10 +139,12 @@ public class HoodieAvroReaderContext extends HoodieReaderContext<IndexedRecord>
       Option<InstantRange> instantRangeOpt,
       Option<Predicate> filterOpt,
       Map<StoragePath, HoodieAvroFileReader> reusableFileReaders,
-      String payloadClassName) {
+      String payloadClassName,
+      HoodieConfig hoodieConfig) {
     super(storageConfiguration, tableConfig, instantRangeOpt, filterOpt, new AvroRecordContext(tableConfig, payloadClassName));
     this.reusableFileReaders = reusableFileReaders;
     this.isMultiFormat = tableConfig.isMultipleBaseFileFormatsEnabled();
+    this.hoodieConfig = hoodieConfig;
   }
 
   @Override
@@ -152,7 +172,7 @@ public class HoodieAvroReaderContext extends HoodieReaderContext<IndexedRecord>
     } else {
       HoodieFileFormat fileFormat = isMultiFormat && !isLogFile ? HoodieFileFormat.fromFileExtension(filePath.getFileExtension()) : baseFileFormat;
       reader = (HoodieAvroFileReader) HoodieIOFactory.getIOFactory(storage)
-          .getReaderFactory(HoodieRecord.HoodieRecordType.AVRO).getFileReader(new HoodieConfig(),
+          .getReaderFactory(HoodieRecord.HoodieRecordType.AVRO).getFileReader(hoodieConfig,
               filePath, fileFormat, Option.empty());
     }
     if (keyFilterOpt.isEmpty()) {
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala
index a4ad93c02fee..db791fc123c3 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala
@@ -24,6 +24,9 @@ import org.apache.hudi.avro.HoodieAvroUtils
 import org.apache.hudi.client.utils.SparkInternalSchemaConverter
 import org.apache.hudi.common.config.{ConfigProperty, HoodieConfig, HoodieMetadataConfig}
 import org.apache.hudi.common.config.HoodieReaderConfig.USE_NATIVE_HFILE_READER
+import org.apache.hudi.common.config.HoodieReaderConfig.HFILE_BLOCK_CACHE_ENABLED
+import org.apache.hudi.common.config.HoodieReaderConfig.HFILE_BLOCK_CACHE_SIZE
+import org.apache.hudi.common.config.HoodieReaderConfig.HFILE_BLOCK_CACHE_TTL_MINUTES
 import org.apache.hudi.common.fs.FSUtils
 import org.apache.hudi.common.fs.FSUtils.getRelativePartitionPath
 import org.apache.hudi.common.model.{FileSlice, HoodieFileFormat, HoodieRecord}
@@ -843,6 +846,15 @@ object HoodieBaseRelation extends SparkAdapterSupport {
       val hoodieConfig = new HoodieConfig()
       hoodieConfig.setValue(USE_NATIVE_HFILE_READER,
         options.getOrElse(USE_NATIVE_HFILE_READER.key(), USE_NATIVE_HFILE_READER.defaultValue().toString))
+
+      // Add HFile cache configurations from options
+      hoodieConfig.setValue(HFILE_BLOCK_CACHE_ENABLED,
+        options.getOrElse(HFILE_BLOCK_CACHE_ENABLED.key(), HFILE_BLOCK_CACHE_TTL_MINUTES.defaultValue()))
+      hoodieConfig.setValue(HFILE_BLOCK_CACHE_SIZE,
+        options.getOrElse(HFILE_BLOCK_CACHE_SIZE.key(), HFILE_BLOCK_CACHE_TTL_MINUTES.defaultValue()))
+      hoodieConfig.setValue(HFILE_BLOCK_CACHE_TTL_MINUTES,
+        options.getOrElse(HFILE_BLOCK_CACHE_TTL_MINUTES.key(), HFILE_BLOCK_CACHE_TTL_MINUTES.defaultValue()))
+
       val reader = new HoodieSparkIOFactory(
         new HoodieHadoopStorage(filePath, HadoopFSUtils.getStorageConf(hadoopConf)))
         .getReaderFactory(HoodieRecordType.AVRO)