You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vh...@apache.org on 2025/09/17 09:17:49 UTC
(hudi) 01/01: Minor add configurable hfile config
This is an automated email from the ASF dual-hosted git repository.
vhs pushed a commit to branch minor-add-configurable-hfile-config
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit ddf7f9a9c6a67388febd321e77ee78d86a577173
Author: voon <vo...@gmail.com>
AuthorDate: Wed Sep 17 17:17:29 2025 +0800
Minor add configurable hfile config
---
.../apache/hudi/avro/HoodieAvroReaderContext.java | 30 ++++++++++++++++++----
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 12 +++++++++
2 files changed, 37 insertions(+), 5 deletions(-)
diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroReaderContext.java b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroReaderContext.java
index c90dc0186741..4c5bf59dccd1 100644
--- a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroReaderContext.java
+++ b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroReaderContext.java
@@ -66,6 +66,7 @@ import static org.apache.hudi.common.util.ValidationUtils.checkState;
public class HoodieAvroReaderContext extends HoodieReaderContext<IndexedRecord> {
private final Map<StoragePath, HoodieAvroFileReader> reusableFileReaders;
private final boolean isMultiFormat;
+ private final HoodieConfig hoodieConfig;
/**
* Constructs an instance of the reader context that will read data into Avro records.
@@ -79,7 +80,24 @@ public class HoodieAvroReaderContext extends HoodieReaderContext<IndexedRecord>
HoodieTableConfig tableConfig,
Option<InstantRange> instantRangeOpt,
Option<Predicate> filterOpt) {
- this(storageConfiguration, tableConfig, instantRangeOpt, filterOpt, Collections.emptyMap());
+ this(storageConfiguration, tableConfig, instantRangeOpt, filterOpt, Collections.emptyMap(), tableConfig.getPayloadClass(), new HoodieConfig());
+ }
+
+ /**
+ * Constructs an instance of the reader context that will read data into Avro records with custom configuration.
+ * @param storageConfiguration the storage configuration to use for reading files
+ * @param tableConfig the configuration of the Hudi table being read
+ * @param instantRangeOpt the set of valid instants for this read
+ * @param filterOpt an optional filter to apply on the record keys
+ * @param hoodieConfig the configuration that contains cache and other settings
+ */
+ public HoodieAvroReaderContext(
+ StorageConfiguration<?> storageConfiguration,
+ HoodieTableConfig tableConfig,
+ Option<InstantRange> instantRangeOpt,
+ Option<Predicate> filterOpt,
+ HoodieConfig hoodieConfig) {
+ this(storageConfiguration, tableConfig, instantRangeOpt, filterOpt, Collections.emptyMap(), tableConfig.getPayloadClass(), hoodieConfig);
}
/**
@@ -98,7 +116,7 @@ public class HoodieAvroReaderContext extends HoodieReaderContext<IndexedRecord>
Option<InstantRange> instantRangeOpt,
Option<Predicate> filterOpt,
Map<StoragePath, HoodieAvroFileReader> reusableFileReaders) {
- this(storageConfiguration, tableConfig, instantRangeOpt, filterOpt, reusableFileReaders, tableConfig.getPayloadClass());
+ this(storageConfiguration, tableConfig, instantRangeOpt, filterOpt, reusableFileReaders, tableConfig.getPayloadClass(), new HoodieConfig());
}
/**
@@ -112,7 +130,7 @@ public class HoodieAvroReaderContext extends HoodieReaderContext<IndexedRecord>
StorageConfiguration<?> storageConfiguration,
HoodieTableConfig tableConfig,
String payloadClassName) {
- this(storageConfiguration, tableConfig, Option.empty(), Option.empty(), Collections.emptyMap(), payloadClassName);
+ this(storageConfiguration, tableConfig, Option.empty(), Option.empty(), Collections.emptyMap(), payloadClassName, new HoodieConfig());
}
private HoodieAvroReaderContext(
@@ -121,10 +139,12 @@ public class HoodieAvroReaderContext extends HoodieReaderContext<IndexedRecord>
Option<InstantRange> instantRangeOpt,
Option<Predicate> filterOpt,
Map<StoragePath, HoodieAvroFileReader> reusableFileReaders,
- String payloadClassName) {
+ String payloadClassName,
+ HoodieConfig hoodieConfig) {
super(storageConfiguration, tableConfig, instantRangeOpt, filterOpt, new AvroRecordContext(tableConfig, payloadClassName));
this.reusableFileReaders = reusableFileReaders;
this.isMultiFormat = tableConfig.isMultipleBaseFileFormatsEnabled();
+ this.hoodieConfig = hoodieConfig;
}
@Override
@@ -152,7 +172,7 @@ public class HoodieAvroReaderContext extends HoodieReaderContext<IndexedRecord>
} else {
HoodieFileFormat fileFormat = isMultiFormat && !isLogFile ? HoodieFileFormat.fromFileExtension(filePath.getFileExtension()) : baseFileFormat;
reader = (HoodieAvroFileReader) HoodieIOFactory.getIOFactory(storage)
- .getReaderFactory(HoodieRecord.HoodieRecordType.AVRO).getFileReader(new HoodieConfig(),
+ .getReaderFactory(HoodieRecord.HoodieRecordType.AVRO).getFileReader(hoodieConfig,
filePath, fileFormat, Option.empty());
}
if (keyFilterOpt.isEmpty()) {
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala
index a4ad93c02fee..db791fc123c3 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala
@@ -24,6 +24,9 @@ import org.apache.hudi.avro.HoodieAvroUtils
import org.apache.hudi.client.utils.SparkInternalSchemaConverter
import org.apache.hudi.common.config.{ConfigProperty, HoodieConfig, HoodieMetadataConfig}
import org.apache.hudi.common.config.HoodieReaderConfig.USE_NATIVE_HFILE_READER
+import org.apache.hudi.common.config.HoodieReaderConfig.HFILE_BLOCK_CACHE_ENABLED
+import org.apache.hudi.common.config.HoodieReaderConfig.HFILE_BLOCK_CACHE_SIZE
+import org.apache.hudi.common.config.HoodieReaderConfig.HFILE_BLOCK_CACHE_TTL_MINUTES
import org.apache.hudi.common.fs.FSUtils
import org.apache.hudi.common.fs.FSUtils.getRelativePartitionPath
import org.apache.hudi.common.model.{FileSlice, HoodieFileFormat, HoodieRecord}
@@ -843,6 +846,15 @@ object HoodieBaseRelation extends SparkAdapterSupport {
val hoodieConfig = new HoodieConfig()
hoodieConfig.setValue(USE_NATIVE_HFILE_READER,
options.getOrElse(USE_NATIVE_HFILE_READER.key(), USE_NATIVE_HFILE_READER.defaultValue().toString))
+
+ // Add HFile cache configurations from options
+ hoodieConfig.setValue(HFILE_BLOCK_CACHE_ENABLED,
+ options.getOrElse(HFILE_BLOCK_CACHE_ENABLED.key(), HFILE_BLOCK_CACHE_TTL_MINUTES.defaultValue()))
+ hoodieConfig.setValue(HFILE_BLOCK_CACHE_SIZE,
+ options.getOrElse(HFILE_BLOCK_CACHE_SIZE.key(), HFILE_BLOCK_CACHE_TTL_MINUTES.defaultValue()))
+ hoodieConfig.setValue(HFILE_BLOCK_CACHE_TTL_MINUTES,
+ options.getOrElse(HFILE_BLOCK_CACHE_TTL_MINUTES.key(), HFILE_BLOCK_CACHE_TTL_MINUTES.defaultValue()))
+
val reader = new HoodieSparkIOFactory(
new HoodieHadoopStorage(filePath, HadoopFSUtils.getStorageConf(hadoopConf)))
.getReaderFactory(HoodieRecordType.AVRO)